You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ae...@apache.org on 2015/09/29 22:30:15 UTC
[13/50] [abbrv] hadoop git commit: HDFS-8053. Move DFSIn/OutputStream
and related classes to hadoop-hdfs-client. Contributed by Mingliang Liu.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf37d3d8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/HdfsDataOutputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/HdfsDataOutputStream.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/HdfsDataOutputStream.java
deleted file mode 100644
index 745ca7e..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/HdfsDataOutputStream.java
+++ /dev/null
@@ -1,112 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.client;
-
-import java.io.IOException;
-import java.io.OutputStream;
-import java.util.EnumSet;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.crypto.CryptoOutputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.hdfs.DFSOutputStream;
-
-import com.google.common.base.Preconditions;
-
-/**
- * The Hdfs implementation of {@link FSDataOutputStream}.
- */
-@InterfaceAudience.Public
-@InterfaceStability.Evolving
-public class HdfsDataOutputStream extends FSDataOutputStream {
- public HdfsDataOutputStream(DFSOutputStream out, FileSystem.Statistics stats,
- long startPosition) throws IOException {
- super(out, stats, startPosition);
- }
-
- public HdfsDataOutputStream(DFSOutputStream out, FileSystem.Statistics stats
- ) throws IOException {
- this(out, stats, 0L);
- }
-
- public HdfsDataOutputStream(CryptoOutputStream out, FileSystem.Statistics stats,
- long startPosition) throws IOException {
- super(out, stats, startPosition);
- Preconditions.checkArgument(out.getWrappedStream() instanceof DFSOutputStream,
- "CryptoOutputStream should wrap a DFSOutputStream");
- }
-
- public HdfsDataOutputStream(CryptoOutputStream out, FileSystem.Statistics stats)
- throws IOException {
- this(out, stats, 0L);
- }
-
- /**
- * Get the actual number of replicas of the current block.
- *
- * This can be different from the designated replication factor of the file
- * because the namenode does not maintain replication for the blocks which are
- * currently being written to. Depending on the configuration, the client may
- * continue to write to a block even if a few datanodes in the write pipeline
- * have failed, or the client may add a new datanodes once a datanode has
- * failed.
- *
- * @return the number of valid replicas of the current block
- */
- public synchronized int getCurrentBlockReplication() throws IOException {
- OutputStream wrappedStream = getWrappedStream();
- if (wrappedStream instanceof CryptoOutputStream) {
- wrappedStream = ((CryptoOutputStream) wrappedStream).getWrappedStream();
- }
- return ((DFSOutputStream) wrappedStream).getCurrentBlockReplication();
- }
-
- /**
- * Sync buffered data to DataNodes (flush to disk devices).
- *
- * @param syncFlags
- * Indicate the detailed semantic and actions of the hsync.
- * @throws IOException
- * @see FSDataOutputStream#hsync()
- */
- public void hsync(EnumSet<SyncFlag> syncFlags) throws IOException {
- OutputStream wrappedStream = getWrappedStream();
- if (wrappedStream instanceof CryptoOutputStream) {
- ((CryptoOutputStream) wrappedStream).flush();
- wrappedStream = ((CryptoOutputStream) wrappedStream).getWrappedStream();
- }
- ((DFSOutputStream) wrappedStream).hsync(syncFlags);
- }
-
- public static enum SyncFlag {
-
- /**
- * When doing sync to DataNodes, also update the metadata (block length) in
- * the NameNode.
- */
- UPDATE_LENGTH,
-
- /**
- * Sync the data to DataNode, close the current block, and allocate a new
- * block
- */
- END_BLOCK;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf37d3d8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/impl/LeaseRenewer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/impl/LeaseRenewer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/impl/LeaseRenewer.java
deleted file mode 100644
index b41e2c3..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/client/impl/LeaseRenewer.java
+++ /dev/null
@@ -1,524 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.client.impl;
-
-import java.io.IOException;
-import java.net.SocketTimeoutException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Comparator;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.HadoopIllegalArgumentException;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.hdfs.DFSClient;
-import org.apache.hadoop.hdfs.DFSOutputStream;
-import org.apache.hadoop.hdfs.protocol.HdfsConstants;
-import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.util.Daemon;
-import org.apache.hadoop.util.StringUtils;
-import org.apache.hadoop.util.Time;
-import com.google.common.annotations.VisibleForTesting;
-
-/**
- * <p>
- * Used by {@link org.apache.hadoop.hdfs.DFSClient} for renewing file-being-written leases
- * on the namenode.
- * When a file is opened for write (create or append),
- * namenode stores a file lease for recording the identity of the writer.
- * The writer (i.e. the DFSClient) is required to renew the lease periodically.
- * When the lease is not renewed before it expires,
- * the namenode considers the writer as failed and then it may either let
- * another writer to obtain the lease or close the file.
- * </p>
- * <p>
- * This class also provides the following functionality:
- * <ul>
- * <li>
- * It maintains a map from (namenode, user) pairs to lease renewers.
- * The same {@link LeaseRenewer} instance is used for renewing lease
- * for all the {@link org.apache.hadoop.hdfs.DFSClient} to the same namenode and the same user.
- * </li>
- * <li>
- * Each renewer maintains a list of {@link org.apache.hadoop.hdfs.DFSClient}.
- * Periodically the leases for all the clients are renewed.
- * A client is removed from the list when the client is closed.
- * </li>
- * <li>
- * A thread per namenode per user is used by the {@link LeaseRenewer}
- * to renew the leases.
- * </li>
- * </ul>
- * </p>
- */
-@InterfaceAudience.Private
-public class LeaseRenewer {
- static final Log LOG = LogFactory.getLog(LeaseRenewer.class);
-
- static final long LEASE_RENEWER_GRACE_DEFAULT = 60*1000L;
- static final long LEASE_RENEWER_SLEEP_DEFAULT = 1000L;
-
- /** Get a {@link LeaseRenewer} instance */
- public static LeaseRenewer getInstance(final String authority,
- final UserGroupInformation ugi, final DFSClient dfsc) throws IOException {
- final LeaseRenewer r = Factory.INSTANCE.get(authority, ugi);
- r.addClient(dfsc);
- return r;
- }
-
- /**
- * A factory for sharing {@link LeaseRenewer} objects
- * among {@link DFSClient} instances
- * so that there is only one renewer per authority per user.
- */
- private static class Factory {
- private static final Factory INSTANCE = new Factory();
-
- private static class Key {
- /** Namenode info */
- final String authority;
- /** User info */
- final UserGroupInformation ugi;
-
- private Key(final String authority, final UserGroupInformation ugi) {
- if (authority == null) {
- throw new HadoopIllegalArgumentException("authority == null");
- } else if (ugi == null) {
- throw new HadoopIllegalArgumentException("ugi == null");
- }
-
- this.authority = authority;
- this.ugi = ugi;
- }
-
- @Override
- public int hashCode() {
- return authority.hashCode() ^ ugi.hashCode();
- }
-
- @Override
- public boolean equals(Object obj) {
- if (obj == this) {
- return true;
- }
- if (obj != null && obj instanceof Key) {
- final Key that = (Key)obj;
- return this.authority.equals(that.authority)
- && this.ugi.equals(that.ugi);
- }
- return false;
- }
-
- @Override
- public String toString() {
- return ugi.getShortUserName() + "@" + authority;
- }
- }
-
- /** A map for per user per namenode renewers. */
- private final Map<Key, LeaseRenewer> renewers = new HashMap<Key, LeaseRenewer>();
-
- /** Get a renewer. */
- private synchronized LeaseRenewer get(final String authority,
- final UserGroupInformation ugi) {
- final Key k = new Key(authority, ugi);
- LeaseRenewer r = renewers.get(k);
- if (r == null) {
- r = new LeaseRenewer(k);
- renewers.put(k, r);
- }
- return r;
- }
-
- /** Remove the given renewer. */
- private synchronized void remove(final LeaseRenewer r) {
- final LeaseRenewer stored = renewers.get(r.factorykey);
- //Since a renewer may expire, the stored renewer can be different.
- if (r == stored) {
- if (!r.clientsRunning()) {
- renewers.remove(r.factorykey);
- }
- }
- }
- }
-
- /** The time in milliseconds that the map became empty. */
- private long emptyTime = Long.MAX_VALUE;
- /** A fixed lease renewal time period in milliseconds */
- private long renewal = HdfsConstants.LEASE_SOFTLIMIT_PERIOD / 2;
-
- /** A daemon for renewing lease */
- private Daemon daemon = null;
- /** Only the daemon with currentId should run. */
- private int currentId = 0;
-
- /**
- * A period in milliseconds that the lease renewer thread should run
- * after the map became empty.
- * In other words,
- * if the map is empty for a time period longer than the grace period,
- * the renewer should terminate.
- */
- private long gracePeriod;
- /**
- * The time period in milliseconds
- * that the renewer sleeps for each iteration.
- */
- private long sleepPeriod;
-
- private final Factory.Key factorykey;
-
- /** A list of clients corresponding to this renewer. */
- private final List<DFSClient> dfsclients = new ArrayList<DFSClient>();
-
- /**
- * A stringified stack trace of the call stack when the Lease Renewer
- * was instantiated. This is only generated if trace-level logging is
- * enabled on this class.
- */
- private final String instantiationTrace;
-
- private LeaseRenewer(Factory.Key factorykey) {
- this.factorykey = factorykey;
- unsyncSetGraceSleepPeriod(LEASE_RENEWER_GRACE_DEFAULT);
-
- if (LOG.isTraceEnabled()) {
- instantiationTrace = StringUtils.stringifyException(
- new Throwable("TRACE"));
- } else {
- instantiationTrace = null;
- }
- }
-
- /** @return the renewal time in milliseconds. */
- private synchronized long getRenewalTime() {
- return renewal;
- }
-
- /** Used for testing only. */
- @VisibleForTesting
- public synchronized void setRenewalTime(final long renewal) {
- this.renewal = renewal;
- }
-
- /** Add a client. */
- private synchronized void addClient(final DFSClient dfsc) {
- for(DFSClient c : dfsclients) {
- if (c == dfsc) {
- //client already exists, nothing to do.
- return;
- }
- }
- //client not found, add it
- dfsclients.add(dfsc);
-
- //update renewal time
- final int hdfsTimeout = dfsc.getConf().getHdfsTimeout();
- if (hdfsTimeout > 0) {
- final long half = hdfsTimeout/2;
- if (half < renewal) {
- this.renewal = half;
- }
- }
- }
-
- private synchronized boolean clientsRunning() {
- for(Iterator<DFSClient> i = dfsclients.iterator(); i.hasNext(); ) {
- if (!i.next().isClientRunning()) {
- i.remove();
- }
- }
- return !dfsclients.isEmpty();
- }
-
- private synchronized long getSleepPeriod() {
- return sleepPeriod;
- }
-
- /** Set the grace period and adjust the sleep period accordingly. */
- synchronized void setGraceSleepPeriod(final long gracePeriod) {
- unsyncSetGraceSleepPeriod(gracePeriod);
- }
-
- private void unsyncSetGraceSleepPeriod(final long gracePeriod) {
- if (gracePeriod < 100L) {
- throw new HadoopIllegalArgumentException(gracePeriod
- + " = gracePeriod < 100ms is too small.");
- }
- this.gracePeriod = gracePeriod;
- final long half = gracePeriod/2;
- this.sleepPeriod = half < LEASE_RENEWER_SLEEP_DEFAULT?
- half: LEASE_RENEWER_SLEEP_DEFAULT;
- }
-
- /** Is the daemon running? */
- synchronized boolean isRunning() {
- return daemon != null && daemon.isAlive();
- }
-
- /** Does this renewer have nothing to renew? */
- public boolean isEmpty() {
- return dfsclients.isEmpty();
- }
-
- /** Used only by tests */
- synchronized String getDaemonName() {
- return daemon.getName();
- }
-
- /** Is the empty period longer than the grace period? */
- private synchronized boolean isRenewerExpired() {
- return emptyTime != Long.MAX_VALUE
- && Time.monotonicNow() - emptyTime > gracePeriod;
- }
-
- public synchronized void put(final long inodeId, final DFSOutputStream out,
- final DFSClient dfsc) {
- if (dfsc.isClientRunning()) {
- if (!isRunning() || isRenewerExpired()) {
- //start a new deamon with a new id.
- final int id = ++currentId;
- daemon = new Daemon(new Runnable() {
- @Override
- public void run() {
- try {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Lease renewer daemon for " + clientsString()
- + " with renew id " + id + " started");
- }
- LeaseRenewer.this.run(id);
- } catch(InterruptedException e) {
- if (LOG.isDebugEnabled()) {
- LOG.debug(LeaseRenewer.this.getClass().getSimpleName()
- + " is interrupted.", e);
- }
- } finally {
- synchronized(LeaseRenewer.this) {
- Factory.INSTANCE.remove(LeaseRenewer.this);
- }
- if (LOG.isDebugEnabled()) {
- LOG.debug("Lease renewer daemon for " + clientsString()
- + " with renew id " + id + " exited");
- }
- }
- }
-
- @Override
- public String toString() {
- return String.valueOf(LeaseRenewer.this);
- }
- });
- daemon.start();
- }
- dfsc.putFileBeingWritten(inodeId, out);
- emptyTime = Long.MAX_VALUE;
- }
- }
-
- @VisibleForTesting
- synchronized void setEmptyTime(long time) {
- emptyTime = time;
- }
-
- /** Close a file. */
- public void closeFile(final long inodeId, final DFSClient dfsc) {
- dfsc.removeFileBeingWritten(inodeId);
-
- synchronized(this) {
- if (dfsc.isFilesBeingWrittenEmpty()) {
- dfsclients.remove(dfsc);
- }
- //update emptyTime if necessary
- if (emptyTime == Long.MAX_VALUE) {
- for(DFSClient c : dfsclients) {
- if (!c.isFilesBeingWrittenEmpty()) {
- //found a non-empty file-being-written map
- return;
- }
- }
- //discover the first time that all file-being-written maps are empty.
- emptyTime = Time.monotonicNow();
- }
- }
- }
-
- /** Close the given client. */
- public synchronized void closeClient(final DFSClient dfsc) {
- dfsclients.remove(dfsc);
- if (dfsclients.isEmpty()) {
- if (!isRunning() || isRenewerExpired()) {
- Factory.INSTANCE.remove(LeaseRenewer.this);
- return;
- }
- if (emptyTime == Long.MAX_VALUE) {
- //discover the first time that the client list is empty.
- emptyTime = Time.monotonicNow();
- }
- }
-
- //update renewal time
- if (renewal == dfsc.getConf().getHdfsTimeout()/2) {
- long min = HdfsConstants.LEASE_SOFTLIMIT_PERIOD;
- for(DFSClient c : dfsclients) {
- final int timeout = c.getConf().getHdfsTimeout();
- if (timeout > 0 && timeout < min) {
- min = timeout;
- }
- }
- renewal = min/2;
- }
- }
-
- public void interruptAndJoin() throws InterruptedException {
- Daemon daemonCopy = null;
- synchronized (this) {
- if (isRunning()) {
- daemon.interrupt();
- daemonCopy = daemon;
- }
- }
-
- if (daemonCopy != null) {
- if(LOG.isDebugEnabled()) {
- LOG.debug("Wait for lease checker to terminate");
- }
- daemonCopy.join();
- }
- }
-
- private void renew() throws IOException {
- final List<DFSClient> copies;
- synchronized(this) {
- copies = new ArrayList<DFSClient>(dfsclients);
- }
- //sort the client names for finding out repeated names.
- Collections.sort(copies, new Comparator<DFSClient>() {
- @Override
- public int compare(final DFSClient left, final DFSClient right) {
- return left.getClientName().compareTo(right.getClientName());
- }
- });
- String previousName = "";
- for(int i = 0; i < copies.size(); i++) {
- final DFSClient c = copies.get(i);
- //skip if current client name is the same as the previous name.
- if (!c.getClientName().equals(previousName)) {
- if (!c.renewLease()) {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Did not renew lease for client " +
- c);
- }
- continue;
- }
- previousName = c.getClientName();
- if (LOG.isDebugEnabled()) {
- LOG.debug("Lease renewed for client " + previousName);
- }
- }
- }
- }
-
- /**
- * Periodically check in with the namenode and renew all the leases
- * when the lease period is half over.
- */
- private void run(final int id) throws InterruptedException {
- for(long lastRenewed = Time.monotonicNow(); !Thread.interrupted();
- Thread.sleep(getSleepPeriod())) {
- final long elapsed = Time.monotonicNow() - lastRenewed;
- if (elapsed >= getRenewalTime()) {
- try {
- renew();
- if (LOG.isDebugEnabled()) {
- LOG.debug("Lease renewer daemon for " + clientsString()
- + " with renew id " + id + " executed");
- }
- lastRenewed = Time.monotonicNow();
- } catch (SocketTimeoutException ie) {
- LOG.warn("Failed to renew lease for " + clientsString() + " for "
- + (elapsed/1000) + " seconds. Aborting ...", ie);
- synchronized (this) {
- while (!dfsclients.isEmpty()) {
- DFSClient dfsClient = dfsclients.get(0);
- dfsClient.closeAllFilesBeingWritten(true);
- closeClient(dfsClient);
- }
- //Expire the current LeaseRenewer thread.
- emptyTime = 0;
- }
- break;
- } catch (IOException ie) {
- LOG.warn("Failed to renew lease for " + clientsString() + " for "
- + (elapsed/1000) + " seconds. Will retry shortly ...", ie);
- }
- }
-
- synchronized(this) {
- if (id != currentId || isRenewerExpired()) {
- if (LOG.isDebugEnabled()) {
- if (id != currentId) {
- LOG.debug("Lease renewer daemon for " + clientsString()
- + " with renew id " + id + " is not current");
- } else {
- LOG.debug("Lease renewer daemon for " + clientsString()
- + " with renew id " + id + " expired");
- }
- }
- //no longer the current daemon or expired
- return;
- }
-
- // if no clients are in running state or there is no more clients
- // registered with this renewer, stop the daemon after the grace
- // period.
- if (!clientsRunning() && emptyTime == Long.MAX_VALUE) {
- emptyTime = Time.monotonicNow();
- }
- }
- }
- }
-
- @Override
- public String toString() {
- String s = getClass().getSimpleName() + ":" + factorykey;
- if (LOG.isTraceEnabled()) {
- return s + ", clients=" + clientsString()
- + ", created at " + instantiationTrace;
- }
- return s;
- }
-
- /** Get the names of all clients */
- private synchronized String clientsString() {
- if (dfsclients.isEmpty()) {
- return "[]";
- } else {
- final StringBuilder b = new StringBuilder("[").append(
- dfsclients.get(0).getClientName());
- for(int i = 1; i < dfsclients.size(); i++) {
- b.append(", ").append(dfsclients.get(i).getClientName());
- }
- return b.append("]").toString();
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf37d3d8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/inotify/MissingEventsException.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/inotify/MissingEventsException.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/inotify/MissingEventsException.java
deleted file mode 100644
index e4b51c5..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/inotify/MissingEventsException.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hdfs.inotify;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-
-@InterfaceAudience.Public
-@InterfaceStability.Evolving
-public class MissingEventsException extends Exception {
- private static final long serialVersionUID = 1L;
-
- private long expectedTxid;
- private long actualTxid;
-
- public MissingEventsException() {}
-
- public MissingEventsException(long expectedTxid, long actualTxid) {
- this.expectedTxid = expectedTxid;
- this.actualTxid = actualTxid;
- }
-
- public long getExpectedTxid() {
- return expectedTxid;
- }
-
- public long getActualTxid() {
- return actualTxid;
- }
-
- @Override
- public String toString() {
- return "We expected the next batch of events to start with transaction ID "
- + expectedTxid + ", but it instead started with transaction ID " +
- actualTxid + ". Most likely the intervening transactions were cleaned "
- + "up as part of checkpointing.";
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf37d3d8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/AclException.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/AclException.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/AclException.java
deleted file mode 100644
index 1210999..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/AclException.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.protocol;
-
-import java.io.IOException;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-
-/**
- * Indicates a failure manipulating an ACL.
- */
-@InterfaceAudience.Private
-public class AclException extends IOException {
- private static final long serialVersionUID = 1L;
-
- /**
- * Creates a new AclException.
- *
- * @param message String message
- */
- public AclException(String message) {
- super(message);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf37d3d8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirectiveIterator.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirectiveIterator.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirectiveIterator.java
deleted file mode 100644
index 923cdb4..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CacheDirectiveIterator.java
+++ /dev/null
@@ -1,130 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hdfs.protocol;
-
-import java.io.IOException;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.fs.BatchedRemoteIterator;
-import org.apache.hadoop.fs.InvalidRequestException;
-import org.apache.hadoop.ipc.RemoteException;
-import org.apache.htrace.Sampler;
-import org.apache.htrace.Trace;
-import org.apache.htrace.TraceScope;
-
-import com.google.common.base.Preconditions;
-
-/**
- * CacheDirectiveIterator is a remote iterator that iterates cache directives.
- * It supports retrying in case of namenode failover.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-public class CacheDirectiveIterator
- extends BatchedRemoteIterator<Long, CacheDirectiveEntry> {
-
- private CacheDirectiveInfo filter;
- private final ClientProtocol namenode;
- private final Sampler<?> traceSampler;
-
- public CacheDirectiveIterator(ClientProtocol namenode,
- CacheDirectiveInfo filter, Sampler<?> traceSampler) {
- super(0L);
- this.namenode = namenode;
- this.filter = filter;
- this.traceSampler = traceSampler;
- }
-
- private static CacheDirectiveInfo removeIdFromFilter(CacheDirectiveInfo filter) {
- CacheDirectiveInfo.Builder builder = new CacheDirectiveInfo.Builder(filter);
- builder.setId(null);
- return builder.build();
- }
-
- /**
- * Used for compatibility when communicating with a server version that
- * does not support filtering directives by ID.
- */
- private static class SingleEntry implements
- BatchedEntries<CacheDirectiveEntry> {
-
- private final CacheDirectiveEntry entry;
-
- public SingleEntry(final CacheDirectiveEntry entry) {
- this.entry = entry;
- }
-
- @Override
- public CacheDirectiveEntry get(int i) {
- if (i > 0) {
- return null;
- }
- return entry;
- }
-
- @Override
- public int size() {
- return 1;
- }
-
- @Override
- public boolean hasMore() {
- return false;
- }
- }
-
- @Override
- public BatchedEntries<CacheDirectiveEntry> makeRequest(Long prevKey)
- throws IOException {
- BatchedEntries<CacheDirectiveEntry> entries = null;
- TraceScope scope = Trace.startSpan("listCacheDirectives", traceSampler);
- try {
- entries = namenode.listCacheDirectives(prevKey, filter);
- } catch (IOException e) {
- if (e.getMessage().contains("Filtering by ID is unsupported")) {
- // Retry case for old servers, do the filtering client-side
- long id = filter.getId();
- filter = removeIdFromFilter(filter);
- // Using id - 1 as prevId should get us a window containing the id
- // This is somewhat brittle, since it depends on directives being
- // returned in order of ascending ID.
- entries = namenode.listCacheDirectives(id - 1, filter);
- for (int i=0; i<entries.size(); i++) {
- CacheDirectiveEntry entry = entries.get(i);
- if (entry.getInfo().getId().equals((Long)id)) {
- return new SingleEntry(entry);
- }
- }
- throw new RemoteException(InvalidRequestException.class.getName(),
- "Did not find requested id " + id);
- }
- throw e;
- } finally {
- scope.close();
- }
- Preconditions.checkNotNull(entries);
- return entries;
- }
-
- @Override
- public Long elementToPrevKey(CacheDirectiveEntry entry) {
- return entry.getInfo().getId();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf37d3d8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CachePoolIterator.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CachePoolIterator.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CachePoolIterator.java
deleted file mode 100644
index e9481f7..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/CachePoolIterator.java
+++ /dev/null
@@ -1,63 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hdfs.protocol;
-
-import java.io.IOException;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.fs.BatchedRemoteIterator;
-import org.apache.htrace.Sampler;
-import org.apache.htrace.Trace;
-import org.apache.htrace.TraceScope;
-
-/**
- * CachePoolIterator is a remote iterator that iterates cache pools.
- * It supports retrying in case of namenode failover.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-public class CachePoolIterator
- extends BatchedRemoteIterator<String, CachePoolEntry> {
-
- private final ClientProtocol namenode;
- private final Sampler traceSampler;
-
- public CachePoolIterator(ClientProtocol namenode, Sampler traceSampler) {
- super("");
- this.namenode = namenode;
- this.traceSampler = traceSampler;
- }
-
- @Override
- public BatchedEntries<CachePoolEntry> makeRequest(String prevKey)
- throws IOException {
- TraceScope scope = Trace.startSpan("listCachePools", traceSampler);
- try {
- return namenode.listCachePools(prevKey);
- } finally {
- scope.close();
- }
- }
-
- @Override
- public String elementToPrevKey(CachePoolEntry entry) {
- return entry.getInfo().getPoolName();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf37d3d8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/EncryptionZoneIterator.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/EncryptionZoneIterator.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/EncryptionZoneIterator.java
deleted file mode 100644
index 0141215..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/EncryptionZoneIterator.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hdfs.protocol;
-
-import java.io.IOException;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.fs.BatchedRemoteIterator;
-import org.apache.htrace.Sampler;
-import org.apache.htrace.Trace;
-import org.apache.htrace.TraceScope;
-
-/**
- * EncryptionZoneIterator is a remote iterator that iterates over encryption
- * zones. It supports retrying in case of namenode failover.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-public class EncryptionZoneIterator
- extends BatchedRemoteIterator<Long, EncryptionZone> {
-
- private final ClientProtocol namenode;
- private final Sampler<?> traceSampler;
-
- public EncryptionZoneIterator(ClientProtocol namenode,
- Sampler<?> traceSampler) {
- super(Long.valueOf(0));
- this.namenode = namenode;
- this.traceSampler = traceSampler;
- }
-
- @Override
- public BatchedEntries<EncryptionZone> makeRequest(Long prevId)
- throws IOException {
- TraceScope scope = Trace.startSpan("listEncryptionZones", traceSampler);
- try {
- return namenode.listEncryptionZones(prevId);
- } finally {
- scope.close();
- }
- }
-
- @Override
- public Long elementToPrevKey(EncryptionZone entry) {
- return entry.getId();
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf37d3d8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/QuotaByStorageTypeExceededException.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/QuotaByStorageTypeExceededException.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/QuotaByStorageTypeExceededException.java
deleted file mode 100644
index 25084c7..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/QuotaByStorageTypeExceededException.java
+++ /dev/null
@@ -1,56 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hdfs.protocol;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.fs.StorageType;
-
-import static org.apache.hadoop.util.StringUtils.TraditionalBinaryPrefix.long2String;
-
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-public class QuotaByStorageTypeExceededException extends QuotaExceededException {
- protected static final long serialVersionUID = 1L;
- protected StorageType type;
-
- public QuotaByStorageTypeExceededException() {}
-
- public QuotaByStorageTypeExceededException(String msg) {
- super(msg);
- }
-
- public QuotaByStorageTypeExceededException(long quota, long count, StorageType type) {
- super(quota, count);
- this.type = type;
- }
-
- @Override
- public String getMessage() {
- String msg = super.getMessage();
- if (msg == null) {
- return "Quota by storage type : " + type.toString() +
- " on path : " + (pathName==null ? "": pathName) +
- " is exceeded. quota = " + long2String(quota, "B", 2) +
- " but space consumed = " + long2String(count, "B", 2);
- } else {
- return msg;
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf37d3d8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/UnresolvedPathException.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/UnresolvedPathException.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/UnresolvedPathException.java
deleted file mode 100644
index 03fb704..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/UnresolvedPathException.java
+++ /dev/null
@@ -1,87 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hdfs.protocol;
-
-import java.io.IOException;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.fs.UnresolvedLinkException;
-import org.apache.hadoop.fs.Path;
-
-/**
- * Thrown when a symbolic link is encountered in a path.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-public final class UnresolvedPathException extends UnresolvedLinkException {
- private static final long serialVersionUID = 1L;
- private String path; // The path containing the link
- private String preceding; // The path part preceding the link
- private String remainder; // The path part following the link
- private String linkTarget; // The link's target
-
- /**
- * Used by RemoteException to instantiate an UnresolvedPathException.
- */
- public UnresolvedPathException(String msg) {
- super(msg);
- }
-
- public UnresolvedPathException(String path, String preceding,
- String remainder, String linkTarget) {
- this.path = path;
- this.preceding = preceding;
- this.remainder = remainder;
- this.linkTarget = linkTarget;
- }
-
- /**
- * Return a path with the link resolved with the target.
- */
- public Path getResolvedPath() throws IOException {
- // If the path is absolute we cam throw out the preceding part and
- // just append the remainder to the target, otherwise append each
- // piece to resolve the link in path.
- boolean noRemainder = (remainder == null || "".equals(remainder));
- Path target = new Path(linkTarget);
- if (target.isUriPathAbsolute()) {
- return noRemainder ? target : new Path(target, remainder);
- } else {
- return noRemainder
- ? new Path(preceding, target)
- : new Path(new Path(preceding, linkTarget), remainder);
- }
- }
-
- @Override
- public String getMessage() {
- String msg = super.getMessage();
- if (msg != null) {
- return msg;
- }
- String myMsg = "Unresolved path " + path;
- try {
- return getResolvedPath().toString();
- } catch (IOException e) {
- // Ignore
- }
- return myMsg;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf37d3d8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/ReplaceDatanodeOnFailure.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/ReplaceDatanodeOnFailure.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/ReplaceDatanodeOnFailure.java
deleted file mode 100644
index c69986a..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocol/datatransfer/ReplaceDatanodeOnFailure.java
+++ /dev/null
@@ -1,200 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.protocol.datatransfer;
-
-import org.apache.hadoop.HadoopIllegalArgumentException;
-import org.apache.hadoop.classification.InterfaceAudience;
-import org.apache.hadoop.classification.InterfaceStability;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.client.HdfsClientConfigKeys;
-import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
-
-/**
- * The setting of replace-datanode-on-failure feature.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-public class ReplaceDatanodeOnFailure {
- /** The replacement policies */
- public enum Policy {
- /** The feature is disabled in the entire site. */
- DISABLE(Condition.FALSE),
- /** Never add a new datanode. */
- NEVER(Condition.FALSE),
- /** @see ReplaceDatanodeOnFailure.Condition#DEFAULT */
- DEFAULT(Condition.DEFAULT),
- /** Always add a new datanode when an existing datanode is removed. */
- ALWAYS(Condition.TRUE);
-
- private final Condition condition;
-
- private Policy(Condition condition) {
- this.condition = condition;
- }
-
- Condition getCondition() {
- return condition;
- }
- }
-
- /** Datanode replacement condition */
- private static interface Condition {
- /** Return true unconditionally. */
- static final Condition TRUE = new Condition() {
- @Override
- public boolean satisfy(short replication, DatanodeInfo[] existings,
- int nExistings, boolean isAppend, boolean isHflushed) {
- return true;
- }
- };
-
- /** Return false unconditionally. */
- static final Condition FALSE = new Condition() {
- @Override
- public boolean satisfy(short replication, DatanodeInfo[] existings,
- int nExistings, boolean isAppend, boolean isHflushed) {
- return false;
- }
- };
-
- /**
- * DEFAULT condition:
- * Let r be the replication number.
- * Let n be the number of existing datanodes.
- * Add a new datanode only if r >= 3 and either
- * (1) floor(r/2) >= n; or
- * (2) r > n and the block is hflushed/appended.
- */
- static final Condition DEFAULT = new Condition() {
- @Override
- public boolean satisfy(final short replication,
- final DatanodeInfo[] existings, final int n, final boolean isAppend,
- final boolean isHflushed) {
- if (replication < 3) {
- return false;
- } else {
- if (n <= (replication/2)) {
- return true;
- } else {
- return isAppend || isHflushed;
- }
- }
- }
- };
-
- /** Is the condition satisfied? */
- public boolean satisfy(short replication, DatanodeInfo[] existings,
- int nExistings, boolean isAppend, boolean isHflushed);
- }
-
- private final Policy policy;
- private final boolean bestEffort;
-
- public ReplaceDatanodeOnFailure(Policy policy, boolean bestEffort) {
- this.policy = policy;
- this.bestEffort = bestEffort;
- }
-
- /** Check if the feature is enabled. */
- public void checkEnabled() {
- if (policy == Policy.DISABLE) {
- throw new UnsupportedOperationException(
- "This feature is disabled. Please refer to "
- + HdfsClientConfigKeys.BlockWrite.ReplaceDatanodeOnFailure.ENABLE_KEY
- + " configuration property.");
- }
- }
-
- /**
- * Best effort means that the client will try to replace the failed datanode
- * (provided that the policy is satisfied), however, it will continue the
- * write operation in case that the datanode replacement also fails.
- *
- * @return Suppose the datanode replacement fails.
- * false: An exception should be thrown so that the write will fail.
- * true : The write should be resumed with the remaining datandoes.
- */
- public boolean isBestEffort() {
- return bestEffort;
- }
-
- /** Does it need a replacement according to the policy? */
- public boolean satisfy(
- final short replication, final DatanodeInfo[] existings,
- final boolean isAppend, final boolean isHflushed) {
- final int n = existings == null? 0: existings.length;
- if (n == 0 || n >= replication) {
- //don't need to add datanode for any policy.
- return false;
- } else {
- return policy.getCondition().satisfy(
- replication, existings, n, isAppend, isHflushed);
- }
- }
-
- @Override
- public String toString() {
- return policy.toString();
- }
-
- /** Get the setting from configuration. */
- public static ReplaceDatanodeOnFailure get(final Configuration conf) {
- final Policy policy = getPolicy(conf);
- final boolean bestEffort = conf.getBoolean(
- HdfsClientConfigKeys.BlockWrite.ReplaceDatanodeOnFailure.BEST_EFFORT_KEY,
- HdfsClientConfigKeys.BlockWrite.ReplaceDatanodeOnFailure.BEST_EFFORT_DEFAULT);
-
- return new ReplaceDatanodeOnFailure(policy, bestEffort);
- }
-
- private static Policy getPolicy(final Configuration conf) {
- final boolean enabled = conf.getBoolean(
- HdfsClientConfigKeys.BlockWrite.ReplaceDatanodeOnFailure.ENABLE_KEY,
- HdfsClientConfigKeys.BlockWrite.ReplaceDatanodeOnFailure.ENABLE_DEFAULT);
- if (!enabled) {
- return Policy.DISABLE;
- }
-
- final String policy = conf.get(
- HdfsClientConfigKeys.BlockWrite.ReplaceDatanodeOnFailure.POLICY_KEY,
- HdfsClientConfigKeys.BlockWrite.ReplaceDatanodeOnFailure.POLICY_DEFAULT);
- for(int i = 1; i < Policy.values().length; i++) {
- final Policy p = Policy.values()[i];
- if (p.name().equalsIgnoreCase(policy)) {
- return p;
- }
- }
- throw new HadoopIllegalArgumentException("Illegal configuration value for "
- + HdfsClientConfigKeys.BlockWrite.ReplaceDatanodeOnFailure.POLICY_KEY
- + ": " + policy);
- }
-
- /** Write the setting to configuration. */
- public static void write(final Policy policy,
- final boolean bestEffort, final Configuration conf) {
- conf.setBoolean(
- HdfsClientConfigKeys.BlockWrite.ReplaceDatanodeOnFailure.ENABLE_KEY,
- policy != Policy.DISABLE);
- conf.set(
- HdfsClientConfigKeys.BlockWrite.ReplaceDatanodeOnFailure.POLICY_KEY,
- policy.name());
- conf.setBoolean(
- HdfsClientConfigKeys.BlockWrite.ReplaceDatanodeOnFailure.BEST_EFFORT_KEY,
- bestEffort);
- }
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf37d3d8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java
index be1a9ef..ff2d762 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/balancer/Dispatcher.java
@@ -47,6 +47,7 @@ import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.hdfs.DFSUtil;
+import org.apache.hadoop.hdfs.DFSUtilClient;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
@@ -877,7 +878,7 @@ public class Dispatcher {
this.saslClient = new SaslDataTransferClient(conf,
DataTransferSaslUtil.getSaslPropertiesResolver(conf),
TrustedChannelResolver.getInstance(conf), nnc.fallbackToSimpleAuth);
- this.ioFileBufferSize = DFSUtil.getIoFileBufferSize(conf);
+ this.ioFileBufferSize = DFSUtilClient.getIoFileBufferSize(conf);
}
public DistributedFileSystem getDistributedFileSystem() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf37d3d8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java
index 957b2c7..eec2b2d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockReceiver.java
@@ -38,7 +38,7 @@ import org.apache.commons.logging.Log;
import org.apache.hadoop.fs.ChecksumException;
import org.apache.hadoop.fs.FSOutputSummer;
import org.apache.hadoop.fs.StorageType;
-import org.apache.hadoop.hdfs.DFSUtil;
+import org.apache.hadoop.hdfs.DFSUtilClient;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.datatransfer.BlockConstructionStage;
@@ -248,7 +248,7 @@ class BlockReceiver implements Closeable {
out.getClass());
}
this.checksumOut = new DataOutputStream(new BufferedOutputStream(
- streams.getChecksumOut(), DFSUtil.getSmallBufferSize(
+ streams.getChecksumOut(), DFSUtilClient.getSmallBufferSize(
datanode.getConf())));
// write data chunk header if creating a new replica
if (isCreate) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf37d3d8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java
index fb8b132..d7e62bb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockSender.java
@@ -34,7 +34,7 @@ import java.util.Arrays;
import org.apache.commons.logging.Log;
import org.apache.hadoop.fs.ChecksumException;
-import org.apache.hadoop.hdfs.DFSUtil;
+import org.apache.hadoop.hdfs.DFSUtilClient;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.datatransfer.PacketHeader;
@@ -111,7 +111,7 @@ class BlockSender implements java.io.Closeable {
private static final int IO_FILE_BUFFER_SIZE;
static {
HdfsConfiguration conf = new HdfsConfiguration();
- IO_FILE_BUFFER_SIZE = DFSUtil.getIoFileBufferSize(conf);
+ IO_FILE_BUFFER_SIZE = DFSUtilClient.getIoFileBufferSize(conf);
}
private static final int TRANSFERTO_BUFFER_SIZE = Math.max(
IO_FILE_BUFFER_SIZE, MIN_BUFFER_WITH_TRANSFERTO);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf37d3d8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
index 7c935d3..1bb4485 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataNode.java
@@ -108,6 +108,7 @@ import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSUtil;
+import org.apache.hadoop.hdfs.DFSUtilClient;
import org.apache.hadoop.hdfs.HDFSPolicyProvider;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.client.BlockReportOptions;
@@ -2156,7 +2157,7 @@ public class DataNode extends ReconfigurableBase
unbufIn = saslStreams.in;
out = new DataOutputStream(new BufferedOutputStream(unbufOut,
- DFSUtil.getSmallBufferSize(conf)));
+ DFSUtilClient.getSmallBufferSize(conf)));
in = new DataInputStream(unbufIn);
blockSender = new BlockSender(b, 0, b.getNumBytes(),
false, false, true, DataNode.this, null, cachingStrategy);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf37d3d8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java
index 8c4e38a..665432e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/DataXceiver.java
@@ -49,7 +49,7 @@ import java.util.Arrays;
import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.logging.Log;
import org.apache.hadoop.fs.StorageType;
-import org.apache.hadoop.hdfs.DFSUtil;
+import org.apache.hadoop.hdfs.DFSUtilClient;
import org.apache.hadoop.hdfs.ExtendedBlockId;
import org.apache.hadoop.hdfs.net.Peer;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
@@ -134,8 +134,8 @@ class DataXceiver extends Receiver implements Runnable {
this.datanode = datanode;
this.dataXceiverServer = dataXceiverServer;
this.connectToDnViaHostname = datanode.getDnConf().connectToDnViaHostname;
- this.ioFileBufferSize = DFSUtil.getIoFileBufferSize(datanode.getConf());
- this.smallBufferSize = DFSUtil.getSmallBufferSize(datanode.getConf());
+ this.ioFileBufferSize = DFSUtilClient.getIoFileBufferSize(datanode.getConf());
+ this.smallBufferSize = DFSUtilClient.getSmallBufferSize(datanode.getConf());
remoteAddress = peer.getRemoteAddressString();
final int colonIdx = remoteAddress.indexOf(':');
remoteAddressWithoutPort =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf37d3d8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaNotFoundException.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaNotFoundException.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaNotFoundException.java
deleted file mode 100644
index b159d3a..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/ReplicaNotFoundException.java
+++ /dev/null
@@ -1,53 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-package org.apache.hadoop.hdfs.server.datanode;
-
-import java.io.IOException;
-
-import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
-
-/**
- * Exception indicating that DataNode does not have a replica
- * that matches the target block.
- */
-public class ReplicaNotFoundException extends IOException {
- private static final long serialVersionUID = 1L;
- public final static String NON_RBW_REPLICA = "Cannot recover a non-RBW replica ";
- public final static String UNFINALIZED_REPLICA =
- "Cannot append to an unfinalized replica ";
- public final static String UNFINALIZED_AND_NONRBW_REPLICA =
- "Cannot recover append/close to a replica that's not FINALIZED and not RBW ";
- public final static String NON_EXISTENT_REPLICA =
- "Cannot append to a non-existent replica ";
- public final static String UNEXPECTED_GS_REPLICA =
- "Cannot append to a replica with unexpected generation stamp ";
-
- public ReplicaNotFoundException() {
- super();
- }
-
- public ReplicaNotFoundException(ExtendedBlock b) {
- super("Replica not found for " + b);
- }
-
- public ReplicaNotFoundException(String msg) {
- super(msg);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf37d3d8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java
index a586a7c..68c951a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/BlockPoolSlice.java
@@ -38,7 +38,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.DU;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.hdfs.DFSConfigKeys;
-import org.apache.hadoop.hdfs.DFSUtil;
+import org.apache.hadoop.hdfs.DFSUtilClient;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.BlockListAsLongs;
import org.apache.hadoop.hdfs.protocol.BlockListAsLongs.BlockReportReplica;
@@ -111,7 +111,7 @@ class BlockPoolSlice {
}
}
- this.ioFileBufferSize = DFSUtil.getIoFileBufferSize(conf);
+ this.ioFileBufferSize = DFSUtilClient.getIoFileBufferSize(conf);
this.deleteDuplicateReplicas = conf.getBoolean(
DFSConfigKeys.DFS_DATANODE_DUPLICATE_REPLICA_DELETION,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf37d3d8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
index 32eb724..466c7e9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/FsDatasetImpl.java
@@ -57,7 +57,7 @@ import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.hdfs.DFSConfigKeys;
-import org.apache.hadoop.hdfs.DFSUtil;
+import org.apache.hadoop.hdfs.DFSUtilClient;
import org.apache.hadoop.hdfs.ExtendedBlockId;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.BlockListAsLongs;
@@ -263,8 +263,8 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
this.datanode = datanode;
this.dataStorage = storage;
this.conf = conf;
- this.smallBufferSize = DFSUtil.getSmallBufferSize(conf);
- // The number of volumes required for operation is the total number
+ this.smallBufferSize = DFSUtilClient.getSmallBufferSize(conf);
+ // The number of volumes required for operation is the total number
// of volumes minus the number of failed volumes we can tolerate.
volFailuresTolerated =
conf.getInt(DFSConfigKeys.DFS_DATANODE_FAILED_VOLUMES_TOLERATED_KEY,
@@ -956,7 +956,7 @@ class FsDatasetImpl implements FsDatasetSpi<FsVolumeImpl> {
File blockFile, int smallBufferSize, final Configuration conf)
throws IOException {
final DataChecksum checksum = BlockMetadataHeader.readDataChecksum(srcMeta,
- DFSUtil.getIoFileBufferSize(conf));
+ DFSUtilClient.getIoFileBufferSize(conf));
final byte[] data = new byte[1 << 16];
final byte[] crcs = new byte[checksum.getChecksumSize(data.length)];
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf37d3d8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/RamDiskAsyncLazyPersistService.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/RamDiskAsyncLazyPersistService.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/RamDiskAsyncLazyPersistService.java
index 2a4c191..9b467ea 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/RamDiskAsyncLazyPersistService.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/RamDiskAsyncLazyPersistService.java
@@ -21,7 +21,7 @@ package org.apache.hadoop.hdfs.server.datanode.fsdataset.impl;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.DFSUtil;
+import org.apache.hadoop.hdfs.DFSUtilClient;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsVolumeReference;
@@ -240,7 +240,7 @@ class RamDiskAsyncLazyPersistService {
boolean succeeded = false;
final FsDatasetImpl dataset = (FsDatasetImpl)datanode.getFSDataset();
try (FsVolumeReference ref = this.targetVolume) {
- int smallBufferSize = DFSUtil.getSmallBufferSize(EMPTY_HDFS_CONF);
+ int smallBufferSize = DFSUtilClient.getSmallBufferSize(EMPTY_HDFS_CONF);
// No FsDatasetImpl lock for the file copy
File targetFiles[] = FsDatasetImpl.copyBlockFiles(
blockId, genStamp, metaFile, blockFile, lazyPersistDir, true,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf37d3d8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/RetryStartFileException.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/RetryStartFileException.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/RetryStartFileException.java
deleted file mode 100644
index 0bdd2a5..0000000
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/RetryStartFileException.java
+++ /dev/null
@@ -1,36 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hdfs.server.namenode;
-
-import java.io.IOException;
-
-import org.apache.hadoop.classification.InterfaceAudience;
-
-@InterfaceAudience.Private
-public class RetryStartFileException extends IOException {
- private static final long serialVersionUID = 1L;
-
- public RetryStartFileException() {
- super("Preconditions for creating a file failed because of a " +
- "transient error, retry create later.");
- }
-
- public RetryStartFileException(String s) {
- super(s);
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf37d3d8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java
index afecf99..4af2021 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/TransferFsImage.java
@@ -43,7 +43,7 @@ import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.hdfs.DFSConfigKeys;
-import org.apache.hadoop.hdfs.DFSUtil;
+import org.apache.hadoop.hdfs.DFSUtilClient;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.server.common.Storage;
import org.apache.hadoop.hdfs.server.common.Storage.StorageDirectory;
@@ -115,7 +115,7 @@ public class TransferFsImage {
connectionFactory = URLConnectionFactory
.newDefaultURLConnectionFactory(conf);
isSpnegoEnabled = UserGroupInformation.isSecurityEnabled();
- IO_FILE_BUFFER_SIZE = DFSUtil.getIoFileBufferSize(conf);
+ IO_FILE_BUFFER_SIZE = DFSUtilClient.getIoFileBufferSize(conf);
}
private static final Log LOG = LogFactory.getLog(TransferFsImage.class);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf37d3d8/hadoop-hdfs-project/hadoop-hdfs/src/test/aop/org/apache/hadoop/hdfs/server/datanode/TestFiDataTransferProtocol2.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/aop/org/apache/hadoop/hdfs/server/datanode/TestFiDataTransferProtocol2.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/aop/org/apache/hadoop/hdfs/server/datanode/TestFiDataTransferProtocol2.java
index 0b0da21..52cd9c1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/aop/org/apache/hadoop/hdfs/server/datanode/TestFiDataTransferProtocol2.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/aop/org/apache/hadoop/hdfs/server/datanode/TestFiDataTransferProtocol2.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.hdfs.server.datanode;
import java.io.IOException;
import java.util.Random;
-import org.apache.commons.logging.impl.Log4JLogger;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fi.DataTransferTestUtil;
import org.apache.hadoop.fi.DataTransferTestUtil.CountdownDoosAction;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf37d3d8/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
index a7e80ca..bce8b64 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
@@ -982,7 +982,7 @@ public class DFSTestUtil {
final long writeTimeout = dfsClient.getDatanodeWriteTimeout(datanodes.length);
final DataOutputStream out = new DataOutputStream(new BufferedOutputStream(
NetUtils.getOutputStream(s, writeTimeout),
- DFSUtil.getSmallBufferSize(dfsClient.getConfiguration())));
+ DFSUtilClient.getSmallBufferSize(dfsClient.getConfiguration())));
final DataInputStream in = new DataInputStream(NetUtils.getInputStream(s));
// send the request
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf37d3d8/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSClusterWithNodeGroup.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSClusterWithNodeGroup.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSClusterWithNodeGroup.java
index b9682a1..5c011e3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSClusterWithNodeGroup.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/MiniDFSClusterWithNodeGroup.java
@@ -28,8 +28,8 @@ import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.hdfs.server.datanode.SecureDataNodeStarter;
-import org.apache.hadoop.hdfs.server.datanode.SimulatedFSDataset;
import org.apache.hadoop.hdfs.server.datanode.SecureDataNodeStarter.SecureResources;
+import org.apache.hadoop.hdfs.server.datanode.SimulatedFSDataset;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.FsDatasetSpi;
import org.apache.hadoop.hdfs.server.datanode.fsdataset.impl.FsVolumeImpl;
import org.apache.hadoop.net.NetUtils;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf37d3d8/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
index cfc317f..62d1a3e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestBlockStoragePolicy.java
@@ -36,7 +36,6 @@ import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.hdfs.protocol.*;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
import org.apache.hadoop.hdfs.server.blockmanagement.*;
-import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf37d3d8/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestClientProtocolForPipelineRecovery.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestClientProtocolForPipelineRecovery.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestClientProtocolForPipelineRecovery.java
index ca98441..b83157d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestClientProtocolForPipelineRecovery.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestClientProtocolForPipelineRecovery.java
@@ -124,8 +124,8 @@ public class TestClientProtocolForPipelineRecovery {
public void testPipelineRecoveryForLastBlock() throws IOException {
DFSClientFaultInjector faultInjector
= Mockito.mock(DFSClientFaultInjector.class);
- DFSClientFaultInjector oldInjector = DFSClientFaultInjector.instance;
- DFSClientFaultInjector.instance = faultInjector;
+ DFSClientFaultInjector oldInjector = DFSClientFaultInjector.get();
+ DFSClientFaultInjector.set(faultInjector);
Configuration conf = new HdfsConfiguration();
conf.setInt(HdfsClientConfigKeys.BlockWrite.LOCATEFOLLOWINGBLOCK_RETRIES_KEY, 3);
@@ -153,7 +153,7 @@ public class TestClientProtocolForPipelineRecovery {
+ " corrupt replicas.");
}
} finally {
- DFSClientFaultInjector.instance = oldInjector;
+ DFSClientFaultInjector.set(oldInjector);
if (cluster != null) {
cluster.shutdown();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf37d3d8/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestCrcCorruption.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestCrcCorruption.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestCrcCorruption.java
index 2b11fbd..3850ff2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestCrcCorruption.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestCrcCorruption.java
@@ -76,7 +76,7 @@ public class TestCrcCorruption {
@Before
public void setUp() throws IOException {
faultInjector = Mockito.mock(DFSClientFaultInjector.class);
- DFSClientFaultInjector.instance = faultInjector;
+ DFSClientFaultInjector.set(faultInjector);
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf37d3d8/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java
index a821c30..3435b7f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSUtil.java
@@ -903,16 +903,16 @@ public class TestDFSUtil {
Configuration conf = new Configuration(false);
conf.unset(DFSConfigKeys.DFS_ENCRYPTION_KEY_PROVIDER_URI);
assertFalse("encryption enabled on no provider key",
- DFSUtil.isHDFSEncryptionEnabled(conf));
+ DFSUtilClient.isHDFSEncryptionEnabled(conf));
conf.set(DFSConfigKeys.DFS_ENCRYPTION_KEY_PROVIDER_URI, "");
assertFalse("encryption enabled on empty provider key",
- DFSUtil.isHDFSEncryptionEnabled(conf));
+ DFSUtilClient.isHDFSEncryptionEnabled(conf));
conf.set(DFSConfigKeys.DFS_ENCRYPTION_KEY_PROVIDER_URI, "\n\t\n");
assertFalse("encryption enabled on whitespace provider key",
- DFSUtil.isHDFSEncryptionEnabled(conf));
+ DFSUtilClient.isHDFSEncryptionEnabled(conf));
conf.set(DFSConfigKeys.DFS_ENCRYPTION_KEY_PROVIDER_URI, "http://hadoop.apache.org");
assertTrue("encryption disabled on valid provider key",
- DFSUtil.isHDFSEncryptionEnabled(conf));
+ DFSUtilClient.isHDFSEncryptionEnabled(conf));
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/bf37d3d8/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestPread.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestPread.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestPread.java
index 43650a8..2789e33 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestPread.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestPread.java
@@ -292,9 +292,8 @@ public class TestPread {
hedgedReadTimeoutMillis);
conf.setInt(HdfsClientConfigKeys.Retry.WINDOW_BASE_KEY, 0);
// Set up the InjectionHandler
- DFSClientFaultInjector.instance = Mockito
- .mock(DFSClientFaultInjector.class);
- DFSClientFaultInjector injector = DFSClientFaultInjector.instance;
+ DFSClientFaultInjector.set(Mockito.mock(DFSClientFaultInjector.class));
+ DFSClientFaultInjector injector = DFSClientFaultInjector.get();
final int sleepMs = 100;
Mockito.doAnswer(new Answer<Void>() {
@Override
@@ -368,9 +367,8 @@ public class TestPread {
initialHedgedReadTimeoutMillis);
// Set up the InjectionHandler
- DFSClientFaultInjector.instance = Mockito
- .mock(DFSClientFaultInjector.class);
- DFSClientFaultInjector injector = DFSClientFaultInjector.instance;
+ DFSClientFaultInjector.set(Mockito.mock(DFSClientFaultInjector.class));
+ DFSClientFaultInjector injector = DFSClientFaultInjector.get();
// make preads sleep for 50ms
Mockito.doAnswer(new Answer<Void>() {
@Override