You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ar...@apache.org on 2015/07/01 23:09:47 UTC
[01/46] hadoop git commit: YARN-2871.
TestRMRestart#testRMRestartGetApplicationList sometime fails in trunk.
Contributed by zhihai xu
Repository: hadoop
Updated Branches:
refs/heads/HDFS-7240 71dfa130c -> 6551e11c0
YARN-2871. TestRMRestart#testRMRestartGetApplicationList sometime fails
in trunk. Contributed by zhihai xu
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fe6c1bd7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fe6c1bd7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fe6c1bd7
Branch: refs/heads/HDFS-7240
Commit: fe6c1bd73aee188ed58df4d33bbc2d2fe0779a97
Parents: aa07dea
Author: Xuan <xg...@apache.org>
Authored: Fri Jun 26 19:43:59 2015 -0700
Committer: Xuan <xg...@apache.org>
Committed: Fri Jun 26 19:43:59 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +++
.../hadoop/yarn/server/resourcemanager/TestRMRestart.java | 8 +++++---
2 files changed, 8 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fe6c1bd7/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index d74e8a9..e2bf706 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -559,6 +559,9 @@ Release 2.8.0 - UNRELEASED
YARN-3745. SerializedException should also try to instantiate internal
exception with the default constructor. (Lavkesh Lahngir via devaraj)
+ YARN-2871. TestRMRestart#testRMRestartGetApplicationList sometime fails in trunk.
+ (zhihai xu via xgong)
+
Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fe6c1bd7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
index 628e9be..de17acd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.yarn.server.resourcemanager;
import static org.mockito.Matchers.isA;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.timeout;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@@ -976,9 +977,10 @@ public class TestRMRestart extends ParameterizedSchedulerTestBase {
List<ApplicationReport> appList2 = response2.getApplicationList();
Assert.assertTrue(3 == appList2.size());
- // check application summary is logged for the completed apps after RM restart.
- verify(rm2.getRMAppManager(), times(3)).logApplicationSummary(
- isA(ApplicationId.class));
+ // check application summary is logged for the completed apps with timeout
+ // to make sure APP_COMPLETED events are processed, after RM restart.
+ verify(rm2.getRMAppManager(), timeout(1000).times(3)).
+ logApplicationSummary(isA(ApplicationId.class));
}
private MockAM launchAM(RMApp app, MockRM rm, MockNM nm)
[10/46] hadoop git commit: HDFS-8628. Update missing command option
for fetchdt (Contributed by J.Andreina)
Posted by ar...@apache.org.
HDFS-8628. Update missing command option for fetchdt (Contributed by J.Andreina)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fde20ffc
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fde20ffc
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fde20ffc
Branch: refs/heads/HDFS-7240
Commit: fde20ffcef6dcd477f501b5f80c86665296711c5
Parents: c815344
Author: Vinayakumar B <vi...@apache.org>
Authored: Mon Jun 29 16:50:48 2015 +0530
Committer: Vinayakumar B <vi...@apache.org>
Committed: Mon Jun 29 16:50:48 2015 +0530
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../hadoop-hdfs/src/site/markdown/HDFSCommands.md | 10 +++++++---
2 files changed, 10 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fde20ffc/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 3800184..e55f340 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -971,6 +971,9 @@ Release 2.8.0 - UNRELEASED
HDFS-8586. Dead Datanode is allocated for write when client is from deadnode
(Brahma Reddy Battula via vinayakumarb)
+ HDFS-8628. Update missing command option for fetchdt
+ (J.Andreina via vinayakumarb)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fde20ffc/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
index 6841201..094dda1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
@@ -89,12 +89,16 @@ Run a filesystem command on the file system supported in Hadoop. The various COM
### `fetchdt`
-Usage: `hdfs fetchdt [--webservice <namenode_http_addr>] <path> `
+Usage: `hdfs fetchdt <opts> <token_file_path> `
| COMMAND\_OPTION | Description |
|:---- |:---- |
-| `--webservice` *https\_address* | use http protocol instead of RPC |
-| *fileName* | File name to store the token into. |
+| `--webservice` *NN_Url* | Url to contact NN on (starts with http or https)|
+| `--renewer` *name* | Name of the delegation token renewer |
+| `--cancel` | Cancel the delegation token |
+| `--renew` | Renew the delegation token. Delegation token must have been fetched using the --renewer *name* option.|
+| `--print` | Print the delegation token |
+| *token_file_path* | File path to store the token into. |
Gets Delegation Token from a NameNode. See [fetchdt](./HdfsUserGuide.html#fetchdt) for more info.
[28/46] hadoop git commit: MAPREDUCE-6384. Add the last reporting
reducer info for too many fetch failure diagnostics. Contributed by Chang Li
Posted by ar...@apache.org.
MAPREDUCE-6384. Add the last reporting reducer info for too many fetch failure diagnostics. Contributed by Chang Li
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b6ba5645
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b6ba5645
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b6ba5645
Branch: refs/heads/HDFS-7240
Commit: b6ba56457c6b01dae795c11d587c3fe3855ee707
Parents: 147e020
Author: Jason Lowe <jl...@apache.org>
Authored: Tue Jun 30 21:22:30 2015 +0000
Committer: Jason Lowe <jl...@apache.org>
Committed: Tue Jun 30 21:22:30 2015 +0000
----------------------------------------------------------------------
hadoop-mapreduce-project/CHANGES.txt | 3 +
.../event/JobTaskAttemptFetchFailureEvent.java | 9 +-
.../TaskAttemptTooManyFetchFailureEvent.java | 50 +++++++
.../mapreduce/v2/app/job/impl/JobImpl.java | 7 +-
.../v2/app/job/impl/TaskAttemptImpl.java | 15 +-
.../mapreduce/v2/app/TestFetchFailure.java | 31 ++--
.../v2/app/job/impl/TestTaskAttempt.java | 143 ++++++++++---------
7 files changed, 168 insertions(+), 90 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6ba5645/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index 785fce8..5a4d826 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -364,6 +364,9 @@ Release 2.8.0 - UNRELEASED
MAPREDUCE-6408. Queue name and user name should be printed on the job page.
(Siqi Li via gera)
+ MAPREDUCE-6384. Add the last reporting reducer info for too many fetch
+ failure diagnostics (Chang Li via jlowe)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6ba5645/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/JobTaskAttemptFetchFailureEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/JobTaskAttemptFetchFailureEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/JobTaskAttemptFetchFailureEvent.java
index 37e2034..787711c 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/JobTaskAttemptFetchFailureEvent.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/JobTaskAttemptFetchFailureEvent.java
@@ -28,13 +28,15 @@ public class JobTaskAttemptFetchFailureEvent extends JobEvent {
private final TaskAttemptId reduce;
private final List<TaskAttemptId> maps;
+ private final String hostname;
public JobTaskAttemptFetchFailureEvent(TaskAttemptId reduce,
- List<TaskAttemptId> maps) {
- super(reduce.getTaskId().getJobId(),
+ List<TaskAttemptId> maps, String host) {
+ super(reduce.getTaskId().getJobId(),
JobEventType.JOB_TASK_ATTEMPT_FETCH_FAILURE);
this.reduce = reduce;
this.maps = maps;
+ this.hostname = host;
}
public List<TaskAttemptId> getMaps() {
@@ -45,4 +47,7 @@ public class JobTaskAttemptFetchFailureEvent extends JobEvent {
return reduce;
}
+ public String getHost() {
+ return hostname;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6ba5645/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/TaskAttemptTooManyFetchFailureEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/TaskAttemptTooManyFetchFailureEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/TaskAttemptTooManyFetchFailureEvent.java
new file mode 100644
index 0000000..662e712
--- /dev/null
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/event/TaskAttemptTooManyFetchFailureEvent.java
@@ -0,0 +1,50 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.mapreduce.v2.app.job.event;
+
+import org.apache.hadoop.mapreduce.v2.api.records.TaskAttemptId;
+
+/**
+ * TaskAttemptTooManyFetchFailureEvent is used for TA_TOO_MANY_FETCH_FAILURE.
+ */
+public class TaskAttemptTooManyFetchFailureEvent extends TaskAttemptEvent {
+ private TaskAttemptId reduceID;
+ private String reduceHostname;
+
+ /**
+ * Create a new TaskAttemptTooManyFetchFailureEvent.
+ * @param attemptId the id of the mapper task attempt
+ * @param reduceId the id of the reporting reduce task attempt.
+ * @param reduceHost the hostname of the reporting reduce task attempt.
+ */
+ public TaskAttemptTooManyFetchFailureEvent(TaskAttemptId attemptId,
+ TaskAttemptId reduceId, String reduceHost) {
+ super(attemptId, TaskAttemptEventType.TA_TOO_MANY_FETCH_FAILURE);
+ this.reduceID = reduceId;
+ this.reduceHostname = reduceHost;
+ }
+
+ public TaskAttemptId getReduceId() {
+ return reduceID;
+ }
+
+ public String getReduceHost() {
+ return reduceHostname;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6ba5645/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
index 6e9f13c..2c48019 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/JobImpl.java
@@ -103,9 +103,8 @@ import org.apache.hadoop.mapreduce.v2.app.job.event.JobTaskAttemptCompletedEvent
import org.apache.hadoop.mapreduce.v2.app.job.event.JobTaskAttemptFetchFailureEvent;
import org.apache.hadoop.mapreduce.v2.app.job.event.JobTaskEvent;
import org.apache.hadoop.mapreduce.v2.app.job.event.JobUpdatedNodesEvent;
-import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEvent;
-import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEventType;
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptKillEvent;
+import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptTooManyFetchFailureEvent;
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskEvent;
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskEventType;
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskRecoverEvent;
@@ -1914,8 +1913,8 @@ public class JobImpl implements org.apache.hadoop.mapreduce.v2.app.job.Job,
&& failureRate >= job.getMaxAllowedFetchFailuresFraction()) {
LOG.info("Too many fetch-failures for output of task attempt: " +
mapId + " ... raising fetch failure to map");
- job.eventHandler.handle(new TaskAttemptEvent(mapId,
- TaskAttemptEventType.TA_TOO_MANY_FETCH_FAILURE));
+ job.eventHandler.handle(new TaskAttemptTooManyFetchFailureEvent(mapId,
+ fetchfailureEvent.getReduce(), fetchfailureEvent.getHost()));
job.fetchFailuresMapping.remove(mapId);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6ba5645/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
index 3055a25..3fa42fe 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TaskAttemptImpl.java
@@ -95,6 +95,7 @@ import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptKillEvent;
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptRecoverEvent;
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptStatusUpdateEvent;
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptStatusUpdateEvent.TaskAttemptStatus;
+import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptTooManyFetchFailureEvent;
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskEventType;
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskTAttemptEvent;
import org.apache.hadoop.mapreduce.v2.app.launcher.ContainerLauncher;
@@ -1916,12 +1917,17 @@ public abstract class TaskAttemptImpl implements
@SuppressWarnings("unchecked")
@Override
public void transition(TaskAttemptImpl taskAttempt, TaskAttemptEvent event) {
+ TaskAttemptTooManyFetchFailureEvent fetchFailureEvent =
+ (TaskAttemptTooManyFetchFailureEvent) event;
// too many fetch failure can only happen for map tasks
Preconditions
.checkArgument(taskAttempt.getID().getTaskId().getTaskType() == TaskType.MAP);
//add to diagnostic
- taskAttempt.addDiagnosticInfo("Too Many fetch failures.Failing the attempt");
-
+ taskAttempt.addDiagnosticInfo("Too many fetch failures."
+ + " Failing the attempt. Last failure reported by " +
+ fetchFailureEvent.getReduceId() +
+ " from host " + fetchFailureEvent.getReduceHost());
+
if (taskAttempt.getLaunchTime() != 0) {
taskAttempt.eventHandler
.handle(createJobCounterUpdateEventTAFailed(taskAttempt, true));
@@ -2225,8 +2231,11 @@ public abstract class TaskAttemptImpl implements
//this only will happen in reduce attempt type
if (taskAttempt.reportedStatus.fetchFailedMaps != null &&
taskAttempt.reportedStatus.fetchFailedMaps.size() > 0) {
+ String hostname = taskAttempt.container == null ? "UNKNOWN"
+ : taskAttempt.container.getNodeId().getHost();
taskAttempt.eventHandler.handle(new JobTaskAttemptFetchFailureEvent(
- taskAttempt.attemptId, taskAttempt.reportedStatus.fetchFailedMaps));
+ taskAttempt.attemptId, taskAttempt.reportedStatus.fetchFailedMaps,
+ hostname));
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6ba5645/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestFetchFailure.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestFetchFailure.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestFetchFailure.java
index 4e4e2e7..8d25079 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestFetchFailure.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/TestFetchFailure.java
@@ -94,10 +94,10 @@ public class TestFetchFailure {
app.waitForState(reduceAttempt, TaskAttemptState.RUNNING);
//send 3 fetch failures from reduce to trigger map re execution
- sendFetchFailure(app, reduceAttempt, mapAttempt1);
- sendFetchFailure(app, reduceAttempt, mapAttempt1);
- sendFetchFailure(app, reduceAttempt, mapAttempt1);
-
+ sendFetchFailure(app, reduceAttempt, mapAttempt1, "host");
+ sendFetchFailure(app, reduceAttempt, mapAttempt1, "host");
+ sendFetchFailure(app, reduceAttempt, mapAttempt1, "host");
+
//wait for map Task state move back to RUNNING
app.waitForState(mapTask, TaskState.RUNNING);
@@ -215,9 +215,9 @@ public class TestFetchFailure {
app.waitForState(reduceAttempt, TaskAttemptState.RUNNING);
//send 3 fetch failures from reduce to trigger map re execution
- sendFetchFailure(app, reduceAttempt, mapAttempt1);
- sendFetchFailure(app, reduceAttempt, mapAttempt1);
- sendFetchFailure(app, reduceAttempt, mapAttempt1);
+ sendFetchFailure(app, reduceAttempt, mapAttempt1, "host");
+ sendFetchFailure(app, reduceAttempt, mapAttempt1, "host");
+ sendFetchFailure(app, reduceAttempt, mapAttempt1, "host");
//wait for map Task state move back to RUNNING
app.waitForState(mapTask, TaskState.RUNNING);
@@ -324,8 +324,8 @@ public class TestFetchFailure {
updateStatus(app, reduceAttempt3, Phase.SHUFFLE);
//send 2 fetch failures from reduce to prepare for map re execution
- sendFetchFailure(app, reduceAttempt, mapAttempt1);
- sendFetchFailure(app, reduceAttempt, mapAttempt1);
+ sendFetchFailure(app, reduceAttempt, mapAttempt1, "host1");
+ sendFetchFailure(app, reduceAttempt2, mapAttempt1, "host2");
//We should not re-launch the map task yet
assertEquals(TaskState.SUCCEEDED, mapTask.getState());
@@ -333,7 +333,7 @@ public class TestFetchFailure {
updateStatus(app, reduceAttempt3, Phase.REDUCE);
//send 3rd fetch failures from reduce to trigger map re execution
- sendFetchFailure(app, reduceAttempt, mapAttempt1);
+ sendFetchFailure(app, reduceAttempt3, mapAttempt1, "host3");
//wait for map Task state move back to RUNNING
app.waitForState(mapTask, TaskState.RUNNING);
@@ -342,6 +342,11 @@ public class TestFetchFailure {
Assert.assertEquals("Map TaskAttempt state not correct",
TaskAttemptState.FAILED, mapAttempt1.getState());
+ Assert.assertEquals(mapAttempt1.getDiagnostics().get(0),
+ "Too many fetch failures. Failing the attempt. "
+ + "Last failure reported by "
+ + reduceAttempt3.getID().toString() + " from host host3");
+
Assert.assertEquals("Num attempts in Map Task not correct",
2, mapTask.getAttempts().size());
@@ -410,7 +415,6 @@ public class TestFetchFailure {
Assert.assertEquals("Unexpected map event", convertedEvents[2],
mapEvents[0]);
}
-
private void updateStatus(MRApp app, TaskAttempt attempt, Phase phase) {
TaskAttemptStatusUpdateEvent.TaskAttemptStatus status = new TaskAttemptStatusUpdateEvent.TaskAttemptStatus();
@@ -430,11 +434,12 @@ public class TestFetchFailure {
}
private void sendFetchFailure(MRApp app, TaskAttempt reduceAttempt,
- TaskAttempt mapAttempt) {
+ TaskAttempt mapAttempt, String hostname) {
app.getContext().getEventHandler().handle(
new JobTaskAttemptFetchFailureEvent(
reduceAttempt.getID(),
- Arrays.asList(new TaskAttemptId[] {mapAttempt.getID()})));
+ Arrays.asList(new TaskAttemptId[] {mapAttempt.getID()}),
+ hostname));
}
static class MRAppWithHistory extends MRApp {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6ba5645/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java
index 79b88d8..a88a935 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/job/impl/TestTaskAttempt.java
@@ -70,6 +70,7 @@ import org.apache.hadoop.mapreduce.v2.app.job.event.JobEventType;
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptContainerAssignedEvent;
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptContainerLaunchedEvent;
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptDiagnosticsUpdateEvent;
+import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptTooManyFetchFailureEvent;
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEvent;
import org.apache.hadoop.mapreduce.v2.app.job.event.TaskAttemptEventType;
import org.apache.hadoop.mapreduce.v2.app.rm.ContainerRequestEvent;
@@ -507,6 +508,9 @@ public class TestTaskAttempt{
JobId jobId = MRBuilderUtils.newJobId(appId, 1);
TaskId taskId = MRBuilderUtils.newTaskId(jobId, 1, TaskType.MAP);
TaskAttemptId attemptId = MRBuilderUtils.newTaskAttemptId(taskId, 0);
+ TaskId reduceTaskId = MRBuilderUtils.newTaskId(jobId, 1, TaskType.REDUCE);
+ TaskAttemptId reduceTAId =
+ MRBuilderUtils.newTaskAttemptId(reduceTaskId, 0);
Path jobFile = mock(Path.class);
MockEventHandler eventHandler = new MockEventHandler();
@@ -554,8 +558,8 @@ public class TestTaskAttempt{
assertEquals("Task attempt is not in succeeded state", taImpl.getState(),
TaskAttemptState.SUCCEEDED);
- taImpl.handle(new TaskAttemptEvent(attemptId,
- TaskAttemptEventType.TA_TOO_MANY_FETCH_FAILURE));
+ taImpl.handle(new TaskAttemptTooManyFetchFailureEvent(attemptId,
+ reduceTAId, "Host"));
assertEquals("Task attempt is not in FAILED state", taImpl.getState(),
TaskAttemptState.FAILED);
taImpl.handle(new TaskAttemptEvent(attemptId,
@@ -735,72 +739,75 @@ public class TestTaskAttempt{
@Test
public void testFetchFailureAttemptFinishTime() throws Exception{
- ApplicationId appId = ApplicationId.newInstance(1, 2);
- ApplicationAttemptId appAttemptId =
- ApplicationAttemptId.newInstance(appId, 0);
- JobId jobId = MRBuilderUtils.newJobId(appId, 1);
- TaskId taskId = MRBuilderUtils.newTaskId(jobId, 1, TaskType.MAP);
- TaskAttemptId attemptId = MRBuilderUtils.newTaskAttemptId(taskId, 0);
- Path jobFile = mock(Path.class);
-
- MockEventHandler eventHandler = new MockEventHandler();
- TaskAttemptListener taListener = mock(TaskAttemptListener.class);
- when(taListener.getAddress()).thenReturn(
- new InetSocketAddress("localhost", 0));
-
- JobConf jobConf = new JobConf();
- jobConf.setClass("fs.file.impl", StubbedFS.class, FileSystem.class);
- jobConf.setBoolean("fs.file.impl.disable.cache", true);
- jobConf.set(JobConf.MAPRED_MAP_TASK_ENV, "");
- jobConf.set(MRJobConfig.APPLICATION_ATTEMPT_ID, "10");
-
- TaskSplitMetaInfo splits = mock(TaskSplitMetaInfo.class);
- when(splits.getLocations()).thenReturn(new String[] {"127.0.0.1"});
-
- AppContext appCtx = mock(AppContext.class);
- ClusterInfo clusterInfo = mock(ClusterInfo.class);
- when(appCtx.getClusterInfo()).thenReturn(clusterInfo);
- setupTaskAttemptFinishingMonitor(eventHandler, jobConf, appCtx);
-
- TaskAttemptImpl taImpl =
- new MapTaskAttemptImpl(taskId, 1, eventHandler, jobFile, 1,
- splits, jobConf, taListener,mock(Token.class), new Credentials(),
- new SystemClock(), appCtx);
-
- NodeId nid = NodeId.newInstance("127.0.0.1", 0);
- ContainerId contId = ContainerId.newContainerId(appAttemptId, 3);
- Container container = mock(Container.class);
- when(container.getId()).thenReturn(contId);
- when(container.getNodeId()).thenReturn(nid);
- when(container.getNodeHttpAddress()).thenReturn("localhost:0");
-
- taImpl.handle(new TaskAttemptEvent(attemptId,
- TaskAttemptEventType.TA_SCHEDULE));
- taImpl.handle(new TaskAttemptContainerAssignedEvent(attemptId,
- container, mock(Map.class)));
- taImpl.handle(new TaskAttemptContainerLaunchedEvent(attemptId, 0));
- taImpl.handle(new TaskAttemptEvent(attemptId,
- TaskAttemptEventType.TA_DONE));
- taImpl.handle(new TaskAttemptEvent(attemptId,
- TaskAttemptEventType.TA_CONTAINER_COMPLETED));
-
- assertEquals("Task attempt is not in succeeded state", taImpl.getState(),
- TaskAttemptState.SUCCEEDED);
-
- assertTrue("Task Attempt finish time is not greater than 0",
- taImpl.getFinishTime() > 0);
-
- Long finishTime = taImpl.getFinishTime();
- Thread.sleep(5);
- taImpl.handle(new TaskAttemptEvent(attemptId,
- TaskAttemptEventType.TA_TOO_MANY_FETCH_FAILURE));
-
- assertEquals("Task attempt is not in Too Many Fetch Failure state",
- taImpl.getState(), TaskAttemptState.FAILED);
-
- assertEquals("After TA_TOO_MANY_FETCH_FAILURE,"
- + " Task attempt finish time is not the same ",
- finishTime, Long.valueOf(taImpl.getFinishTime()));
+ ApplicationId appId = ApplicationId.newInstance(1, 2);
+ ApplicationAttemptId appAttemptId =
+ ApplicationAttemptId.newInstance(appId, 0);
+ JobId jobId = MRBuilderUtils.newJobId(appId, 1);
+ TaskId taskId = MRBuilderUtils.newTaskId(jobId, 1, TaskType.MAP);
+ TaskAttemptId attemptId = MRBuilderUtils.newTaskAttemptId(taskId, 0);
+ TaskId reducetaskId = MRBuilderUtils.newTaskId(jobId, 1, TaskType.REDUCE);
+ TaskAttemptId reduceTAId =
+ MRBuilderUtils.newTaskAttemptId(reducetaskId, 0);
+ Path jobFile = mock(Path.class);
+
+ MockEventHandler eventHandler = new MockEventHandler();
+ TaskAttemptListener taListener = mock(TaskAttemptListener.class);
+ when(taListener.getAddress()).thenReturn(
+ new InetSocketAddress("localhost", 0));
+
+ JobConf jobConf = new JobConf();
+ jobConf.setClass("fs.file.impl", StubbedFS.class, FileSystem.class);
+ jobConf.setBoolean("fs.file.impl.disable.cache", true);
+ jobConf.set(JobConf.MAPRED_MAP_TASK_ENV, "");
+ jobConf.set(MRJobConfig.APPLICATION_ATTEMPT_ID, "10");
+
+ TaskSplitMetaInfo splits = mock(TaskSplitMetaInfo.class);
+ when(splits.getLocations()).thenReturn(new String[] {"127.0.0.1"});
+
+ AppContext appCtx = mock(AppContext.class);
+ ClusterInfo clusterInfo = mock(ClusterInfo.class);
+ when(appCtx.getClusterInfo()).thenReturn(clusterInfo);
+ setupTaskAttemptFinishingMonitor(eventHandler, jobConf, appCtx);
+
+ TaskAttemptImpl taImpl =
+ new MapTaskAttemptImpl(taskId, 1, eventHandler, jobFile, 1,
+ splits, jobConf, taListener,mock(Token.class), new Credentials(),
+ new SystemClock(), appCtx);
+
+ NodeId nid = NodeId.newInstance("127.0.0.1", 0);
+ ContainerId contId = ContainerId.newContainerId(appAttemptId, 3);
+ Container container = mock(Container.class);
+ when(container.getId()).thenReturn(contId);
+ when(container.getNodeId()).thenReturn(nid);
+ when(container.getNodeHttpAddress()).thenReturn("localhost:0");
+
+ taImpl.handle(new TaskAttemptEvent(attemptId,
+ TaskAttemptEventType.TA_SCHEDULE));
+ taImpl.handle(new TaskAttemptContainerAssignedEvent(attemptId,
+ container, mock(Map.class)));
+ taImpl.handle(new TaskAttemptContainerLaunchedEvent(attemptId, 0));
+ taImpl.handle(new TaskAttemptEvent(attemptId,
+ TaskAttemptEventType.TA_DONE));
+ taImpl.handle(new TaskAttemptEvent(attemptId,
+ TaskAttemptEventType.TA_CONTAINER_COMPLETED));
+
+ assertEquals("Task attempt is not in succeeded state", taImpl.getState(),
+ TaskAttemptState.SUCCEEDED);
+
+ assertTrue("Task Attempt finish time is not greater than 0",
+ taImpl.getFinishTime() > 0);
+
+ Long finishTime = taImpl.getFinishTime();
+ Thread.sleep(5);
+ taImpl.handle(new TaskAttemptTooManyFetchFailureEvent(attemptId,
+ reduceTAId, "Host"));
+
+ assertEquals("Task attempt is not in Too Many Fetch Failure state",
+ taImpl.getState(), TaskAttemptState.FAILED);
+
+ assertEquals("After TA_TOO_MANY_FETCH_FAILURE,"
+ + " Task attempt finish time is not the same ",
+ finishTime, Long.valueOf(taImpl.getFinishTime()));
}
@Test
[40/46] hadoop git commit: YARN-3823. Fix mismatch in default values
for yarn.scheduler.maximum-allocation-vcores property. Contributed by Ray
Chiang.
Posted by ar...@apache.org.
YARN-3823. Fix mismatch in default values for
yarn.scheduler.maximum-allocation-vcores property. Contributed by Ray
Chiang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7405c597
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7405c597
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7405c597
Branch: refs/heads/HDFS-7240
Commit: 7405c59799ed1b8ad1a7c6f1b18fabf49d0b92b2
Parents: 532e38c
Author: Devaraj K <de...@apache.org>
Authored: Wed Jul 1 11:11:26 2015 +0530
Committer: Devaraj K <de...@apache.org>
Committed: Wed Jul 1 11:11:26 2015 +0530
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +++
.../hadoop-yarn-common/src/main/resources/yarn-default.xml | 2 +-
2 files changed, 4 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7405c597/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 5e9243d..8c07e48 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -580,6 +580,9 @@ Release 2.8.0 - UNRELEASED
YARN-3768. ArrayIndexOutOfBoundsException with empty environment variables.
(Zhihai Xu via gera)
+ YARN-3823. Fix mismatch in default values for
+ yarn.scheduler.maximum-allocation-vcores property. (Ray Chiang via devaraj)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7405c597/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index 621198c..4987084 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -276,7 +276,7 @@
in terms of virtual CPU cores. Requests higher than this will throw a
InvalidResourceRequestException.</description>
<name>yarn.scheduler.maximum-allocation-vcores</name>
- <value>32</value>
+ <value>4</value>
</property>
<property>
[16/46] hadoop git commit: HADOOP-12089. StorageException complaining
" no lease ID" when updating FolderLastModifiedTime in WASB. Contributed by
Duo Xu.
Posted by ar...@apache.org.
HADOOP-12089. StorageException complaining " no lease ID" when updating FolderLastModifiedTime in WASB. Contributed by Duo Xu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/460e98f7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/460e98f7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/460e98f7
Branch: refs/heads/HDFS-7240
Commit: 460e98f7b3ec84f3c5afcb2aad4f4e7031d16e3a
Parents: 62e583c
Author: cnauroth <cn...@apache.org>
Authored: Mon Jun 29 13:48:02 2015 -0700
Committer: cnauroth <cn...@apache.org>
Committed: Mon Jun 29 13:48:02 2015 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java | 5 ++---
2 files changed, 5 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/460e98f7/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 9008ead..a9b44e3 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -893,6 +893,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-12119. hadoop fs -expunge does not work for federated namespace
(J.Andreina via vinayakumarb)
+ HADOOP-12089. StorageException complaining " no lease ID" when updating
+ FolderLastModifiedTime in WASB. (Duo Xu via cnauroth)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/460e98f7/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
index 623645a..a567b33 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
@@ -1500,7 +1500,7 @@ public class NativeAzureFileSystem extends FileSystem {
createPermissionStatus(FsPermission.getDefault()));
} else {
if (!skipParentFolderLastModifidedTimeUpdate) {
- store.updateFolderLastModifiedTime(parentKey, null);
+ updateParentFolderLastModifiedTime(key);
}
}
}
@@ -1561,9 +1561,8 @@ public class NativeAzureFileSystem extends FileSystem {
// Update parent directory last modified time
Path parent = absolutePath.getParent();
if (parent != null && parent.getParent() != null) { // not root
- String parentKey = pathToKey(parent);
if (!skipParentFolderLastModifidedTimeUpdate) {
- store.updateFolderLastModifiedTime(parentKey, null);
+ updateParentFolderLastModifiedTime(key);
}
}
instrumentation.directoryDeleted();
[13/46] hadoop git commit: HADOOP-12104. Migrate Hadoop Pipes native
build to new CMake framework (Alan Burlison via Colin P. McCabe)
Posted by ar...@apache.org.
HADOOP-12104. Migrate Hadoop Pipes native build to new CMake framework (Alan Burlison via Colin P. McCabe)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5a27c3fd
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5a27c3fd
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5a27c3fd
Branch: refs/heads/HDFS-7240
Commit: 5a27c3fd7616215937264c2b1f015205e60f2d73
Parents: 2ffd842
Author: Colin Patrick Mccabe <cm...@cloudera.com>
Authored: Mon Jun 29 12:01:17 2015 -0700
Committer: Colin Patrick Mccabe <cm...@cloudera.com>
Committed: Mon Jun 29 12:31:26 2015 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 ++
hadoop-tools/hadoop-pipes/src/CMakeLists.txt | 49 +++++++++-----------
2 files changed, 24 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a27c3fd/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 0a964a3..e8e85a0 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -685,6 +685,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-11885. hadoop-dist dist-layout-stitching.sh does not work with dash.
(wang)
+ HADOOP-12104. Migrate Hadoop Pipes native build to new CMake framework
+ (alanburlison via cmccabe)
+
HADOOP-12036. Consolidate all of the cmake extensions in one directory
(alanburlison via cmccabe)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5a27c3fd/hadoop-tools/hadoop-pipes/src/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-pipes/src/CMakeLists.txt b/hadoop-tools/hadoop-pipes/src/CMakeLists.txt
index 170af49..3b0b28c 100644
--- a/hadoop-tools/hadoop-pipes/src/CMakeLists.txt
+++ b/hadoop-tools/hadoop-pipes/src/CMakeLists.txt
@@ -6,7 +6,7 @@
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
-
+#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
@@ -17,25 +17,11 @@
#
cmake_minimum_required(VERSION 2.6 FATAL_ERROR)
-find_package(OpenSSL REQUIRED)
-
-set(CMAKE_BUILD_TYPE, Release)
-set(PIPES_FLAGS "-g -Wall -O2 -D_REENTRANT -D_GNU_SOURCE")
-set(PIPES_FLAGS "${PIPES_FLAGS} -D_LARGEFILE_SOURCE -D_FILE_OFFSET_BITS=64")
-set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${PIPES_FLAGS}")
-set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${PIPES_FLAGS}")
+list(APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/../../../hadoop-common-project/hadoop-common)
+include(HadoopCommon)
-include(../../../hadoop-common-project/hadoop-common/src/JNIFlags.cmake NO_POLICY_SCOPE)
-
-function(output_directory TGT DIR)
- SET_TARGET_PROPERTIES(${TGT} PROPERTIES
- RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${DIR}")
- SET_TARGET_PROPERTIES(${TGT} PROPERTIES
- ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${DIR}")
- SET_TARGET_PROPERTIES(${TGT} PROPERTIES
- LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${DIR}")
-endfunction(output_directory TGT DIR)
+find_package(OpenSSL REQUIRED)
include_directories(
main/native/utils/api
@@ -47,19 +33,19 @@ include_directories(
# Example programs
add_executable(wordcount-simple main/native/examples/impl/wordcount-simple.cc)
target_link_libraries(wordcount-simple hadooppipes hadooputils)
-output_directory(wordcount-simple examples)
+hadoop_output_directory(wordcount-simple examples)
add_executable(wordcount-part main/native/examples/impl/wordcount-part.cc)
target_link_libraries(wordcount-part hadooppipes hadooputils)
-output_directory(wordcount-part examples)
+hadoop_output_directory(wordcount-part examples)
add_executable(wordcount-nopipe main/native/examples/impl/wordcount-nopipe.cc)
target_link_libraries(wordcount-nopipe hadooppipes hadooputils)
-output_directory(wordcount-nopipe examples)
+hadoop_output_directory(wordcount-nopipe examples)
add_executable(pipes-sort main/native/examples/impl/sort.cc)
target_link_libraries(pipes-sort hadooppipes hadooputils)
-output_directory(pipes-sort examples)
+hadoop_output_directory(pipes-sort examples)
add_library(hadooputils STATIC
main/native/utils/impl/StringUtils.cc
@@ -70,15 +56,22 @@ add_library(hadooppipes STATIC
main/native/pipes/impl/HadoopPipes.cc
)
-INCLUDE(CheckLibraryExists)
-CHECK_LIBRARY_EXISTS(dl dlopen "" NEED_LINK_DL)
+include(CheckLibraryExists)
+check_library_exists(dl dlopen "" NEED_LINK_DL)
+
+if(NEED_LINK_DL)
+ set(LIB_DL "dl")
+endif()
-if (NEED_LINK_DL)
- set(LIB_DL dl)
-endif (NEED_LINK_DL)
+if(${CMAKE_SYSTEM_NAME} MATCHES "SunOS")
+ exec_program("uname" ARGS "-r" OUTPUT_VARIABLE OS_VERSION)
+ if(OS_VERSION VERSION_LESS "5.12")
+ set(LIB_NET "socket" "nsl")
+ endif()
+endif()
target_link_libraries(hadooppipes
${OPENSSL_LIBRARIES}
${LIB_DL}
- pthread
+ ${LIB_NET}
)
[20/46] hadoop git commit: HDFS-8659. Block scanner INFO message is
spamming logs. Contributed by Yongjun Zhang.
Posted by ar...@apache.org.
HDFS-8659. Block scanner INFO message is spamming logs. Contributed by Yongjun Zhang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/43a1288f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/43a1288f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/43a1288f
Branch: refs/heads/HDFS-7240
Commit: 43a1288fff5e323895fde66b9e3eb31a7412fcc2
Parents: 34ee0b9
Author: Yongjun Zhang <yz...@cloudera.com>
Authored: Mon Jun 29 14:35:39 2015 -0700
Committer: Yongjun Zhang <yz...@cloudera.com>
Committed: Mon Jun 29 14:43:56 2015 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 2 ++
.../org/apache/hadoop/hdfs/server/datanode/BlockScanner.java | 2 +-
.../apache/hadoop/hdfs/server/datanode/VolumeScanner.java | 8 ++++----
3 files changed, 7 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/43a1288f/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 0c56f2b..eb006eb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -985,6 +985,8 @@ Release 2.7.2 - UNRELEASED
IMPROVEMENTS
+ HDFS-8659. Block scanner INFO message is spamming logs. (Yongjun Zhang)
+
OPTIMIZATIONS
BUG FIXES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/43a1288f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockScanner.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockScanner.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockScanner.java
index 9c4dd10..be6aa83 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockScanner.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockScanner.java
@@ -317,7 +317,7 @@ public class BlockScanner {
*/
synchronized void markSuspectBlock(String storageId, ExtendedBlock block) {
if (!isEnabled()) {
- LOG.info("Not scanning suspicious block {} on {}, because the block " +
+ LOG.debug("Not scanning suspicious block {} on {}, because the block " +
"scanner is disabled.", block, storageId);
return;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/43a1288f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/VolumeScanner.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/VolumeScanner.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/VolumeScanner.java
index 615abe9..ff655c2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/VolumeScanner.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/VolumeScanner.java
@@ -656,24 +656,24 @@ public class VolumeScanner extends Thread {
public synchronized void markSuspectBlock(ExtendedBlock block) {
if (stopping) {
- LOG.info("{}: Not scheduling suspect block {} for " +
+ LOG.debug("{}: Not scheduling suspect block {} for " +
"rescanning, because this volume scanner is stopping.", this, block);
return;
}
Boolean recent = recentSuspectBlocks.getIfPresent(block);
if (recent != null) {
- LOG.info("{}: Not scheduling suspect block {} for " +
+ LOG.debug("{}: Not scheduling suspect block {} for " +
"rescanning, because we rescanned it recently.", this, block);
return;
}
if (suspectBlocks.contains(block)) {
- LOG.info("{}: suspect block {} is already queued for " +
+ LOG.debug("{}: suspect block {} is already queued for " +
"rescanning.", this, block);
return;
}
suspectBlocks.add(block);
recentSuspectBlocks.put(block, true);
- LOG.info("{}: Scheduling suspect block {} for rescanning.", this, block);
+ LOG.debug("{}: Scheduling suspect block {} for rescanning.", this, block);
notify(); // wake scanner thread.
}
[33/46] hadoop git commit: YARN-3827. Migrate YARN native build to
new CMake framework (Alan Burlison via Colin P. McCabe)
Posted by ar...@apache.org.
YARN-3827. Migrate YARN native build to new CMake framework (Alan Burlison via Colin P. McCabe)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d0cc0380
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d0cc0380
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d0cc0380
Branch: refs/heads/HDFS-7240
Commit: d0cc0380b57db5fdeb41775bb9ca42dac65928b8
Parents: 9c63825
Author: Colin Patrick Mccabe <cm...@cloudera.com>
Authored: Tue Jun 30 16:24:19 2015 -0700
Committer: Colin Patrick Mccabe <cm...@cloudera.com>
Committed: Tue Jun 30 16:24:19 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +++
.../src/CMakeLists.txt | 23 ++++++++++----------
2 files changed, 14 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0cc0380/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 94bc8fc..5e9243d 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -317,6 +317,9 @@ Release 2.8.0 - UNRELEASED
YARN-3834. Scrub debug logging of tokens during resource localization.
(Chris Nauroth via xgong)
+ YARN-3827. Migrate YARN native build to new CMake framework (Alan Burlison
+ via Colin P. McCabe)
+
OPTIMIZATIONS
YARN-3339. TestDockerContainerExecutor should pull a single image and not
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d0cc0380/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt
index f691a9c..d4d6ae1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt
@@ -16,25 +16,24 @@
cmake_minimum_required(VERSION 2.6 FATAL_ERROR)
-set(CMAKE_BUILD_TYPE, Release)
+list(APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/../../../../../hadoop-common-project/hadoop-common)
+include(HadoopCommon)
-include(../../../../../hadoop-common-project/hadoop-common/src/JNIFlags.cmake NO_POLICY_SCOPE)
+# Note: can't use -D_FILE_OFFSET_BITS=64, see MAPREDUCE-4258
+string(REPLACE "-D_FILE_OFFSET_BITS=64" "" CMAKE_C_FLAGS "${CMAKE_C_FLAGS}")
+string(REPLACE "-D_FILE_OFFSET_BITS=64" "" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}")
include(CheckFunctionExists)
-CHECK_FUNCTION_EXISTS(fcloseall HAVE_FCLOSEALL)
+check_function_exists(fcloseall HAVE_FCLOSEALL)
function(output_directory TGT DIR)
- SET_TARGET_PROPERTIES(${TGT} PROPERTIES
+ set_target_properties(${TGT} PROPERTIES
RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${DIR}")
- SET_TARGET_PROPERTIES(${TGT} PROPERTIES
+ set_target_properties(${TGT} PROPERTIES
ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${DIR}")
- SET_TARGET_PROPERTIES(${TGT} PROPERTIES
+ set_target_properties(${TGT} PROPERTIES
LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${DIR}")
-endfunction(output_directory TGT DIR)
-
-set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -Wall -O2 -D_GNU_SOURCE")
-# note: can't enable -D_LARGEFILE: see MAPREDUCE-4258
-set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D_REENTRANT")
+endfunction()
include_directories(
${CMAKE_CURRENT_SOURCE_DIR}
@@ -42,7 +41,7 @@ include_directories(
main/native/container-executor
main/native/container-executor/impl
)
-CONFIGURE_FILE(${CMAKE_SOURCE_DIR}/config.h.cmake ${CMAKE_BINARY_DIR}/config.h)
+configure_file(${CMAKE_SOURCE_DIR}/config.h.cmake ${CMAKE_BINARY_DIR}/config.h)
add_library(container
main/native/container-executor/impl/configuration.c
[18/46] hadoop git commit: HADOOP-12107. long running apps may have a
huge number of StatisticsData instances under FileSystem (Sangjin Lee via
Ming Ma)
Posted by ar...@apache.org.
HADOOP-12107. long running apps may have a huge number of StatisticsData instances under FileSystem (Sangjin Lee via Ming Ma)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8e1bdc17
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8e1bdc17
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8e1bdc17
Branch: refs/heads/HDFS-7240
Commit: 8e1bdc17d9134e01115ae7c929503d8ac0325207
Parents: 460e98f
Author: Ming Ma <mi...@apache.org>
Authored: Mon Jun 29 14:37:38 2015 -0700
Committer: Ming Ma <mi...@apache.org>
Committed: Mon Jun 29 14:37:38 2015 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +
.../java/org/apache/hadoop/fs/FileSystem.java | 140 +++++++++++++------
.../apache/hadoop/fs/FCStatisticsBaseTest.java | 56 +++++++-
3 files changed, 155 insertions(+), 44 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8e1bdc17/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index a9b44e3..50192ae 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -490,6 +490,9 @@ Trunk (Unreleased)
HADOOP-11347. RawLocalFileSystem#mkdir and create should honor umask (Varun
Saxena via Colin P. McCabe)
+ HADOOP-12107. long running apps may have a huge number of StatisticsData
+ instances under FileSystem (Sangjin Lee via Ming Ma)
+
OPTIMIZATIONS
HADOOP-7761. Improve the performance of raw comparisons. (todd)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8e1bdc17/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
index 3f9e3bd..1d7bc87 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
@@ -20,7 +20,8 @@ package org.apache.hadoop.fs;
import java.io.Closeable;
import java.io.FileNotFoundException;
import java.io.IOException;
-import java.lang.ref.WeakReference;
+import java.lang.ref.PhantomReference;
+import java.lang.ref.ReferenceQueue;
import java.net.URI;
import java.net.URISyntaxException;
import java.security.PrivilegedExceptionAction;
@@ -32,7 +33,6 @@ import java.util.HashMap;
import java.util.HashSet;
import java.util.IdentityHashMap;
import java.util.Iterator;
-import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.NoSuchElementException;
@@ -2920,16 +2920,6 @@ public abstract class FileSystem extends Configured implements Closeable {
volatile int readOps;
volatile int largeReadOps;
volatile int writeOps;
- /**
- * Stores a weak reference to the thread owning this StatisticsData.
- * This allows us to remove StatisticsData objects that pertain to
- * threads that no longer exist.
- */
- final WeakReference<Thread> owner;
-
- StatisticsData(WeakReference<Thread> owner) {
- this.owner = owner;
- }
/**
* Add another StatisticsData object to this one.
@@ -3000,17 +2990,37 @@ public abstract class FileSystem extends Configured implements Closeable {
* Thread-local data.
*/
private final ThreadLocal<StatisticsData> threadData;
-
+
/**
- * List of all thread-local data areas. Protected by the Statistics lock.
+ * Set of all thread-local data areas. Protected by the Statistics lock.
+ * The references to the statistics data are kept using phantom references
+ * to the associated threads. Proper clean-up is performed by the cleaner
+ * thread when the threads are garbage collected.
*/
- private LinkedList<StatisticsData> allData;
+ private final Set<StatisticsDataReference> allData;
+
+ /**
+ * Global reference queue and a cleaner thread that manage statistics data
+ * references from all filesystem instances.
+ */
+ private static final ReferenceQueue<Thread> STATS_DATA_REF_QUEUE;
+ private static final Thread STATS_DATA_CLEANER;
+
+ static {
+ STATS_DATA_REF_QUEUE = new ReferenceQueue<Thread>();
+ // start a single daemon cleaner thread
+ STATS_DATA_CLEANER = new Thread(new StatisticsDataReferenceCleaner());
+ STATS_DATA_CLEANER.
+ setName(StatisticsDataReferenceCleaner.class.getName());
+ STATS_DATA_CLEANER.setDaemon(true);
+ STATS_DATA_CLEANER.start();
+ }
public Statistics(String scheme) {
this.scheme = scheme;
- this.rootData = new StatisticsData(null);
+ this.rootData = new StatisticsData();
this.threadData = new ThreadLocal<StatisticsData>();
- this.allData = null;
+ this.allData = new HashSet<StatisticsDataReference>();
}
/**
@@ -3020,7 +3030,7 @@ public abstract class FileSystem extends Configured implements Closeable {
*/
public Statistics(Statistics other) {
this.scheme = other.scheme;
- this.rootData = new StatisticsData(null);
+ this.rootData = new StatisticsData();
other.visitAll(new StatisticsAggregator<Void>() {
@Override
public void accept(StatisticsData data) {
@@ -3032,6 +3042,63 @@ public abstract class FileSystem extends Configured implements Closeable {
}
});
this.threadData = new ThreadLocal<StatisticsData>();
+ this.allData = new HashSet<StatisticsDataReference>();
+ }
+
+ /**
+ * A phantom reference to a thread that also includes the data associated
+ * with that thread. On the thread being garbage collected, it is enqueued
+ * to the reference queue for clean-up.
+ */
+ private class StatisticsDataReference extends PhantomReference<Thread> {
+ private final StatisticsData data;
+
+ public StatisticsDataReference(StatisticsData data, Thread thread) {
+ super(thread, STATS_DATA_REF_QUEUE);
+ this.data = data;
+ }
+
+ public StatisticsData getData() {
+ return data;
+ }
+
+ /**
+ * Performs clean-up action when the associated thread is garbage
+ * collected.
+ */
+ public void cleanUp() {
+ // use the statistics lock for safety
+ synchronized (Statistics.this) {
+ /*
+ * If the thread that created this thread-local data no longer exists,
+ * remove the StatisticsData from our list and fold the values into
+ * rootData.
+ */
+ rootData.add(data);
+ allData.remove(this);
+ }
+ }
+ }
+
+ /**
+ * Background action to act on references being removed.
+ */
+ private static class StatisticsDataReferenceCleaner implements Runnable {
+ @Override
+ public void run() {
+ while (true) {
+ try {
+ StatisticsDataReference ref =
+ (StatisticsDataReference)STATS_DATA_REF_QUEUE.remove();
+ ref.cleanUp();
+ } catch (Throwable th) {
+ // the cleaner thread should continue to run even if there are
+ // exceptions, including InterruptedException
+ LOG.warn("exception in the cleaner thread but it will continue to "
+ + "run", th);
+ }
+ }
+ }
}
/**
@@ -3040,14 +3107,12 @@ public abstract class FileSystem extends Configured implements Closeable {
public StatisticsData getThreadStatistics() {
StatisticsData data = threadData.get();
if (data == null) {
- data = new StatisticsData(
- new WeakReference<Thread>(Thread.currentThread()));
+ data = new StatisticsData();
threadData.set(data);
+ StatisticsDataReference ref =
+ new StatisticsDataReference(data, Thread.currentThread());
synchronized(this) {
- if (allData == null) {
- allData = new LinkedList<StatisticsData>();
- }
- allData.add(data);
+ allData.add(ref);
}
}
return data;
@@ -3105,21 +3170,9 @@ public abstract class FileSystem extends Configured implements Closeable {
*/
private synchronized <T> T visitAll(StatisticsAggregator<T> visitor) {
visitor.accept(rootData);
- if (allData != null) {
- for (Iterator<StatisticsData> iter = allData.iterator();
- iter.hasNext(); ) {
- StatisticsData data = iter.next();
- visitor.accept(data);
- if (data.owner.get() == null) {
- /*
- * If the thread that created this thread-local data no
- * longer exists, remove the StatisticsData from our list
- * and fold the values into rootData.
- */
- rootData.add(data);
- iter.remove();
- }
- }
+ for (StatisticsDataReference ref: allData) {
+ StatisticsData data = ref.getData();
+ visitor.accept(data);
}
return visitor.aggregate();
}
@@ -3226,7 +3279,7 @@ public abstract class FileSystem extends Configured implements Closeable {
@Override
public String toString() {
return visitAll(new StatisticsAggregator<String>() {
- private StatisticsData total = new StatisticsData(null);
+ private StatisticsData total = new StatisticsData();
@Override
public void accept(StatisticsData data) {
@@ -3259,7 +3312,7 @@ public abstract class FileSystem extends Configured implements Closeable {
*/
public void reset() {
visitAll(new StatisticsAggregator<Void>() {
- private StatisticsData total = new StatisticsData(null);
+ private StatisticsData total = new StatisticsData();
@Override
public void accept(StatisticsData data) {
@@ -3281,6 +3334,11 @@ public abstract class FileSystem extends Configured implements Closeable {
public String getScheme() {
return scheme;
}
+
+ @VisibleForTesting
+ synchronized int getAllThreadLocalDataSize() {
+ return allData.size();
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8e1bdc17/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FCStatisticsBaseTest.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FCStatisticsBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FCStatisticsBaseTest.java
index 90337a6..3e33362 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FCStatisticsBaseTest.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FCStatisticsBaseTest.java
@@ -18,26 +18,34 @@
package org.apache.hadoop.fs;
+import static org.apache.hadoop.fs.FileContextTestHelper.createFile;
+
import java.io.IOException;
import java.net.URI;
import java.net.URISyntaxException;
+import java.util.ArrayList;
+import java.util.List;
import java.util.Map;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
import org.apache.hadoop.fs.FileSystem.Statistics;
+import org.apache.hadoop.test.GenericTestUtils;
import org.junit.Assert;
import org.junit.Test;
+import com.google.common.base.Supplier;
import com.google.common.util.concurrent.Uninterruptibles;
-import static org.apache.hadoop.fs.FileContextTestHelper.*;
-
/**
* <p>
* Base class to test {@link FileContext} Statistics.
* </p>
*/
public abstract class FCStatisticsBaseTest {
-
static protected int blockSize = 512;
static protected int numBlocks = 1;
@@ -102,6 +110,48 @@ public abstract class FCStatisticsBaseTest {
fc.delete(filePath, true);
}
+ @Test(timeout=60000)
+ public void testStatisticsThreadLocalDataCleanUp() throws Exception {
+ final Statistics stats = new Statistics("test");
+ // create a small thread pool to test the statistics
+ final int size = 2;
+ ExecutorService es = Executors.newFixedThreadPool(size);
+ List<Callable<Boolean>> tasks = new ArrayList<Callable<Boolean>>(size);
+ for (int i = 0; i < size; i++) {
+ tasks.add(new Callable<Boolean>() {
+ public Boolean call() {
+ // this populates the data set in statistics
+ stats.incrementReadOps(1);
+ return true;
+ }
+ });
+ }
+ // run the threads
+ es.invokeAll(tasks);
+ // assert that the data size is exactly the number of threads
+ final AtomicInteger allDataSize = new AtomicInteger(0);
+ allDataSize.set(stats.getAllThreadLocalDataSize());
+ Assert.assertEquals(size, allDataSize.get());
+ Assert.assertEquals(size, stats.getReadOps());
+ // force the GC to collect the threads by shutting down the thread pool
+ es.shutdownNow();
+ es.awaitTermination(1, TimeUnit.MINUTES);
+ es = null;
+ System.gc();
+
+ // wait for up to 10 seconds
+ GenericTestUtils.waitFor(new Supplier<Boolean>() {
+ @Override
+ public Boolean get() {
+ int size = stats.getAllThreadLocalDataSize();
+ allDataSize.set(size);
+ return size == 0;
+ }
+ }, 1000, 10*1000);
+ Assert.assertEquals(0, allDataSize.get());
+ Assert.assertEquals(size, stats.getReadOps());
+ }
+
/**
* Bytes read may be different for different file systems. This method should
* throw assertion error if bytes read are incorrect.
[39/46] hadoop git commit: HADOOP-12164. Fix TestMove and
TestFsShellReturnCode failed to get command name using reflection. (Lei Xu)
Posted by ar...@apache.org.
HADOOP-12164. Fix TestMove and TestFsShellReturnCode failed to get command name using reflection. (Lei Xu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/532e38cb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/532e38cb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/532e38cb
Branch: refs/heads/HDFS-7240
Commit: 532e38cb7f70606c2c96d05259670e1e91d60ab3
Parents: b8e792c
Author: Lei Xu <le...@apache.org>
Authored: Tue Jun 30 17:26:49 2015 -0700
Committer: Lei Xu <le...@apache.org>
Committed: Tue Jun 30 17:41:59 2015 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +
.../apache/hadoop/fs/TestFsShellReturnCode.java | 60 +++++++++++++++-----
.../org/apache/hadoop/fs/shell/TestMove.java | 1 +
3 files changed, 49 insertions(+), 15 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/532e38cb/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 74d428f..39e2e5e 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -916,6 +916,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-12116. Fix unrecommended syntax usages in hadoop/hdfs/yarn script for
cygwin in branch-2. (Li Lu via cnauroth)
+ HADOOP-12164. Fix TestMove and TestFsShellReturnCode failed to get command
+ name using reflection. (Lei (Eddy) Xu)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/532e38cb/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellReturnCode.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellReturnCode.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellReturnCode.java
index dcc19df..a2276ec 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellReturnCode.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFsShellReturnCode.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.fs;
+import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.HADOOP_SHELL_MISSING_DEFAULT_FS_WARNING_KEY;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@@ -402,7 +403,7 @@ public class TestFsShellReturnCode {
// processing a file throws an interrupt, it should blow on first file
assertEquals(1, InterruptCommand.processed);
assertEquals(130, exitCode);
-
+
exitCode = shell.run(
new String[]{ "-testInterrupt", d.toString() });
// processing a file throws an interrupt, it should blow on file
@@ -412,17 +413,32 @@ public class TestFsShellReturnCode {
}
/**
+ * Faked Chown class for {@link testChownUserAndGroupValidity()}.
+ *
+ * The test only covers argument parsing, so override to skip processing.
+ */
+ private static class FakeChown extends FsShellPermissions.Chown {
+ public static String NAME = "chown";
+ @Override
+ protected void processArgument(PathData item) {
+ }
+ }
+
+ /**
* Tests combinations of valid and invalid user and group arguments to chown.
*/
@Test
public void testChownUserAndGroupValidity() {
- // This test only covers argument parsing, so override to skip processing.
- FsCommand chown = new FsShellPermissions.Chown() {
- @Override
- protected void processArgument(PathData item) {
- }
- };
- chown.setConf(new Configuration());
+ testChownUserAndGroupValidity(true);
+ testChownUserAndGroupValidity(false);
+ }
+
+ private void testChownUserAndGroupValidity(boolean enableWarning) {
+ Configuration conf = new Configuration();
+ conf.setBoolean(
+ HADOOP_SHELL_MISSING_DEFAULT_FS_WARNING_KEY, enableWarning);
+ FsCommand chown = new FakeChown();
+ chown.setConf(conf);
// The following are valid (no exception expected).
chown.run("user", "/path");
@@ -447,17 +463,31 @@ public class TestFsShellReturnCode {
}
/**
+ * Faked Chgrp class for {@link testChgrpGroupValidity()}.
+ * The test only covers argument parsing, so override to skip processing.
+ */
+ private static class FakeChgrp extends FsShellPermissions.Chgrp {
+ public static String NAME = "chgrp";
+ @Override
+ protected void processArgument(PathData item) {
+ }
+ }
+
+ /**
* Tests valid and invalid group arguments to chgrp.
*/
@Test
public void testChgrpGroupValidity() {
- // This test only covers argument parsing, so override to skip processing.
- FsCommand chgrp = new FsShellPermissions.Chgrp() {
- @Override
- protected void processArgument(PathData item) {
- }
- };
- chgrp.setConf(new Configuration());
+ testChgrpGroupValidity(true);
+ testChgrpGroupValidity(false);
+ }
+
+ private void testChgrpGroupValidity(boolean enableWarning) {
+ Configuration conf = new Configuration();
+ conf.setBoolean(
+ HADOOP_SHELL_MISSING_DEFAULT_FS_WARNING_KEY, enableWarning);
+ FsShellPermissions.Chgrp chgrp = new FakeChgrp();
+ chgrp.setConf(conf);
// The following are valid (no exception expected).
chgrp.run("group", "/path");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/532e38cb/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestMove.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestMove.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestMove.java
index 6599edf..94930e5 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestMove.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/shell/TestMove.java
@@ -114,6 +114,7 @@ public class TestMove {
}
private static class InstrumentedRenameCommand extends MoveCommands.Rename {
+ public static String NAME = "InstrumentedRename";
private Exception error = null;
@Override
public void displayError(Exception e) {
[34/46] hadoop git commit: HADOOP-10798. globStatus() should always
return a sorted list of files (cmccabe)
Posted by ar...@apache.org.
HADOOP-10798. globStatus() should always return a sorted list of files (cmccabe)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/68e588cb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/68e588cb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/68e588cb
Branch: refs/heads/HDFS-7240
Commit: 68e588cbee660d55dba518892d064bee3795a002
Parents: d0cc038
Author: Colin Patrick Mccabe <cm...@cloudera.com>
Authored: Tue Jun 30 16:40:21 2015 -0700
Committer: Colin Patrick Mccabe <cm...@cloudera.com>
Committed: Tue Jun 30 16:40:21 2015 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../main/java/org/apache/hadoop/fs/Globber.java | 11 ++++++++-
.../org/apache/hadoop/fs/TestGlobPaths.java | 25 ++++++++++++++++++++
3 files changed, 38 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/68e588cb/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 2aab8b4..4d69a229 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -905,6 +905,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-12154. FileSystem#getUsed() returns the file length only from root '/'
(J.Andreina via vinayakumarb)
+ HADOOP-10798. globStatus() should always return a sorted list of files
+ (cmccabe)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/68e588cb/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Globber.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Globber.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Globber.java
index 8a8137a..9cb810f 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Globber.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/Globber.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.fs;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.List;
import org.apache.commons.logging.LogFactory;
@@ -285,6 +286,14 @@ class Globber {
(flattenedPatterns.size() <= 1)) {
return null;
}
- return results.toArray(new FileStatus[0]);
+ /*
+ * In general, the results list will already be sorted, since listStatus
+ * returns results in sorted order for many Hadoop filesystems. However,
+ * not all Hadoop filesystems have this property. So we sort here in order
+ * to get consistent results. See HADOOP-10798 for details.
+ */
+ FileStatus ret[] = results.toArray(new FileStatus[0]);
+ Arrays.sort(ret);
+ return ret;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/68e588cb/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestGlobPaths.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestGlobPaths.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestGlobPaths.java
index 50e2e5b..afd8fb2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestGlobPaths.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/fs/TestGlobPaths.java
@@ -21,9 +21,11 @@ import static org.junit.Assert.*;
import java.io.IOException;
import java.security.PrivilegedExceptionAction;
+import java.util.ArrayList;
import java.util.UUID;
import java.util.regex.Pattern;
+import com.google.common.collect.Ordering;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.permission.FsPermission;
@@ -1284,4 +1286,27 @@ public class TestGlobPaths {
public void testNonTerminalGlobsOnFC() throws Exception {
testOnFileContext(new TestNonTerminalGlobs(true));
}
+
+ @Test
+ public void testLocalFilesystem() throws Exception {
+ Configuration conf = new Configuration();
+ FileSystem fs = FileSystem.getLocal(conf);
+ String localTmp = System.getProperty("java.io.tmpdir");
+ Path base = new Path(new Path(localTmp), UUID.randomUUID().toString());
+ Assert.assertTrue(fs.mkdirs(base));
+ Assert.assertTrue(fs.mkdirs(new Path(base, "e")));
+ Assert.assertTrue(fs.mkdirs(new Path(base, "c")));
+ Assert.assertTrue(fs.mkdirs(new Path(base, "a")));
+ Assert.assertTrue(fs.mkdirs(new Path(base, "d")));
+ Assert.assertTrue(fs.mkdirs(new Path(base, "b")));
+ fs.deleteOnExit(base);
+ FileStatus[] status = fs.globStatus(new Path(base, "*"));
+ ArrayList list = new ArrayList();
+ for (FileStatus f: status) {
+ list.add(f.getPath().toString());
+ }
+ boolean sorted = Ordering.natural().isOrdered(list);
+ Assert.assertTrue(sorted);
+ }
}
+
[05/46] hadoop git commit: HDFS-8681. BlockScanner is incorrectly
disabled by default. (Contributed by Arpit Agarwal)
Posted by ar...@apache.org.
HDFS-8681. BlockScanner is incorrectly disabled by default. (Contributed by Arpit Agarwal)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c6793dd8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c6793dd8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c6793dd8
Branch: refs/heads/HDFS-7240
Commit: c6793dd8cc69ea994eb23c3e1349efe4b9feca9a
Parents: 3dfa816
Author: Arpit Agarwal <ar...@apache.org>
Authored: Sun Jun 28 14:51:17 2015 -0700
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Sun Jun 28 14:51:36 2015 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 2 +-
.../hdfs/server/datanode/BlockScanner.java | 36 ++++++++++++++++----
.../src/main/resources/hdfs-default.xml | 9 +++--
.../fsdataset/impl/TestFsDatasetImpl.java | 1 +
5 files changed, 41 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c6793dd8/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index e287ea4..94477fe 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -1113,6 +1113,9 @@ Release 2.7.1 - UNRELEASED
HDFS08656. Preserve compatibility of ClientProtocol#rollingUpgrade after
finalization. (wang)
+ HDFS-8681. BlockScanner is incorrectly disabled by default.
+ (Arpit Agarwal)
+
Release 2.7.0 - 2015-04-20
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c6793dd8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index ebd668f..0e569f0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -378,7 +378,7 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
public static final String DFS_DATANODE_MAX_RECEIVER_THREADS_KEY = "dfs.datanode.max.transfer.threads";
public static final int DFS_DATANODE_MAX_RECEIVER_THREADS_DEFAULT = 4096;
public static final String DFS_DATANODE_SCAN_PERIOD_HOURS_KEY = "dfs.datanode.scan.period.hours";
- public static final int DFS_DATANODE_SCAN_PERIOD_HOURS_DEFAULT = 0;
+ public static final int DFS_DATANODE_SCAN_PERIOD_HOURS_DEFAULT = 21 * 24; // 3 weeks.
public static final String DFS_BLOCK_SCANNER_VOLUME_BYTES_PER_SECOND = "dfs.block.scanner.volume.bytes.per.second";
public static final long DFS_BLOCK_SCANNER_VOLUME_BYTES_PER_SECOND_DEFAULT = 1048576L;
public static final String DFS_DATANODE_TRANSFERTO_ALLOWED_KEY = "dfs.datanode.transferTo.allowed";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c6793dd8/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockScanner.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockScanner.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockScanner.java
index b0248c5..9c4dd10 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockScanner.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/BlockScanner.java
@@ -115,6 +115,34 @@ public class BlockScanner {
}
}
+ /**
+ * Determine the configured block scanner interval.
+ *
+ * For compatibility with prior releases of HDFS, if the
+ * configured value is zero then the scan period is
+ * set to 3 weeks.
+ *
+ * If the configured value is less than zero then the scanner
+ * is disabled.
+ *
+ * @param conf Configuration object.
+ * @return block scan period in milliseconds.
+ */
+ private static long getConfiguredScanPeriodMs(Configuration conf) {
+ long tempScanPeriodMs = getUnitTestLong(
+ conf, INTERNAL_DFS_DATANODE_SCAN_PERIOD_MS,
+ TimeUnit.MILLISECONDS.convert(conf.getLong(
+ DFS_DATANODE_SCAN_PERIOD_HOURS_KEY,
+ DFS_DATANODE_SCAN_PERIOD_HOURS_DEFAULT), TimeUnit.HOURS));
+
+ if (tempScanPeriodMs == 0) {
+ tempScanPeriodMs = TimeUnit.MILLISECONDS.convert(
+ DFS_DATANODE_SCAN_PERIOD_HOURS_DEFAULT, TimeUnit.HOURS);
+ }
+
+ return tempScanPeriodMs;
+ }
+
@SuppressWarnings("unchecked")
Conf(Configuration conf) {
this.targetBytesPerSec = Math.max(0L, conf.getLong(
@@ -123,11 +151,7 @@ public class BlockScanner {
this.maxStalenessMs = Math.max(0L, getUnitTestLong(conf,
INTERNAL_DFS_BLOCK_SCANNER_MAX_STALENESS_MS,
INTERNAL_DFS_BLOCK_SCANNER_MAX_STALENESS_MS_DEFAULT));
- this.scanPeriodMs = Math.max(0L,
- getUnitTestLong(conf, INTERNAL_DFS_DATANODE_SCAN_PERIOD_MS,
- TimeUnit.MILLISECONDS.convert(conf.getLong(
- DFS_DATANODE_SCAN_PERIOD_HOURS_KEY,
- DFS_DATANODE_SCAN_PERIOD_HOURS_DEFAULT), TimeUnit.HOURS)));
+ this.scanPeriodMs = getConfiguredScanPeriodMs(conf);
this.cursorSaveMs = Math.max(0L, getUnitTestLong(conf,
INTERNAL_DFS_BLOCK_SCANNER_CURSOR_SAVE_INTERVAL_MS,
INTERNAL_DFS_BLOCK_SCANNER_CURSOR_SAVE_INTERVAL_MS_DEFAULT));
@@ -159,7 +183,7 @@ public class BlockScanner {
* no threads will start.
*/
public boolean isEnabled() {
- return (conf.scanPeriodMs) > 0 && (conf.targetBytesPerSec > 0);
+ return (conf.scanPeriodMs > 0) && (conf.targetBytesPerSec > 0);
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c6793dd8/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index 76161a5..8cb7d5f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -1071,11 +1071,14 @@
<property>
<name>dfs.datanode.scan.period.hours</name>
- <value>0</value>
+ <value>504</value>
<description>
- If this is 0 or negative, the DataNode's block scanner will be
- disabled. If this is positive, the DataNode will not scan any
+ If this is positive, the DataNode will not scan any
individual block more than once in the specified scan period.
+ If this is negative, the block scanner is disabled.
+ If this is set to zero, then the default value of 504 hours
+ or 3 weeks is used. Prior versions of HDFS incorrectly documented
+ that setting this key to zero will disable the block scanner.
</description>
</property>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c6793dd8/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsDatasetImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsDatasetImpl.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsDatasetImpl.java
index 59c7ade..d03fa2c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsDatasetImpl.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsDatasetImpl.java
@@ -274,6 +274,7 @@ public class TestFsDatasetImpl {
public void testChangeVolumeWithRunningCheckDirs() throws IOException {
RoundRobinVolumeChoosingPolicy<FsVolumeImpl> blockChooser =
new RoundRobinVolumeChoosingPolicy<>();
+ conf.setLong(DFSConfigKeys.DFS_DATANODE_SCAN_PERIOD_HOURS_KEY, -1);
final BlockScanner blockScanner = new BlockScanner(datanode, conf);
final FsVolumeList volumeList = new FsVolumeList(
Collections.<VolumeFailureInfo>emptyList(), blockScanner, blockChooser);
[46/46] hadoop git commit: Merge remote-tracking branch
'apache-commit/trunk' into HDFS-7240
Posted by ar...@apache.org.
Merge remote-tracking branch 'apache-commit/trunk' into HDFS-7240
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6551e11c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6551e11c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6551e11c
Branch: refs/heads/HDFS-7240
Commit: 6551e11c0e6c3fa870a1560aef8699f088c0b190
Parents: 71dfa13 eac1d18
Author: Arpit Agarwal <ar...@apache.org>
Authored: Wed Jul 1 14:09:14 2015 -0700
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Wed Jul 1 14:09:14 2015 -0700
----------------------------------------------------------------------
dev-support/releasedocmaker.py | 76 +++-
dev-support/test-patch.sh | 4 +-
hadoop-common-project/hadoop-common/CHANGES.txt | 53 +++
.../java/org/apache/hadoop/fs/FileSystem.java | 146 +++++---
.../java/org/apache/hadoop/fs/FileUtil.java | 42 +++
.../main/java/org/apache/hadoop/fs/FsShell.java | 16 +-
.../main/java/org/apache/hadoop/fs/Globber.java | 11 +-
.../java/org/apache/hadoop/fs/shell/Delete.java | 17 +-
.../main/java/org/apache/hadoop/util/Shell.java | 6 +-
.../org/apache/hadoop/io/nativeio/NativeIO.c | 1 +
.../org/apache/hadoop/net/unix/DomainSocket.c | 8 +-
.../src/org/apache/hadoop/util/NativeCrc32.c | 4 +-
...yptoStreamsWithOpensslAesCtrCryptoCodec.java | 9 +-
.../apache/hadoop/fs/FCStatisticsBaseTest.java | 56 ++-
.../java/org/apache/hadoop/fs/TestFileUtil.java | 112 +++++-
.../apache/hadoop/fs/TestFsShellReturnCode.java | 60 ++-
.../java/org/apache/hadoop/fs/TestTrash.java | 14 +-
.../org/apache/hadoop/fs/shell/TestMove.java | 1 +
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 44 +++
.../hadoop-hdfs/src/CMakeLists.txt | 129 ++-----
.../src/contrib/libwebhdfs/CMakeLists.txt | 8 +-
.../libwebhdfs/resources/FindJansson.cmake | 9 +-
.../hadoop-hdfs/src/main/bin/hdfs | 1 +
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 2 +-
.../server/blockmanagement/BlockManager.java | 18 +-
.../BlockPlacementPolicyDefault.java | 3 +-
.../blockmanagement/BlockStatsMXBean.java | 36 ++
.../CacheReplicationMonitor.java | 4 +-
.../blockmanagement/DatanodeDescriptor.java | 33 +-
.../server/blockmanagement/DatanodeManager.java | 26 +-
.../blockmanagement/DatanodeStatistics.java | 6 +
.../blockmanagement/DatanodeStorageInfo.java | 13 +-
.../blockmanagement/HeartbeatManager.java | 96 ++++-
.../blockmanagement/StorageTypeStats.java | 115 ++++++
.../hdfs/server/datanode/BlockScanner.java | 38 +-
.../hdfs/server/datanode/VolumeScanner.java | 8 +-
.../hdfs/server/namenode/FSDirTruncateOp.java | 361 +++++++++++++++++++
.../hdfs/server/namenode/FSDirectory.java | 95 -----
.../hdfs/server/namenode/FSEditLogLoader.java | 6 +-
.../hdfs/server/namenode/FSNamesystem.java | 237 ++----------
.../org/apache/hadoop/hdfs/tools/DFSck.java | 1 -
.../hdfs/tools/DelegationTokenFetcher.java | 19 +-
.../src/main/native/fuse-dfs/CMakeLists.txt | 91 ++---
.../src/main/resources/hdfs-default.xml | 9 +-
.../src/site/markdown/HDFSCommands.md | 12 +-
.../org/apache/hadoop/fs/TestGlobPaths.java | 25 ++
.../org/apache/hadoop/hdfs/TestDFSShell.java | 26 ++
.../blockmanagement/TestBlockStatsMXBean.java | 146 ++++++++
.../fsdataset/impl/TestFsDatasetImpl.java | 1 +
.../hdfs/server/namenode/TestDeadDatanode.java | 42 +++
.../hdfs/server/namenode/TestFileTruncate.java | 8 +-
.../hdfs/tools/TestDelegationTokenFetcher.java | 15 +
hadoop-mapreduce-project/CHANGES.txt | 30 ++
.../hadoop/mapred/LocalContainerLauncher.java | 2 +-
.../jobhistory/JobHistoryEventHandler.java | 19 +-
.../hadoop/mapreduce/v2/app/MRAppMaster.java | 14 +-
.../event/JobTaskAttemptFetchFailureEvent.java | 9 +-
.../TaskAttemptTooManyFetchFailureEvent.java | 50 +++
.../mapreduce/v2/app/job/impl/JobImpl.java | 7 +-
.../v2/app/job/impl/TaskAttemptImpl.java | 15 +-
.../hadoop/mapreduce/jobhistory/TestEvents.java | 3 +-
.../apache/hadoop/mapreduce/v2/app/MRApp.java | 15 +
.../mapreduce/v2/app/TestFetchFailure.java | 31 +-
.../v2/app/job/impl/TestTaskAttempt.java | 143 ++++----
.../mapreduce/v2/jobhistory/JHAdminConfig.java | 7 +
.../hadoop/mapreduce/JobResourceUploader.java | 38 +-
.../mapreduce/jobhistory/EventReader.java | 12 +-
.../mapreduce/jobhistory/EventWriter.java | 29 +-
.../src/main/resources/mapred-default.xml | 9 +
.../src/CMakeLists.txt | 337 +++++++----------
.../src/JNIFlags.cmake | 118 ------
.../hadoop/fs/azure/NativeAzureFileSystem.java | 5 +-
.../java/org/apache/hadoop/tools/DistCp.java | 3 +-
.../apache/hadoop/tools/util/DistCpUtils.java | 39 --
hadoop-tools/hadoop-pipes/src/CMakeLists.txt | 49 ++-
hadoop-yarn-project/CHANGES.txt | 39 ++
.../hadoop/yarn/client/cli/RMAdminCLI.java | 10 +-
.../hadoop/yarn/client/cli/TestRMAdminCLI.java | 4 +
.../impl/pb/SerializedExceptionPBImpl.java | 2 +-
.../apache/hadoop/yarn/client/ServerProxy.java | 21 +-
.../java/org/apache/hadoop/yarn/util/Apps.java | 19 +-
.../src/main/resources/yarn-default.xml | 2 +-
.../impl/pb/TestSerializedExceptionPBImpl.java | 12 +-
.../org/apache/hadoop/yarn/util/TestApps.java | 61 ++++
.../src/CMakeLists.txt | 23 +-
.../containermanager/TestNMProxy.java | 81 +++--
.../scheduler/AbstractYarnScheduler.java | 34 +-
.../scheduler/capacity/LeafQueue.java | 3 +-
.../server/resourcemanager/TestRMRestart.java | 8 +-
.../scheduler/TestAbstractYarnScheduler.java | 74 +++-
90 files changed, 2469 insertions(+), 1227 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6551e11c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
----------------------------------------------------------------------
diff --cc hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index 983064e,0b60a97..2004a9c
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@@ -39,9 -39,8 +39,11 @@@ import java.util.TreeSet
import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.atomic.AtomicLong;
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Sets;
+ import javax.management.ObjectName;
+
import org.apache.hadoop.HadoopIllegalArgumentException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
@@@ -75,12 -83,21 +77,17 @@@ import org.apache.hadoop.hdfs.server.pr
import org.apache.hadoop.hdfs.server.protocol.DatanodeRegistration;
import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage;
import org.apache.hadoop.hdfs.server.protocol.DatanodeStorage.State;
-import org.apache.hadoop.hdfs.server.protocol.KeyUpdateCommand;
-import org.apache.hadoop.hdfs.server.protocol.ReceivedDeletedBlockInfo;
-import org.apache.hadoop.hdfs.server.protocol.StorageReceivedDeletedBlocks;
import org.apache.hadoop.hdfs.util.LightWeightLinkedSet;
+ import org.apache.hadoop.metrics2.util.MBeans;
import org.apache.hadoop.net.Node;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.Daemon;
import org.apache.hadoop.util.LightWeightGSet;
import org.apache.hadoop.util.Time;
+
+ import com.google.common.annotations.VisibleForTesting;
+ import com.google.common.base.Preconditions;
+ import com.google.common.collect.Sets;
-
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6551e11c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6551e11c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6551e11c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/datanode/VolumeScanner.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6551e11c/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/datanode/fsdataset/impl/TestFsDatasetImpl.java
----------------------------------------------------------------------
[09/46] hadoop git commit: HADOOP-12119. hadoop fs -expunge does not
work for federated namespace (Contributed by J.Andreina)
Posted by ar...@apache.org.
HADOOP-12119. hadoop fs -expunge does not work for federated namespace (Contributed by J.Andreina)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c815344e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c815344e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c815344e
Branch: refs/heads/HDFS-7240
Commit: c815344e2e68d78f6587b65bc2db25e151aa4364
Parents: 88ceb38
Author: Vinayakumar B <vi...@apache.org>
Authored: Mon Jun 29 15:58:54 2015 +0530
Committer: Vinayakumar B <vi...@apache.org>
Committed: Mon Jun 29 15:58:54 2015 +0530
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../java/org/apache/hadoop/fs/shell/Delete.java | 17 ++++++++++++++---
.../test/java/org/apache/hadoop/fs/TestTrash.java | 14 ++++++++++++--
3 files changed, 29 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c815344e/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 219ef25..0a964a3 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -884,6 +884,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-12076. Incomplete Cache Mechanism in CredentialProvider API.
(Larry McCay via cnauroth)
+ HADOOP-12119. hadoop fs -expunge does not work for federated namespace
+ (J.Andreina via vinayakumarb)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c815344e/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Delete.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Delete.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Delete.java
index f882817..40d9478 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Delete.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Delete.java
@@ -25,6 +25,7 @@ import java.util.List;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.PathIOException;
import org.apache.hadoop.fs.PathIsDirectoryException;
import org.apache.hadoop.fs.PathIsNotDirectoryException;
@@ -195,9 +196,19 @@ class Delete {
@Override
protected void processArguments(LinkedList<PathData> args)
throws IOException {
- Trash trash = new Trash(getConf());
- trash.expunge();
- trash.checkpoint();
+ FileSystem[] childFileSystems =
+ FileSystem.get(getConf()).getChildFileSystems();
+ if (null != childFileSystems) {
+ for (FileSystem fs : childFileSystems) {
+ Trash trash = new Trash(fs, getConf());
+ trash.expunge();
+ trash.checkpoint();
+ }
+ } else {
+ Trash trash = new Trash(getConf());
+ trash.expunge();
+ trash.checkpoint();
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c815344e/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java
index a675e30..9a91733 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestTrash.java
@@ -594,8 +594,18 @@ public class TestTrash extends TestCase {
TestLFS() {
this(new Path(TEST_DIR, "user/test"));
}
- TestLFS(Path home) {
- super();
+ TestLFS(final Path home) {
+ super(new RawLocalFileSystem() {
+ @Override
+ protected Path getInitialWorkingDirectory() {
+ return makeQualified(home);
+ }
+
+ @Override
+ public Path getHomeDirectory() {
+ return makeQualified(home);
+ }
+ });
this.home = home;
}
@Override
[30/46] hadoop git commit: HADOOP-12158. Improve error message in
TestCryptoStreamsWithOpensslAesCtrCryptoCodec when OpenSSL is not installed.
Posted by ar...@apache.org.
HADOOP-12158. Improve error message in TestCryptoStreamsWithOpensslAesCtrCryptoCodec when OpenSSL is not installed.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9ee7b6e6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9ee7b6e6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9ee7b6e6
Branch: refs/heads/HDFS-7240
Commit: 9ee7b6e6c4ab6bee6304fa7904993c7cbd9a6cd2
Parents: 6f2a41e
Author: Andrew Wang <wa...@apache.org>
Authored: Tue Jun 30 15:04:18 2015 -0700
Committer: Andrew Wang <wa...@apache.org>
Committed: Tue Jun 30 15:04:33 2015 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 5 +++++
.../TestCryptoStreamsWithOpensslAesCtrCryptoCodec.java | 9 +++++++--
2 files changed, 12 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ee7b6e6/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index e332ea8..2aab8b4 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -666,6 +666,11 @@ Release 2.8.0 - UNRELEASED
HADOOP-11958. MetricsSystemImpl fails to show backtrace when an error
occurs (Jason Lowe via jeagles)
+
+ HADOOP-12158. Improve error message in
+ TestCryptoStreamsWithOpensslAesCtrCryptoCodec when OpenSSL is not
+ installed. (wang)
+
OPTIMIZATIONS
HADOOP-11785. Reduce the number of listStatus operation in distcp
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9ee7b6e6/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreamsWithOpensslAesCtrCryptoCodec.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreamsWithOpensslAesCtrCryptoCodec.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreamsWithOpensslAesCtrCryptoCodec.java
index 4f90a0c..684ec09 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreamsWithOpensslAesCtrCryptoCodec.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/TestCryptoStreamsWithOpensslAesCtrCryptoCodec.java
@@ -19,9 +19,11 @@ package org.apache.hadoop.crypto;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
-import org.junit.Assert;
import org.junit.BeforeClass;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
public class TestCryptoStreamsWithOpensslAesCtrCryptoCodec
extends TestCryptoStreams {
@@ -32,7 +34,10 @@ public class TestCryptoStreamsWithOpensslAesCtrCryptoCodec
CommonConfigurationKeysPublic.HADOOP_SECURITY_CRYPTO_CODEC_CLASSES_AES_CTR_NOPADDING_KEY,
OpensslAesCtrCryptoCodec.class.getName());
codec = CryptoCodec.getInstance(conf);
- Assert.assertEquals(OpensslAesCtrCryptoCodec.class.getCanonicalName(),
+ assertNotNull("Unable to instantiate codec " +
+ OpensslAesCtrCryptoCodec.class.getName() + ", is the required "
+ + "version of OpenSSL installed?", codec);
+ assertEquals(OpensslAesCtrCryptoCodec.class.getCanonicalName(),
codec.getClass().getCanonicalName());
}
}
[27/46] hadoop git commit: HADOOP-12149. copy all of test-patch
BINDIR prior to re-exec (aw)
Posted by ar...@apache.org.
HADOOP-12149. copy all of test-patch BINDIR prior to re-exec (aw)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/147e020c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/147e020c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/147e020c
Branch: refs/heads/HDFS-7240
Commit: 147e020c7aef3ba42eddcef3be1b4ae7c7910371
Parents: 076948d
Author: Allen Wittenauer <aw...@apache.org>
Authored: Tue Jun 30 11:31:26 2015 -0700
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Tue Jun 30 11:31:26 2015 -0700
----------------------------------------------------------------------
dev-support/test-patch.sh | 4 ++--
hadoop-common-project/hadoop-common/CHANGES.txt | 2 ++
2 files changed, 4 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/147e020c/dev-support/test-patch.sh
----------------------------------------------------------------------
diff --git a/dev-support/test-patch.sh b/dev-support/test-patch.sh
index cd91a5c..efcd614 100755
--- a/dev-support/test-patch.sh
+++ b/dev-support/test-patch.sh
@@ -1521,8 +1521,8 @@ function check_reexec
cd "${CWD}"
mkdir -p "${PATCH_DIR}/dev-support-test"
- cp -pr "${BASEDIR}"/dev-support/test-patch* "${PATCH_DIR}/dev-support-test"
- cp -pr "${BASEDIR}"/dev-support/smart-apply* "${PATCH_DIR}/dev-support-test"
+ (cd "${BINDIR}"; tar cpf - . ) \
+ | (cd "${PATCH_DIR}/dev-support-test"; tar xpf - )
big_console_header "exec'ing test-patch.sh now..."
http://git-wip-us.apache.org/repos/asf/hadoop/blob/147e020c/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index c010ff1..e332ea8 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -232,6 +232,8 @@ Trunk (Unreleased)
HADOOP-11807. add a lint mode to releasedocmaker (ramtin via aw)
+ HADOOP-12149. copy all of test-patch BINDIR prior to re-exec (aw)
+
BUG FIXES
HADOOP-11473. test-patch says "-1 overall" even when all checks are +1
[22/46] hadoop git commit: HDFS-8493. Consolidate truncate() related
implementation in a single class. Contributed by Rakesh R.
Posted by ar...@apache.org.
HDFS-8493. Consolidate truncate() related implementation in a single class. Contributed by Rakesh R.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d3797f9f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d3797f9f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d3797f9f
Branch: refs/heads/HDFS-7240
Commit: d3797f9f3cf502b7bfee3b64c641807b276c6faf
Parents: 8e33372
Author: Haohui Mai <wh...@apache.org>
Authored: Mon Jun 29 16:40:46 2015 -0700
Committer: Haohui Mai <wh...@apache.org>
Committed: Mon Jun 29 16:45:35 2015 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
.../hdfs/server/namenode/FSDirTruncateOp.java | 361 +++++++++++++++++++
.../hdfs/server/namenode/FSDirectory.java | 95 -----
.../hdfs/server/namenode/FSEditLogLoader.java | 6 +-
.../hdfs/server/namenode/FSNamesystem.java | 237 ++----------
.../hdfs/server/namenode/TestFileTruncate.java | 8 +-
6 files changed, 402 insertions(+), 308 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3797f9f/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 108a6c0..3535f90 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -685,6 +685,9 @@ Release 2.8.0 - UNRELEASED
HDFS-8653. Code cleanup for DatanodeManager, DatanodeDescriptor and
DatanodeStorageInfo. (Zhe Zhang via wang)
+ HDFS-8493. Consolidate truncate() related implementation in a single class.
+ (Rakesh R via wheat9)
+
OPTIMIZATIONS
HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3797f9f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirTruncateOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirTruncateOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirTruncateOp.java
new file mode 100644
index 0000000..9fc9def
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirTruncateOp.java
@@ -0,0 +1,361 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.namenode;
+
+import java.io.IOException;
+
+import org.apache.hadoop.HadoopIllegalArgumentException;
+import org.apache.hadoop.fs.UnresolvedLinkException;
+import org.apache.hadoop.fs.permission.FsAction;
+import org.apache.hadoop.hdfs.protocol.Block;
+import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
+import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
+import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
+import org.apache.hadoop.hdfs.protocol.SnapshotAccessControlException;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstructionContiguous;
+import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.BlockUCState;
+import org.apache.hadoop.hdfs.server.namenode.FSNamesystem.RecoverLeaseOp;
+import org.apache.hadoop.hdfs.server.namenode.INode.BlocksMapUpdateInfo;
+
+import com.google.common.annotations.VisibleForTesting;
+
+/**
+ * Helper class to perform truncate operation.
+ */
+final class FSDirTruncateOp {
+
+ /**
+ * Private constructor for preventing FSDirTruncateOp object creation.
+ * Static-only class.
+ */
+ private FSDirTruncateOp() {}
+
+ /**
+ * Truncate a file to a given size.
+ *
+ * @param fsn namespace
+ * @param srcArg path name
+ * @param newLength the target file size
+ * @param clientName client name
+ * @param clientMachine client machine info
+ * @param mtime modified time
+ * @param toRemoveBlocks to be removed blocks
+ * @param pc permission checker to check fs permission
+ * @return tuncate result
+ * @throws IOException
+ */
+ static TruncateResult truncate(final FSNamesystem fsn, final String srcArg,
+ final long newLength, final String clientName,
+ final String clientMachine, final long mtime,
+ final BlocksMapUpdateInfo toRemoveBlocks, final FSPermissionChecker pc)
+ throws IOException, UnresolvedLinkException {
+ assert fsn.hasWriteLock();
+
+ FSDirectory fsd = fsn.getFSDirectory();
+ byte[][] pathComponents = FSDirectory
+ .getPathComponentsForReservedPath(srcArg);
+ final String src;
+ final INodesInPath iip;
+ final boolean onBlockBoundary;
+ Block truncateBlock = null;
+ fsd.writeLock();
+ try {
+ src = fsd.resolvePath(pc, srcArg, pathComponents);
+ iip = fsd.getINodesInPath4Write(src, true);
+ if (fsn.isPermissionEnabled()) {
+ fsd.checkPathAccess(pc, iip, FsAction.WRITE);
+ }
+ INodeFile file = INodeFile.valueOf(iip.getLastINode(), src);
+ final BlockStoragePolicy lpPolicy = fsn.getBlockManager()
+ .getStoragePolicy("LAZY_PERSIST");
+
+ if (lpPolicy != null && lpPolicy.getId() == file.getStoragePolicyID()) {
+ throw new UnsupportedOperationException(
+ "Cannot truncate lazy persist file " + src);
+ }
+
+ // Check if the file is already being truncated with the same length
+ final BlockInfo last = file.getLastBlock();
+ if (last != null && last.getBlockUCState()
+ == BlockUCState.UNDER_RECOVERY) {
+ final Block truncatedBlock = ((BlockInfoUnderConstruction) last)
+ .getTruncateBlock();
+ if (truncatedBlock != null) {
+ final long truncateLength = file.computeFileSize(false, false)
+ + truncatedBlock.getNumBytes();
+ if (newLength == truncateLength) {
+ return new TruncateResult(false, fsd.getAuditFileInfo(iip));
+ }
+ }
+ }
+
+ // Opening an existing file for truncate. May need lease recovery.
+ fsn.recoverLeaseInternal(RecoverLeaseOp.TRUNCATE_FILE, iip, src,
+ clientName, clientMachine, false);
+ // Truncate length check.
+ long oldLength = file.computeFileSize();
+ if (oldLength == newLength) {
+ return new TruncateResult(true, fsd.getAuditFileInfo(iip));
+ }
+ if (oldLength < newLength) {
+ throw new HadoopIllegalArgumentException(
+ "Cannot truncate to a larger file size. Current size: " + oldLength
+ + ", truncate size: " + newLength + ".");
+ }
+ // Perform INodeFile truncation.
+ final QuotaCounts delta = new QuotaCounts.Builder().build();
+ onBlockBoundary = unprotectedTruncate(fsn, iip, newLength,
+ toRemoveBlocks, mtime, delta);
+ if (!onBlockBoundary) {
+ // Open file for write, but don't log into edits
+ long lastBlockDelta = file.computeFileSize() - newLength;
+ assert lastBlockDelta > 0 : "delta is 0 only if on block bounday";
+ truncateBlock = prepareFileForTruncate(fsn, iip, clientName,
+ clientMachine, lastBlockDelta, null);
+ }
+
+ // update the quota: use the preferred block size for UC block
+ fsd.updateCountNoQuotaCheck(iip, iip.length() - 1, delta);
+ } finally {
+ fsd.writeUnlock();
+ }
+
+ fsn.getEditLog().logTruncate(src, clientName, clientMachine, newLength,
+ mtime, truncateBlock);
+ return new TruncateResult(onBlockBoundary, fsd.getAuditFileInfo(iip));
+ }
+
+ /**
+ * Unprotected truncate implementation. Unlike
+ * {@link FSDirTruncateOp#truncate}, this will not schedule block recovery.
+ *
+ * @param fsn namespace
+ * @param src path name
+ * @param clientName client name
+ * @param clientMachine client machine info
+ * @param newLength the target file size
+ * @param mtime modified time
+ * @param truncateBlock truncate block
+ * @throws IOException
+ */
+ static void unprotectedTruncate(final FSNamesystem fsn, final String src,
+ final String clientName, final String clientMachine,
+ final long newLength, final long mtime, final Block truncateBlock)
+ throws UnresolvedLinkException, QuotaExceededException,
+ SnapshotAccessControlException, IOException {
+ assert fsn.hasWriteLock();
+
+ FSDirectory fsd = fsn.getFSDirectory();
+ INodesInPath iip = fsd.getINodesInPath(src, true);
+ INodeFile file = iip.getLastINode().asFile();
+ BlocksMapUpdateInfo collectedBlocks = new BlocksMapUpdateInfo();
+ boolean onBlockBoundary = unprotectedTruncate(fsn, iip, newLength,
+ collectedBlocks, mtime, null);
+
+ if (!onBlockBoundary) {
+ BlockInfo oldBlock = file.getLastBlock();
+ Block tBlk = prepareFileForTruncate(fsn, iip, clientName, clientMachine,
+ file.computeFileSize() - newLength, truncateBlock);
+ assert Block.matchingIdAndGenStamp(tBlk, truncateBlock) &&
+ tBlk.getNumBytes() == truncateBlock.getNumBytes() :
+ "Should be the same block.";
+ if (oldBlock.getBlockId() != tBlk.getBlockId()
+ && !file.isBlockInLatestSnapshot(oldBlock)) {
+ fsn.getBlockManager().removeBlockFromMap(oldBlock);
+ }
+ }
+ assert onBlockBoundary == (truncateBlock == null) :
+ "truncateBlock is null iff on block boundary: " + truncateBlock;
+ fsn.removeBlocksAndUpdateSafemodeTotal(collectedBlocks);
+ }
+
+ /**
+ * Convert current INode to UnderConstruction. Recreate lease. Create new
+ * block for the truncated copy. Schedule truncation of the replicas.
+ *
+ * @param fsn namespace
+ * @param iip inodes in the path containing the file
+ * @param leaseHolder lease holder
+ * @param clientMachine client machine info
+ * @param lastBlockDelta last block delta size
+ * @param newBlock new block
+ * @return the returned block will be written to editLog and passed back
+ * into this method upon loading.
+ * @throws IOException
+ */
+ @VisibleForTesting
+ static Block prepareFileForTruncate(FSNamesystem fsn, INodesInPath iip,
+ String leaseHolder, String clientMachine, long lastBlockDelta,
+ Block newBlock) throws IOException {
+ assert fsn.hasWriteLock();
+
+ INodeFile file = iip.getLastINode().asFile();
+ file.recordModification(iip.getLatestSnapshotId());
+ file.toUnderConstruction(leaseHolder, clientMachine);
+ assert file.isUnderConstruction() : "inode should be under construction.";
+ fsn.getLeaseManager().addLease(
+ file.getFileUnderConstructionFeature().getClientName(), file.getId());
+ boolean shouldRecoverNow = (newBlock == null);
+ BlockInfo oldBlock = file.getLastBlock();
+ boolean shouldCopyOnTruncate = shouldCopyOnTruncate(fsn, file, oldBlock);
+ if (newBlock == null) {
+ newBlock = (shouldCopyOnTruncate) ? fsn.createNewBlock() : new Block(
+ oldBlock.getBlockId(), oldBlock.getNumBytes(),
+ fsn.nextGenerationStamp(fsn.getBlockIdManager().isLegacyBlock(
+ oldBlock)));
+ }
+
+ BlockInfoUnderConstruction truncatedBlockUC;
+ if (shouldCopyOnTruncate) {
+ // Add new truncateBlock into blocksMap and
+ // use oldBlock as a source for copy-on-truncate recovery
+ truncatedBlockUC = new BlockInfoUnderConstructionContiguous(newBlock,
+ file.getPreferredBlockReplication());
+ truncatedBlockUC.setNumBytes(oldBlock.getNumBytes() - lastBlockDelta);
+ truncatedBlockUC.setTruncateBlock(oldBlock);
+ file.setLastBlock(truncatedBlockUC,
+ fsn.getBlockManager().getStorages(oldBlock));
+ fsn.getBlockManager().addBlockCollection(truncatedBlockUC, file);
+
+ NameNode.stateChangeLog.debug(
+ "BLOCK* prepareFileForTruncate: Scheduling copy-on-truncate to new"
+ + " size {} new block {} old block {}",
+ truncatedBlockUC.getNumBytes(), newBlock,
+ truncatedBlockUC.getTruncateBlock());
+ } else {
+ // Use new generation stamp for in-place truncate recovery
+ fsn.getBlockManager().convertLastBlockToUnderConstruction(file,
+ lastBlockDelta);
+ oldBlock = file.getLastBlock();
+ assert !oldBlock.isComplete() : "oldBlock should be under construction";
+ truncatedBlockUC = (BlockInfoUnderConstruction) oldBlock;
+ truncatedBlockUC.setTruncateBlock(new Block(oldBlock));
+ truncatedBlockUC.getTruncateBlock().setNumBytes(
+ oldBlock.getNumBytes() - lastBlockDelta);
+ truncatedBlockUC.getTruncateBlock().setGenerationStamp(
+ newBlock.getGenerationStamp());
+
+ NameNode.stateChangeLog.debug(
+ "BLOCK* prepareFileForTruncate: {} Scheduling in-place block "
+ + "truncate to new size {}", truncatedBlockUC.getTruncateBlock()
+ .getNumBytes(), truncatedBlockUC);
+ }
+ if (shouldRecoverNow) {
+ truncatedBlockUC.initializeBlockRecovery(newBlock.getGenerationStamp());
+ }
+
+ return newBlock;
+ }
+
+ /**
+ * Truncate has the following properties:
+ * 1.) Any block deletions occur now.
+ * 2.) INode length is truncated now - new clients can only read up to
+ * the truncated length.
+ * 3.) INode will be set to UC and lastBlock set to UNDER_RECOVERY.
+ * 4.) NN will trigger DN truncation recovery and waits for DNs to report.
+ * 5.) File is considered UNDER_RECOVERY until truncation recovery
+ * completes.
+ * 6.) Soft and hard Lease expiration require truncation recovery to
+ * complete.
+ *
+ * @return true if on the block boundary or false if recovery is need
+ */
+ private static boolean unprotectedTruncate(FSNamesystem fsn,
+ INodesInPath iip, long newLength, BlocksMapUpdateInfo collectedBlocks,
+ long mtime, QuotaCounts delta) throws IOException {
+ assert fsn.hasWriteLock();
+
+ INodeFile file = iip.getLastINode().asFile();
+ int latestSnapshot = iip.getLatestSnapshotId();
+ file.recordModification(latestSnapshot, true);
+
+ verifyQuotaForTruncate(fsn, iip, file, newLength, delta);
+
+ long remainingLength =
+ file.collectBlocksBeyondMax(newLength, collectedBlocks);
+ file.excludeSnapshotBlocks(latestSnapshot, collectedBlocks);
+ file.setModificationTime(mtime);
+ // return whether on a block boundary
+ return (remainingLength - newLength) == 0;
+ }
+
+ private static void verifyQuotaForTruncate(FSNamesystem fsn,
+ INodesInPath iip, INodeFile file, long newLength, QuotaCounts delta)
+ throws QuotaExceededException {
+ FSDirectory fsd = fsn.getFSDirectory();
+ if (!fsn.isImageLoaded() || fsd.shouldSkipQuotaChecks()) {
+ // Do not check quota if edit log is still being processed
+ return;
+ }
+ final BlockStoragePolicy policy = fsd.getBlockStoragePolicySuite()
+ .getPolicy(file.getStoragePolicyID());
+ file.computeQuotaDeltaForTruncate(newLength, policy, delta);
+ fsd.readLock();
+ try {
+ FSDirectory.verifyQuota(iip, iip.length() - 1, delta, null);
+ } finally {
+ fsd.readUnlock();
+ }
+ }
+
+ /**
+ * Defines if a replica needs to be copied on truncate or
+ * can be truncated in place.
+ */
+ private static boolean shouldCopyOnTruncate(FSNamesystem fsn, INodeFile file,
+ BlockInfo blk) {
+ if (!fsn.isUpgradeFinalized()) {
+ return true;
+ }
+ if (fsn.isRollingUpgrade()) {
+ return true;
+ }
+ return file.isBlockInLatestSnapshot(blk);
+ }
+
+ /**
+ * Result of truncate operation.
+ */
+ static class TruncateResult {
+ private final boolean result;
+ private final HdfsFileStatus stat;
+
+ public TruncateResult(boolean result, HdfsFileStatus stat) {
+ this.result = result;
+ this.stat = stat;
+ }
+
+ /**
+ * @return true if client does not need to wait for block recovery,
+ * false if client needs to wait for block recovery.
+ */
+ boolean getResult() {
+ return result;
+ }
+
+ /**
+ * @return file information.
+ */
+ HdfsFileStatus getFileStatus() {
+ return stat;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3797f9f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
index c807fba..ccee1ae 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
@@ -38,7 +38,6 @@ import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.XAttrHelper;
-import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.BlockStoragePolicy;
import org.apache.hadoop.hdfs.protocol.EncryptionZone;
import org.apache.hadoop.hdfs.protocol.FSLimitException.MaxDirectoryItemsExceededException;
@@ -49,11 +48,9 @@ import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
import org.apache.hadoop.hdfs.protocol.SnapshotAccessControlException;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos;
import org.apache.hadoop.hdfs.protocolPB.PBHelper;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockStoragePolicySuite;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants;
-import org.apache.hadoop.hdfs.server.namenode.INode.BlocksMapUpdateInfo;
import org.apache.hadoop.hdfs.util.ByteArray;
import org.apache.hadoop.hdfs.util.EnumCounters;
import org.apache.hadoop.security.AccessControlException;
@@ -908,98 +905,6 @@ public class FSDirectory implements Closeable {
}
/**
- * FSEditLogLoader implementation.
- * Unlike FSNamesystem.truncate, this will not schedule block recovery.
- */
- void unprotectedTruncate(String src, String clientName, String clientMachine,
- long newLength, long mtime, Block truncateBlock)
- throws UnresolvedLinkException, QuotaExceededException,
- SnapshotAccessControlException, IOException {
- INodesInPath iip = getINodesInPath(src, true);
- INodeFile file = iip.getLastINode().asFile();
- BlocksMapUpdateInfo collectedBlocks = new BlocksMapUpdateInfo();
- boolean onBlockBoundary =
- unprotectedTruncate(iip, newLength, collectedBlocks, mtime, null);
-
- if(! onBlockBoundary) {
- BlockInfo oldBlock = file.getLastBlock();
- Block tBlk =
- getFSNamesystem().prepareFileForTruncate(iip,
- clientName, clientMachine, file.computeFileSize() - newLength,
- truncateBlock);
- assert Block.matchingIdAndGenStamp(tBlk, truncateBlock) &&
- tBlk.getNumBytes() == truncateBlock.getNumBytes() :
- "Should be the same block.";
- if(oldBlock.getBlockId() != tBlk.getBlockId() &&
- !file.isBlockInLatestSnapshot(oldBlock)) {
- getBlockManager().removeBlockFromMap(oldBlock);
- }
- }
- assert onBlockBoundary == (truncateBlock == null) :
- "truncateBlock is null iff on block boundary: " + truncateBlock;
- getFSNamesystem().removeBlocksAndUpdateSafemodeTotal(collectedBlocks);
- }
-
- boolean truncate(INodesInPath iip, long newLength,
- BlocksMapUpdateInfo collectedBlocks,
- long mtime, QuotaCounts delta)
- throws IOException {
- writeLock();
- try {
- return unprotectedTruncate(iip, newLength, collectedBlocks, mtime, delta);
- } finally {
- writeUnlock();
- }
- }
-
- /**
- * Truncate has the following properties:
- * 1.) Any block deletions occur now.
- * 2.) INode length is truncated now – new clients can only read up to
- * the truncated length.
- * 3.) INode will be set to UC and lastBlock set to UNDER_RECOVERY.
- * 4.) NN will trigger DN truncation recovery and waits for DNs to report.
- * 5.) File is considered UNDER_RECOVERY until truncation recovery completes.
- * 6.) Soft and hard Lease expiration require truncation recovery to complete.
- *
- * @return true if on the block boundary or false if recovery is need
- */
- boolean unprotectedTruncate(INodesInPath iip, long newLength,
- BlocksMapUpdateInfo collectedBlocks,
- long mtime, QuotaCounts delta) throws IOException {
- assert hasWriteLock();
- INodeFile file = iip.getLastINode().asFile();
- int latestSnapshot = iip.getLatestSnapshotId();
- file.recordModification(latestSnapshot, true);
-
- verifyQuotaForTruncate(iip, file, newLength, delta);
-
- long remainingLength =
- file.collectBlocksBeyondMax(newLength, collectedBlocks);
- file.excludeSnapshotBlocks(latestSnapshot, collectedBlocks);
- file.setModificationTime(mtime);
- // return whether on a block boundary
- return (remainingLength - newLength) == 0;
- }
-
- private void verifyQuotaForTruncate(INodesInPath iip, INodeFile file,
- long newLength, QuotaCounts delta) throws QuotaExceededException {
- if (!getFSNamesystem().isImageLoaded() || shouldSkipQuotaChecks()) {
- // Do not check quota if edit log is still being processed
- return;
- }
- final BlockStoragePolicy policy = getBlockStoragePolicySuite()
- .getPolicy(file.getStoragePolicyID());
- file.computeQuotaDeltaForTruncate(newLength, policy, delta);
- readLock();
- try {
- verifyQuota(iip, iip.length() - 1, delta, null);
- } finally {
- readUnlock();
- }
- }
-
- /**
* This method is always called with writeLock of FSDirectory held.
*/
public final void addToInodeMap(INode inode) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3797f9f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
index df01edd..63ef985 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
@@ -901,9 +901,9 @@ public class FSEditLogLoader {
}
case OP_TRUNCATE: {
TruncateOp truncateOp = (TruncateOp) op;
- fsDir.unprotectedTruncate(truncateOp.src, truncateOp.clientName,
- truncateOp.clientMachine, truncateOp.newLength, truncateOp.timestamp,
- truncateOp.truncateBlock);
+ FSDirTruncateOp.unprotectedTruncate(fsNamesys, truncateOp.src,
+ truncateOp.clientName, truncateOp.clientMachine,
+ truncateOp.newLength, truncateOp.timestamp, truncateOp.truncateBlock);
break;
}
case OP_SET_STORAGE_POLICY: {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3797f9f/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index e95007b..7c6d6a1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -201,7 +201,6 @@ import org.apache.hadoop.hdfs.server.blockmanagement.BlockCollection;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockIdManager;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstruction;
-import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoUnderConstructionContiguous;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeDescriptor;
import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeManager;
@@ -1831,218 +1830,44 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
* block recovery to truncate the last block of the file.
*
* @return true if client does not need to wait for block recovery,
- * false if client needs to wait for block recovery.
+ * false if client needs to wait for block recovery.
*/
- boolean truncate(String src, long newLength,
- String clientName, String clientMachine,
- long mtime)
- throws IOException, UnresolvedLinkException {
+ boolean truncate(String src, long newLength, String clientName,
+ String clientMachine, long mtime) throws IOException,
+ UnresolvedLinkException {
+
requireEffectiveLayoutVersionForFeature(Feature.TRUNCATE);
- boolean ret;
+ final FSDirTruncateOp.TruncateResult r;
try {
- ret = truncateInt(src, newLength, clientName, clientMachine, mtime);
+ NameNode.stateChangeLog.debug(
+ "DIR* NameSystem.truncate: src={} newLength={}", src, newLength);
+ if (newLength < 0) {
+ throw new HadoopIllegalArgumentException(
+ "Cannot truncate to a negative file size: " + newLength + ".");
+ }
+ final FSPermissionChecker pc = getPermissionChecker();
+ checkOperation(OperationCategory.WRITE);
+ writeLock();
+ BlocksMapUpdateInfo toRemoveBlocks = new BlocksMapUpdateInfo();
+ try {
+ checkOperation(OperationCategory.WRITE);
+ checkNameNodeSafeMode("Cannot truncate for " + src);
+ r = FSDirTruncateOp.truncate(this, src, newLength, clientName,
+ clientMachine, mtime, toRemoveBlocks, pc);
+ } finally {
+ writeUnlock();
+ }
+ getEditLog().logSync();
+ if (!toRemoveBlocks.getToDeleteList().isEmpty()) {
+ removeBlocks(toRemoveBlocks);
+ toRemoveBlocks.clear();
+ }
+ logAuditEvent(true, "truncate", src, null, r.getFileStatus());
} catch (AccessControlException e) {
logAuditEvent(false, "truncate", src);
throw e;
}
- return ret;
- }
-
- boolean truncateInt(String srcArg, long newLength,
- String clientName, String clientMachine,
- long mtime)
- throws IOException, UnresolvedLinkException {
- String src = srcArg;
- NameNode.stateChangeLog.debug(
- "DIR* NameSystem.truncate: src={} newLength={}", src, newLength);
- if (newLength < 0) {
- throw new HadoopIllegalArgumentException(
- "Cannot truncate to a negative file size: " + newLength + ".");
- }
- HdfsFileStatus stat = null;
- FSPermissionChecker pc = getPermissionChecker();
- checkOperation(OperationCategory.WRITE);
- boolean res;
- byte[][] pathComponents = FSDirectory.getPathComponentsForReservedPath(src);
- writeLock();
- BlocksMapUpdateInfo toRemoveBlocks = new BlocksMapUpdateInfo();
- try {
- checkOperation(OperationCategory.WRITE);
- checkNameNodeSafeMode("Cannot truncate for " + src);
- src = dir.resolvePath(pc, src, pathComponents);
- res = truncateInternal(src, newLength, clientName,
- clientMachine, mtime, pc, toRemoveBlocks);
- stat = dir.getAuditFileInfo(dir.getINodesInPath4Write(src, false));
- } finally {
- writeUnlock();
- }
- getEditLog().logSync();
- if (!toRemoveBlocks.getToDeleteList().isEmpty()) {
- removeBlocks(toRemoveBlocks);
- toRemoveBlocks.clear();
- }
- logAuditEvent(true, "truncate", src, null, stat);
- return res;
- }
-
- /**
- * Truncate a file to a given size
- * Update the count at each ancestor directory with quota
- */
- boolean truncateInternal(String src, long newLength,
- String clientName, String clientMachine,
- long mtime, FSPermissionChecker pc,
- BlocksMapUpdateInfo toRemoveBlocks)
- throws IOException, UnresolvedLinkException {
- assert hasWriteLock();
- INodesInPath iip = dir.getINodesInPath4Write(src, true);
- if (isPermissionEnabled) {
- dir.checkPathAccess(pc, iip, FsAction.WRITE);
- }
- INodeFile file = INodeFile.valueOf(iip.getLastINode(), src);
- final BlockStoragePolicy lpPolicy =
- blockManager.getStoragePolicy("LAZY_PERSIST");
-
- if (lpPolicy != null &&
- lpPolicy.getId() == file.getStoragePolicyID()) {
- throw new UnsupportedOperationException(
- "Cannot truncate lazy persist file " + src);
- }
-
- // Check if the file is already being truncated with the same length
- final BlockInfo last = file.getLastBlock();
- if (last != null && last.getBlockUCState() == BlockUCState.UNDER_RECOVERY) {
- final Block truncateBlock
- = ((BlockInfoUnderConstruction)last).getTruncateBlock();
- if (truncateBlock != null) {
- final long truncateLength = file.computeFileSize(false, false)
- + truncateBlock.getNumBytes();
- if (newLength == truncateLength) {
- return false;
- }
- }
- }
-
- // Opening an existing file for truncate. May need lease recovery.
- recoverLeaseInternal(RecoverLeaseOp.TRUNCATE_FILE,
- iip, src, clientName, clientMachine, false);
- // Truncate length check.
- long oldLength = file.computeFileSize();
- if(oldLength == newLength) {
- return true;
- }
- if(oldLength < newLength) {
- throw new HadoopIllegalArgumentException(
- "Cannot truncate to a larger file size. Current size: " + oldLength +
- ", truncate size: " + newLength + ".");
- }
- // Perform INodeFile truncation.
- final QuotaCounts delta = new QuotaCounts.Builder().build();
- boolean onBlockBoundary = dir.truncate(iip, newLength, toRemoveBlocks,
- mtime, delta);
- Block truncateBlock = null;
- if(!onBlockBoundary) {
- // Open file for write, but don't log into edits
- long lastBlockDelta = file.computeFileSize() - newLength;
- assert lastBlockDelta > 0 : "delta is 0 only if on block bounday";
- truncateBlock = prepareFileForTruncate(iip, clientName, clientMachine,
- lastBlockDelta, null);
- }
-
- // update the quota: use the preferred block size for UC block
- dir.writeLock();
- try {
- dir.updateCountNoQuotaCheck(iip, iip.length() - 1, delta);
- } finally {
- dir.writeUnlock();
- }
-
- getEditLog().logTruncate(src, clientName, clientMachine, newLength, mtime,
- truncateBlock);
- return onBlockBoundary;
- }
-
- /**
- * Convert current INode to UnderConstruction.
- * Recreate lease.
- * Create new block for the truncated copy.
- * Schedule truncation of the replicas.
- *
- * @return the returned block will be written to editLog and passed back into
- * this method upon loading.
- */
- Block prepareFileForTruncate(INodesInPath iip,
- String leaseHolder,
- String clientMachine,
- long lastBlockDelta,
- Block newBlock)
- throws IOException {
- INodeFile file = iip.getLastINode().asFile();
- file.recordModification(iip.getLatestSnapshotId());
- file.toUnderConstruction(leaseHolder, clientMachine);
- assert file.isUnderConstruction() : "inode should be under construction.";
- leaseManager.addLease(
- file.getFileUnderConstructionFeature().getClientName(), file.getId());
- boolean shouldRecoverNow = (newBlock == null);
- BlockInfo oldBlock = file.getLastBlock();
- boolean shouldCopyOnTruncate = shouldCopyOnTruncate(file, oldBlock);
- if(newBlock == null) {
- newBlock = (shouldCopyOnTruncate) ? createNewBlock() :
- new Block(oldBlock.getBlockId(), oldBlock.getNumBytes(),
- nextGenerationStamp(blockIdManager.isLegacyBlock(oldBlock)));
- }
-
- BlockInfoUnderConstruction truncatedBlockUC;
- if(shouldCopyOnTruncate) {
- // Add new truncateBlock into blocksMap and
- // use oldBlock as a source for copy-on-truncate recovery
- truncatedBlockUC = new BlockInfoUnderConstructionContiguous(newBlock,
- file.getPreferredBlockReplication());
- truncatedBlockUC.setNumBytes(oldBlock.getNumBytes() - lastBlockDelta);
- truncatedBlockUC.setTruncateBlock(oldBlock);
- file.setLastBlock(truncatedBlockUC, blockManager.getStorages(oldBlock));
- getBlockManager().addBlockCollection(truncatedBlockUC, file);
-
- NameNode.stateChangeLog.debug(
- "BLOCK* prepareFileForTruncate: Scheduling copy-on-truncate to new" +
- " size {} new block {} old block {}", truncatedBlockUC.getNumBytes(),
- newBlock, truncatedBlockUC.getTruncateBlock());
- } else {
- // Use new generation stamp for in-place truncate recovery
- blockManager.convertLastBlockToUnderConstruction(file, lastBlockDelta);
- oldBlock = file.getLastBlock();
- assert !oldBlock.isComplete() : "oldBlock should be under construction";
- truncatedBlockUC = (BlockInfoUnderConstruction) oldBlock;
- truncatedBlockUC.setTruncateBlock(new Block(oldBlock));
- truncatedBlockUC.getTruncateBlock().setNumBytes(
- oldBlock.getNumBytes() - lastBlockDelta);
- truncatedBlockUC.getTruncateBlock().setGenerationStamp(
- newBlock.getGenerationStamp());
-
- NameNode.stateChangeLog.debug(
- "BLOCK* prepareFileForTruncate: {} Scheduling in-place block " +
- "truncate to new size {}",
- truncatedBlockUC.getTruncateBlock().getNumBytes(), truncatedBlockUC);
- }
- if (shouldRecoverNow) {
- truncatedBlockUC.initializeBlockRecovery(newBlock.getGenerationStamp());
- }
-
- return newBlock;
- }
-
- /**
- * Defines if a replica needs to be copied on truncate or
- * can be truncated in place.
- */
- boolean shouldCopyOnTruncate(INodeFile file, BlockInfo blk) {
- if(!isUpgradeFinalized()) {
- return true;
- }
- if (isRollingUpgrade()) {
- return true;
- }
- return file.isBlockInLatestSnapshot(blk);
+ return r.getResult();
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3797f9f/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java
index df920e0..e0f9ad2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFileTruncate.java
@@ -1008,8 +1008,8 @@ public class TestFileTruncate {
fsn.writeLock();
try {
Block oldBlock = file.getLastBlock();
- Block truncateBlock =
- fsn.prepareFileForTruncate(iip, client, clientMachine, 1, null);
+ Block truncateBlock = FSDirTruncateOp.prepareFileForTruncate(fsn, iip,
+ client, clientMachine, 1, null);
// In-place truncate uses old block id with new genStamp.
assertThat(truncateBlock.getBlockId(),
is(equalTo(oldBlock.getBlockId())));
@@ -1041,8 +1041,8 @@ public class TestFileTruncate {
fsn.writeLock();
try {
Block oldBlock = file.getLastBlock();
- Block truncateBlock =
- fsn.prepareFileForTruncate(iip, client, clientMachine, 1, null);
+ Block truncateBlock = FSDirTruncateOp.prepareFileForTruncate(fsn, iip,
+ client, clientMachine, 1, null);
// Copy-on-write truncate makes new block with new id and genStamp
assertThat(truncateBlock.getBlockId(),
is(not(equalTo(oldBlock.getBlockId()))));
[04/46] hadoop git commit: HADOOP-12009 Clarify
FileSystem.listStatus() sorting order & fix
FileSystemContractBaseTest:testListStatus. (J.Andreina via stevel)
Posted by ar...@apache.org.
HADOOP-12009 Clarify FileSystem.listStatus() sorting order & fix FileSystemContractBaseTest:testListStatus. (J.Andreina via stevel)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3dfa8161
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3dfa8161
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3dfa8161
Branch: refs/heads/HDFS-7240
Commit: 3dfa8161f9412bcb040f3c29c471344f25f24337
Parents: b543d1a
Author: Steve Loughran <st...@apache.org>
Authored: Sun Jun 28 19:13:48 2015 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Sun Jun 28 19:14:00 2015 +0100
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 4 ++++
.../main/java/org/apache/hadoop/fs/FileSystem.java | 17 ++++++++++++++++-
.../src/site/markdown/filesystem/filesystem.md | 4 ++++
.../hadoop/fs/FileSystemContractBaseTest.java | 11 ++++++++---
4 files changed, 32 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3dfa8161/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 18c73c0..132e4c0 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -661,6 +661,10 @@ Release 2.8.0 - UNRELEASED
HADOOP-11958. MetricsSystemImpl fails to show backtrace when an error
occurs (Jason Lowe via jeagles)
+
+ HADOOP-12009 Clarify FileSystem.listStatus() sorting order & fix
+ FileSystemContractBaseTest:testListStatus. (J.Andreina via stevel)
+
OPTIMIZATIONS
HADOOP-11785. Reduce the number of listStatus operation in distcp
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3dfa8161/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
index 01d4b27..3f9e3bd 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
@@ -1498,7 +1498,9 @@ public abstract class FileSystem extends Configured implements Closeable {
/**
* List the statuses of the files/directories in the given path if the path is
* a directory.
- *
+ * <p>
+ * Does not guarantee to return the List of files/directories status in a
+ * sorted order.
* @param f given path
* @return the statuses of the files/directories in the given patch
* @throws FileNotFoundException when the path does not exist;
@@ -1540,6 +1542,9 @@ public abstract class FileSystem extends Configured implements Closeable {
/**
* Filter files/directories in the given path using the user-supplied path
* filter.
+ * <p>
+ * Does not guarantee to return the List of files/directories status in a
+ * sorted order.
*
* @param f
* a path name
@@ -1560,6 +1565,9 @@ public abstract class FileSystem extends Configured implements Closeable {
/**
* Filter files/directories in the given list of paths using default
* path filter.
+ * <p>
+ * Does not guarantee to return the List of files/directories status in a
+ * sorted order.
*
* @param files
* a list of paths
@@ -1576,6 +1584,9 @@ public abstract class FileSystem extends Configured implements Closeable {
/**
* Filter files/directories in the given list of paths using user-supplied
* path filter.
+ * <p>
+ * Does not guarantee to return the List of files/directories status in a
+ * sorted order.
*
* @param files
* a list of paths
@@ -1736,6 +1747,8 @@ public abstract class FileSystem extends Configured implements Closeable {
* while consuming the entries. Each file system implementation should
* override this method and provide a more efficient implementation, if
* possible.
+ * Does not guarantee to return the iterator that traverses statuses
+ * of the files in a sorted order.
*
* @param p target path
* @return remote iterator
@@ -1763,6 +1776,8 @@ public abstract class FileSystem extends Configured implements Closeable {
/**
* List the statuses and block locations of the files in the given path.
+ * Does not guarantee to return the iterator that traverses statuses
+ * of the files in a sorted order.
*
* If the path is a directory,
* if recursive is false, returns files in the directory;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3dfa8161/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md
index 84e3755..f323374 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md
@@ -183,6 +183,10 @@ to the same path:
forall fs in listStatus(Path) :
fs == getFileStatus(fs.path)
+**Ordering of results**: there is no guarantee of ordering of the listed entries.
+While HDFS currently returns an alphanumerically sorted list, neither the Posix `readdir()`
+nor Java's `File.listFiles()` API calls define any ordering of returned values. Applications
+which require a uniform sort order on the results must perform the sorting themselves.
### Atomicity and Consistency
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3dfa8161/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemContractBaseTest.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemContractBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemContractBaseTest.java
index 2ca81e9..495af00 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemContractBaseTest.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemContractBaseTest.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.fs;
import java.io.FileNotFoundException;
import java.io.IOException;
+import java.util.ArrayList;
import junit.framework.TestCase;
@@ -224,9 +225,13 @@ public abstract class FileSystemContractBaseTest extends TestCase {
paths = fs.listStatus(path("/test/hadoop"));
assertEquals(3, paths.length);
- assertEquals(path("/test/hadoop/a"), paths[0].getPath());
- assertEquals(path("/test/hadoop/b"), paths[1].getPath());
- assertEquals(path("/test/hadoop/c"), paths[2].getPath());
+ ArrayList<String> list = new ArrayList<String>();
+ for (FileStatus fileState : paths) {
+ list.add(fileState.getPath().toString());
+ }
+ assertTrue(list.contains(path("/test/hadoop/a")));
+ assertTrue(list.contains(path("/test/hadoop/b")));
+ assertTrue(list.contains(path("/test/hadoop/c")));
paths = fs.listStatus(path("/test/hadoop/a"));
assertEquals(0, paths.length);
[21/46] hadoop git commit: HDFS-8687. Remove the duplicate usage
message from Dfsck.java. (Contributed by Brahma Reddy Battula)
Posted by ar...@apache.org.
HDFS-8687. Remove the duplicate usage message from Dfsck.java. (Contributed by Brahma Reddy Battula)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8e333720
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8e333720
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8e333720
Branch: refs/heads/HDFS-7240
Commit: 8e333720e13428a4d0d0f65692102f8f2e5da98d
Parents: 43a1288
Author: Arpit Agarwal <ar...@apache.org>
Authored: Mon Jun 29 14:49:00 2015 -0700
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Mon Jun 29 14:56:42 2015 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java | 1 -
2 files changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8e333720/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index eb006eb..108a6c0 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -977,6 +977,9 @@ Release 2.8.0 - UNRELEASED
HDFS-8628. Update missing command option for fetchdt
(J.Andreina via vinayakumarb)
+ HDFS-8687. Remove the duplicate usage message from Dfsck.java. (Brahma
+ Reddy Battula via Arpit Agarwal)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8e333720/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java
index 33033e7..6bb6603 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DFSck.java
@@ -377,7 +377,6 @@ public class DFSck extends Configured implements Tool {
int res = -1;
if ((args.length == 0) || ("-files".equals(args[0]))) {
printUsage(System.err);
- ToolRunner.printGenericCommandUsage(System.err);
} else if (DFSUtil.parseHelpArgument(args, USAGE, System.out, true)) {
res = 0;
} else {
[02/46] hadoop git commit: HADOOP-11807. add a lint mode to
releasedocmaker (ramtin via aw)
Posted by ar...@apache.org.
HADOOP-11807. add a lint mode to releasedocmaker (ramtin via aw)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/79ed0f95
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/79ed0f95
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/79ed0f95
Branch: refs/heads/HDFS-7240
Commit: 79ed0f959ffc490414ca56a73e026500c24e7078
Parents: fe6c1bd
Author: Allen Wittenauer <aw...@apache.org>
Authored: Sat Jun 27 08:59:50 2015 -0700
Committer: Allen Wittenauer <aw...@apache.org>
Committed: Sat Jun 27 08:59:50 2015 -0700
----------------------------------------------------------------------
dev-support/releasedocmaker.py | 76 +++++++++++++++++---
hadoop-common-project/hadoop-common/CHANGES.txt | 2 +
2 files changed, 68 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/79ed0f95/dev-support/releasedocmaker.py
----------------------------------------------------------------------
diff --git a/dev-support/releasedocmaker.py b/dev-support/releasedocmaker.py
index 2ccc1c0..8e68b3c 100755
--- a/dev-support/releasedocmaker.py
+++ b/dev-support/releasedocmaker.py
@@ -87,8 +87,15 @@ def notableclean(str):
str=str.rstrip()
return str
+# clean output dir
+def cleanOutputDir(dir):
+ files = os.listdir(dir)
+ for name in files:
+ os.remove(os.path.join(dir,name))
+ os.rmdir(dir)
+
def mstr(obj):
- if (obj == None):
+ if (obj is None):
return ""
return unicode(obj)
@@ -148,7 +155,7 @@ class Jira:
return mstr(self.fields['description'])
def getReleaseNote(self):
- if (self.notes == None):
+ if (self.notes is None):
field = self.parent.fieldIdMap['Release Note']
if (self.fields.has_key(field)):
self.notes=mstr(self.fields[field])
@@ -159,14 +166,14 @@ class Jira:
def getPriority(self):
ret = ""
pri = self.fields['priority']
- if(pri != None):
+ if(pri is not None):
ret = pri['name']
return mstr(ret)
def getAssignee(self):
ret = ""
mid = self.fields['assignee']
- if(mid != None):
+ if(mid is not None):
ret = mid['displayName']
return mstr(ret)
@@ -182,21 +189,21 @@ class Jira:
def getType(self):
ret = ""
mid = self.fields['issuetype']
- if(mid != None):
+ if(mid is not None):
ret = mid['name']
return mstr(ret)
def getReporter(self):
ret = ""
mid = self.fields['reporter']
- if(mid != None):
+ if(mid is not None):
ret = mid['displayName']
return mstr(ret)
def getProject(self):
ret = ""
mid = self.fields['project']
- if(mid != None):
+ if(mid is not None):
ret = mid['key']
return mstr(ret)
@@ -214,7 +221,7 @@ class Jira:
return False
def getIncompatibleChange(self):
- if (self.incompat == None):
+ if (self.incompat is None):
field = self.parent.fieldIdMap['Hadoop Flags']
self.reviewed=False
self.incompat=False
@@ -227,6 +234,24 @@ class Jira:
self.reviewed=True
return self.incompat
+ def checkMissingComponent(self):
+ if (len(self.fields['components'])>0):
+ return False
+ return True
+
+ def checkMissingAssignee(self):
+ if (self.fields['assignee'] is not None):
+ return False
+ return True
+
+ def checkVersionString(self):
+ field = self.parent.fieldIdMap['Fix Version/s']
+ for h in self.fields[field]:
+ found = re.match('^((\d+)(\.\d+)*).*$|^(\w+\-\d+)$', h['name'])
+ if not found:
+ return True
+ return False
+
def getReleaseDate(self,version):
for j in range(len(self.fields['fixVersions'])):
if self.fields['fixVersions'][j]==version:
@@ -339,9 +364,11 @@ def main():
help="build an index file")
parser.add_option("-u","--usetoday", dest="usetoday", action="store_true",
help="use current date for unreleased versions")
+ parser.add_option("-n","--lint", dest="lint", action="store_true",
+ help="use lint flag to exit on failures")
(options, args) = parser.parse_args()
- if (options.versions == None):
+ if (options.versions is None):
options.versions = []
if (len(args) > 2):
@@ -396,6 +423,9 @@ def main():
reloutputs.writeAll(relhead)
choutputs.writeAll(chhead)
+ errorCount=0
+ warningCount=0
+ lintMessage=""
incompatlist=[]
buglist=[]
improvementlist=[]
@@ -408,6 +438,14 @@ def main():
for jira in sorted(jlist):
if jira.getIncompatibleChange():
incompatlist.append(jira)
+ if (len(jira.getReleaseNote())==0):
+ warningCount+=1
+
+ if jira.checkVersionString():
+ warningCount+=1
+
+ if jira.checkMissingComponent() or jira.checkMissingAssignee():
+ errorCount+=1
elif jira.getType() == "Bug":
buglist.append(jira)
elif jira.getType() == "Improvement":
@@ -431,15 +469,33 @@ def main():
reloutputs.writeKeyRaw(jira.getProject(),"\n---\n\n")
reloutputs.writeKeyRaw(jira.getProject(), line)
line ='\n**WARNING: No release note provided for this incompatible change.**\n\n'
- print 'WARNING: incompatible change %s lacks release notes.' % (notableclean(jira.getId()))
+ lintMessage += "\nWARNING: incompatible change %s lacks release notes." % (notableclean(jira.getId()))
reloutputs.writeKeyRaw(jira.getProject(), line)
+ if jira.checkVersionString():
+ lintMessage += "\nWARNING: Version string problem for %s " % jira.getId()
+
+ if (jira.checkMissingComponent() or jira.checkMissingAssignee()):
+ errorMessage=[]
+ jira.checkMissingComponent() and errorMessage.append("component")
+ jira.checkMissingAssignee() and errorMessage.append("assignee")
+ lintMessage += "\nERROR: missing %s for %s " % (" and ".join(errorMessage) , jira.getId())
+
if (len(jira.getReleaseNote())>0):
reloutputs.writeKeyRaw(jira.getProject(),"\n---\n\n")
reloutputs.writeKeyRaw(jira.getProject(), line)
line ='\n%s\n\n' % (tableclean(jira.getReleaseNote()))
reloutputs.writeKeyRaw(jira.getProject(), line)
+ if (options.lint is True):
+ print lintMessage
+ print "======================================="
+ print "Error:%d, Warning:%d \n" % (errorCount, warningCount)
+
+ if (errorCount>0):
+ cleanOutputDir(version)
+ sys.exit(1)
+
reloutputs.writeAll("\n\n")
reloutputs.close()
http://git-wip-us.apache.org/repos/asf/hadoop/blob/79ed0f95/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 92e1bfa..18c73c0 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -230,6 +230,8 @@ Trunk (Unreleased)
HADOOP-11142. Remove hdfs dfs reference from file system shell
documentation (Kengo Seki via aw)
+ HADOOP-11807. add a lint mode to releasedocmaker (ramtin via aw)
+
BUG FIXES
HADOOP-11473. test-patch says "-1 overall" even when all checks are +1
[35/46] hadoop git commit: HADOOP-12124. Add HTrace support for
FsShell (cmccabe)
Posted by ar...@apache.org.
HADOOP-12124. Add HTrace support for FsShell (cmccabe)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ad608072
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ad608072
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ad608072
Branch: refs/heads/HDFS-7240
Commit: ad60807238c4f7779cb0685e7d39ca0c50e01b2f
Parents: 68e588c
Author: Colin Patrick Mccabe <cm...@cloudera.com>
Authored: Tue Jun 30 16:46:25 2015 -0700
Committer: Colin Patrick Mccabe <cm...@cloudera.com>
Committed: Tue Jun 30 16:46:25 2015 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 2 ++
.../src/main/java/org/apache/hadoop/fs/FsShell.java | 16 ++++++++++++++--
2 files changed, 16 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ad608072/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 4d69a229..19ba620 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -671,6 +671,8 @@ Release 2.8.0 - UNRELEASED
TestCryptoStreamsWithOpensslAesCtrCryptoCodec when OpenSSL is not
installed. (wang)
+ HADOOP-12124. Add HTrace support for FsShell (cmccabe)
+
OPTIMIZATIONS
HADOOP-11785. Reduce the number of listStatus operation in distcp
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ad608072/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsShell.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsShell.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsShell.java
index db73f6d..24b6339 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsShell.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FsShell.java
@@ -33,8 +33,13 @@ import org.apache.hadoop.fs.shell.Command;
import org.apache.hadoop.fs.shell.CommandFactory;
import org.apache.hadoop.fs.shell.FsCommand;
import org.apache.hadoop.tools.TableListing;
+import org.apache.hadoop.tracing.TraceUtils;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
+import org.apache.htrace.Sampler;
+import org.apache.htrace.SamplerBuilder;
+import org.apache.htrace.Trace;
+import org.apache.htrace.TraceScope;
/** Provide command line access to a FileSystem. */
@InterfaceAudience.Private
@@ -47,6 +52,7 @@ public class FsShell extends Configured implements Tool {
private FileSystem fs;
private Trash trash;
protected CommandFactory commandFactory;
+ private Sampler traceSampler;
private final String usagePrefix =
"Usage: hadoop fs [generic options]";
@@ -272,7 +278,8 @@ public class FsShell extends Configured implements Tool {
public int run(String argv[]) throws Exception {
// initialize FsShell
init();
-
+ traceSampler = new SamplerBuilder(TraceUtils.
+ wrapHadoopConf("dfs.shell.htrace.", getConf())).build();
int exitCode = -1;
if (argv.length < 1) {
printUsage(System.err);
@@ -284,7 +291,12 @@ public class FsShell extends Configured implements Tool {
if (instance == null) {
throw new UnknownCommandException();
}
- exitCode = instance.run(Arrays.copyOfRange(argv, 1, argv.length));
+ TraceScope scope = Trace.startSpan(instance.getCommandName(), traceSampler);
+ try {
+ exitCode = instance.run(Arrays.copyOfRange(argv, 1, argv.length));
+ } finally {
+ scope.close();
+ }
} catch (IllegalArgumentException e) {
displayError(cmd, e.getLocalizedMessage());
if (instance != null) {
[14/46] hadoop git commit: HADOOP-12112. Make hadoop-common-project
Native code -Wall-clean (alanburlison via cmccabe)
Posted by ar...@apache.org.
HADOOP-12112. Make hadoop-common-project Native code -Wall-clean (alanburlison via cmccabe)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/fad291ea
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/fad291ea
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/fad291ea
Branch: refs/heads/HDFS-7240
Commit: fad291ea6dbe49782e33a32cd6608088951e2c58
Parents: 5a27c3f
Author: Colin Patrick Mccabe <cm...@cloudera.com>
Authored: Mon Jun 29 12:34:47 2015 -0700
Committer: Colin Patrick Mccabe <cm...@cloudera.com>
Committed: Mon Jun 29 12:35:16 2015 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c | 1 +
.../native/src/org/apache/hadoop/net/unix/DomainSocket.c | 8 ++++----
.../src/main/native/src/org/apache/hadoop/util/NativeCrc32.c | 4 ++--
4 files changed, 10 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fad291ea/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index e8e85a0..9008ead 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -691,6 +691,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-12036. Consolidate all of the cmake extensions in one directory
(alanburlison via cmccabe)
+ HADOOP-12112. Make hadoop-common-project Native code -Wall-clean
+ (alanburlison via cmccabe)
+
BUG FIXES
HADOOP-11802: DomainSocketWatcher thread terminates sometimes after there
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fad291ea/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c
index 071d830..bc78ab2 100644
--- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c
+++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/io/nativeio/NativeIO.c
@@ -592,6 +592,7 @@ done:
#else
THROW(env, "java/io/IOException",
"The function Windows.createFileWithMode0() is not supported on this platform");
+ return NULL;
#endif
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fad291ea/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/net/unix/DomainSocket.c
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/net/unix/DomainSocket.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/net/unix/DomainSocket.c
index 55ab0c3..a3f27ee 100644
--- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/net/unix/DomainSocket.c
+++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/net/unix/DomainSocket.c
@@ -112,9 +112,9 @@ static jthrowable newSocketException(JNIEnv *env, int errnum,
* to the heap if necessary.
*/
struct flexibleBuffer {
- int8_t *curBuf;
- int8_t *allocBuf;
- int8_t stackBuf[8196];
+ jbyte *curBuf;
+ jbyte *allocBuf;
+ jbyte stackBuf[8196];
};
static jthrowable flexBufInit(JNIEnv *env, struct flexibleBuffer *flexBuf, jint length)
@@ -637,7 +637,7 @@ JNIEnv *env, jclass clazz, jint fd)
* @return NULL on success; or the unraised exception representing
* the problem.
*/
-static jthrowable write_fully(JNIEnv *env, int fd, int8_t *buf, int amt)
+static jthrowable write_fully(JNIEnv *env, int fd, jbyte *buf, int amt)
{
int err, res;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/fad291ea/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/NativeCrc32.c
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/NativeCrc32.c b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/NativeCrc32.c
index 74e09e6..3fceb02 100644
--- a/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/NativeCrc32.c
+++ b/hadoop-common-project/hadoop-common/src/main/native/src/org/apache/hadoop/util/NativeCrc32.c
@@ -168,7 +168,7 @@ JNIEXPORT void JNICALL Java_org_apache_hadoop_util_NativeCrc32_nativeComputeChun
// Setup complete. Actually verify checksums.
ret = bulk_crc(data, data_len, sums, crc_type,
bytes_per_checksum, verify ? &error_data : NULL);
- if (likely(verify && ret == CHECKSUMS_VALID || !verify && ret == 0)) {
+ if (likely((verify && ret == CHECKSUMS_VALID) || (!verify && ret == 0))) {
return;
} else if (unlikely(verify && ret == INVALID_CHECKSUM_DETECTED)) {
long pos = base_pos + (error_data.bad_data - data);
@@ -261,7 +261,7 @@ JNIEXPORT void JNICALL Java_org_apache_hadoop_util_NativeCrc32_nativeComputeChun
env, error_data.got_crc, error_data.expected_crc,
j_filename, pos);
return;
- } else if (unlikely(verify && ret != CHECKSUMS_VALID || !verify && ret != 0)) {
+ } else if (unlikely((verify && ret != CHECKSUMS_VALID) || (!verify && ret != 0))) {
THROW(env, "java/lang/AssertionError",
"Bad response code from native bulk_crc");
return;
[43/46] hadoop git commit: HDFS-8706. Fix typo in datanode startup
options in HDFSCommands.html. (Controibuted by Brahma Reddy Battula)
Posted by ar...@apache.org.
HDFS-8706. Fix typo in datanode startup options in HDFSCommands.html. (Controibuted by Brahma Reddy Battula)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ec0dc71c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ec0dc71c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ec0dc71c
Branch: refs/heads/HDFS-7240
Commit: ec0dc71cabb2b243a83fe4a7f6ba744907367666
Parents: 2ac87df
Author: Arpit Agarwal <ar...@apache.org>
Authored: Wed Jul 1 10:36:04 2015 -0700
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Wed Jul 1 10:36:04 2015 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md | 2 +-
2 files changed, 4 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ec0dc71c/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 65f286d..3736932 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -992,6 +992,9 @@ Release 2.8.0 - UNRELEASED
HDFS-8579. Update HDFS usage with missing options
(J.Andreina via vinayakumarb)
+ HDFS-8706. Fix typo in datanode startup options in HDFSCommands.html.
+ (Brahma Reddy Battula via Arpit Agarwal)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ec0dc71c/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
index 094dda1..1c2026c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSCommands.md
@@ -298,7 +298,7 @@ See the [HDFS Transparent Encryption Documentation](./TransparentEncryption.html
### `datanode`
-Usage: `hdfs datanode [-regular | -rollback | -rollingupgrace rollback]`
+Usage: `hdfs datanode [-regular | -rollback | -rollingupgrade rollback]`
| COMMAND\_OPTION | Description |
|:---- |:---- |
[24/46] hadoop git commit: HADOOP-12154. FileSystem#getUsed() returns
the file length only from root '/' (Contributed by J.Andreina)
Posted by ar...@apache.org.
HADOOP-12154. FileSystem#getUsed() returns the file length only from root '/' (Contributed by J.Andreina)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6d99017f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6d99017f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6d99017f
Branch: refs/heads/HDFS-7240
Commit: 6d99017f38f5a158b5cb65c74688b4c833e4e35f
Parents: 77588e1
Author: Vinayakumar B <vi...@apache.org>
Authored: Tue Jun 30 15:25:20 2015 +0530
Committer: Vinayakumar B <vi...@apache.org>
Committed: Tue Jun 30 15:25:20 2015 +0530
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
.../java/org/apache/hadoop/fs/FileSystem.java | 6 ++---
.../org/apache/hadoop/hdfs/TestDFSShell.java | 26 ++++++++++++++++++++
3 files changed, 32 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6d99017f/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 50192ae..2cf9509 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -899,6 +899,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-12089. StorageException complaining " no lease ID" when updating
FolderLastModifiedTime in WASB. (Duo Xu via cnauroth)
+ HADOOP-12154. FileSystem#getUsed() returns the file length only from root '/'
+ (J.Andreina via vinayakumarb)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6d99017f/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
index 1d7bc87..c98074a 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
@@ -2085,9 +2085,9 @@ public abstract class FileSystem extends Configured implements Closeable {
/** Return the total size of all files in the filesystem.*/
public long getUsed() throws IOException{
long used = 0;
- FileStatus[] files = listStatus(new Path("/"));
- for(FileStatus file:files){
- used += file.getLen();
+ RemoteIterator<LocatedFileStatus> files = listFiles(new Path("/"), true);
+ while (files.hasNext()) {
+ used += files.next().getLen();
}
return used;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6d99017f/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java
index 2df31c4..1386124 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java
@@ -913,6 +913,32 @@ public class TestDFSShell {
cluster.shutdown();
}
}
+
+ @Test(timeout = 30000)
+ public void testTotalSizeOfAllFiles() throws Exception {
+ Configuration conf = new HdfsConfiguration();
+ MiniDFSCluster cluster = null;
+ try {
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
+ FileSystem fs = cluster.getFileSystem();
+ // create file under root
+ FSDataOutputStream File1 = fs.create(new Path("/File1"));
+ File1.write("hi".getBytes());
+ File1.close();
+ // create file under sub-folder
+ FSDataOutputStream File2 = fs.create(new Path("/Folder1/File2"));
+ File2.write("hi".getBytes());
+ File2.close();
+ // getUsed() should return total length of all the files in Filesystem
+ assertEquals(4, fs.getUsed());
+ } finally {
+ if (cluster != null) {
+ cluster.shutdown();
+ cluster = null;
+ }
+ }
+ }
+
private static void runCount(String path, long dirs, long files, FsShell shell
) throws IOException {
ByteArrayOutputStream bytes = new ByteArrayOutputStream();
[11/46] hadoop git commit: HDFS-7390. Provide JMX metrics per storage
type. (Benoy Antony)
Posted by ar...@apache.org.
HDFS-7390. Provide JMX metrics per storage type. (Benoy Antony)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d3fed8e6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d3fed8e6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d3fed8e6
Branch: refs/heads/HDFS-7240
Commit: d3fed8e653ed9e18d3a29a11c4b24a628ac770bb
Parents: fde20ff
Author: Benoy Antony <be...@apache.org>
Authored: Mon Jun 29 11:00:22 2015 -0700
Committer: Benoy Antony <be...@apache.org>
Committed: Mon Jun 29 11:00:22 2015 -0700
----------------------------------------------------------------------
.../server/blockmanagement/BlockManager.java | 15 +-
.../blockmanagement/BlockStatsMXBean.java | 36 +++++
.../blockmanagement/DatanodeStatistics.java | 6 +
.../blockmanagement/HeartbeatManager.java | 96 +++++++++++-
.../blockmanagement/StorageTypeStats.java | 115 +++++++++++++++
.../blockmanagement/TestBlockStatsMXBean.java | 146 +++++++++++++++++++
6 files changed, 412 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3fed8e6/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
index 5bd4980..0b60a97 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockManager.java
@@ -39,6 +39,8 @@ import java.util.TreeSet;
import java.util.concurrent.ThreadLocalRandom;
import java.util.concurrent.atomic.AtomicLong;
+import javax.management.ObjectName;
+
import org.apache.hadoop.HadoopIllegalArgumentException;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
@@ -85,6 +87,7 @@ import org.apache.hadoop.hdfs.server.protocol.KeyUpdateCommand;
import org.apache.hadoop.hdfs.server.protocol.ReceivedDeletedBlockInfo;
import org.apache.hadoop.hdfs.server.protocol.StorageReceivedDeletedBlocks;
import org.apache.hadoop.hdfs.util.LightWeightLinkedSet;
+import org.apache.hadoop.metrics2.util.MBeans;
import org.apache.hadoop.net.Node;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.util.Daemon;
@@ -94,6 +97,7 @@ import org.apache.hadoop.util.Time;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.collect.Sets;
+
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -101,7 +105,7 @@ import org.slf4j.LoggerFactory;
* Keeps information related to the blocks stored in the Hadoop cluster.
*/
@InterfaceAudience.Private
-public class BlockManager {
+public class BlockManager implements BlockStatsMXBean {
public static final Logger LOG = LoggerFactory.getLogger(BlockManager.class);
public static final Logger blockLog = NameNode.blockStateChangeLog;
@@ -129,6 +133,7 @@ public class BlockManager {
private final AtomicLong postponedMisreplicatedBlocksCount = new AtomicLong(0L);
private final long startupDelayBlockDeletionInMs;
private final BlockReportLeaseManager blockReportLeaseManager;
+ private ObjectName mxBeanName;
/** Used by metrics */
public long getPendingReplicationBlocksCount() {
@@ -468,6 +473,7 @@ public class BlockManager {
pendingReplications.start();
datanodeManager.activate(conf);
this.replicationThread.start();
+ mxBeanName = MBeans.register("NameNode", "BlockStats", this);
}
public void close() {
@@ -3944,6 +3950,8 @@ public class BlockManager {
public void shutdown() {
stopReplicationInitializer();
blocksMap.close();
+ MBeans.unregister(mxBeanName);
+ mxBeanName = null;
}
public void clear() {
@@ -3954,4 +3962,9 @@ public class BlockManager {
public BlockReportLeaseManager getBlockReportLeaseManager() {
return blockReportLeaseManager;
}
+
+ @Override // BlockStatsMXBean
+ public Map<StorageType, StorageTypeStats> getStorageTypeStats() {
+ return datanodeManager.getDatanodeStatistics().getStorageTypeStats();
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3fed8e6/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockStatsMXBean.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockStatsMXBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockStatsMXBean.java
new file mode 100644
index 0000000..f22c537
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockStatsMXBean.java
@@ -0,0 +1,36 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.blockmanagement;
+
+import java.util.Map;
+
+import org.apache.hadoop.fs.StorageType;
+
+/**
+ * This is an interface used to retrieve statistic information related to
+ * block management.
+ */
+public interface BlockStatsMXBean {
+
+ /**
+ * The statistics of storage types.
+ *
+ * @return get storage statistics per storage type
+ */
+ Map<StorageType, StorageTypeStats> getStorageTypeStats();
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3fed8e6/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStatistics.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStatistics.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStatistics.java
index c9bc3e5..33eca2e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStatistics.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStatistics.java
@@ -17,6 +17,9 @@
*/
package org.apache.hadoop.hdfs.server.blockmanagement;
+import java.util.Map;
+
+import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.hdfs.protocol.ClientProtocol;
/** Datanode statistics */
@@ -71,4 +74,7 @@ public interface DatanodeStatistics {
/** @return the expired heartbeats */
public int getExpiredHeartbeats();
+
+ /** @return Storage Tier statistics*/
+ Map<StorageType, StorageTypeStats> getStorageTypeStats();
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3fed8e6/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HeartbeatManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HeartbeatManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HeartbeatManager.java
index 9017fe1..cc9365d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HeartbeatManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/HeartbeatManager.java
@@ -18,9 +18,15 @@
package org.apache.hadoop.hdfs.server.blockmanagement;
import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.IdentityHashMap;
import java.util.List;
+import java.util.Map;
+import java.util.Set;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.StorageType;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSUtilClient;
import org.apache.hadoop.hdfs.protocol.DatanodeID;
@@ -189,6 +195,11 @@ class HeartbeatManager implements DatanodeStatistics {
return stats.expiredHeartbeats;
}
+ @Override
+ public Map<StorageType, StorageTypeStats> getStorageTypeStats() {
+ return stats.statsMap.get();
+ }
+
synchronized void register(final DatanodeDescriptor d) {
if (!d.isAlive) {
addDatanode(d);
@@ -393,6 +404,9 @@ class HeartbeatManager implements DatanodeStatistics {
* For decommissioning/decommissioned nodes, only used capacity is counted.
*/
private static class Stats {
+
+ private final StorageTypeStatsMap statsMap = new StorageTypeStatsMap();
+
private long capacityTotal = 0L;
private long capacityUsed = 0L;
private long capacityRemaining = 0L;
@@ -420,6 +434,14 @@ class HeartbeatManager implements DatanodeStatistics {
}
cacheCapacity += node.getCacheCapacity();
cacheUsed += node.getCacheUsed();
+ Set<StorageType> storageTypes = new HashSet<>();
+ for (DatanodeStorageInfo storageInfo : node.getStorageInfos()) {
+ statsMap.addStorage(storageInfo, node);
+ storageTypes.add(storageInfo.getStorageType());
+ }
+ for (StorageType storageType : storageTypes) {
+ statsMap.addNode(storageType, node);
+ }
}
private void subtract(final DatanodeDescriptor node) {
@@ -436,6 +458,14 @@ class HeartbeatManager implements DatanodeStatistics {
}
cacheCapacity -= node.getCacheCapacity();
cacheUsed -= node.getCacheUsed();
+ Set<StorageType> storageTypes = new HashSet<>();
+ for (DatanodeStorageInfo storageInfo : node.getStorageInfos()) {
+ statsMap.subtractStorage(storageInfo, node);
+ storageTypes.add(storageInfo.getStorageType());
+ }
+ for (StorageType storageType : storageTypes) {
+ statsMap.subtractNode(storageType, node);
+ }
}
/** Increment expired heartbeat counter. */
@@ -443,5 +473,69 @@ class HeartbeatManager implements DatanodeStatistics {
expiredHeartbeats++;
}
}
-}
+ /** StorageType specific statistics.
+ * For decommissioning/decommissioned nodes, only used capacity is counted.
+ */
+
+ static final class StorageTypeStatsMap {
+
+ private Map<StorageType, StorageTypeStats> storageTypeStatsMap =
+ new IdentityHashMap<>();
+
+ private StorageTypeStatsMap() {}
+
+ private StorageTypeStatsMap(StorageTypeStatsMap other) {
+ storageTypeStatsMap =
+ new IdentityHashMap<>(other.storageTypeStatsMap);
+ for (Map.Entry<StorageType, StorageTypeStats> entry :
+ storageTypeStatsMap.entrySet()) {
+ entry.setValue(new StorageTypeStats(entry.getValue()));
+ }
+ }
+
+ private Map<StorageType, StorageTypeStats> get() {
+ return Collections.unmodifiableMap(storageTypeStatsMap);
+ }
+
+ private void addNode(StorageType storageType,
+ final DatanodeDescriptor node) {
+ StorageTypeStats storageTypeStats =
+ storageTypeStatsMap.get(storageType);
+ if (storageTypeStats == null) {
+ storageTypeStats = new StorageTypeStats();
+ storageTypeStatsMap.put(storageType, storageTypeStats);
+ }
+ storageTypeStats.addNode(node);
+ }
+
+ private void addStorage(final DatanodeStorageInfo info,
+ final DatanodeDescriptor node) {
+ StorageTypeStats storageTypeStats =
+ storageTypeStatsMap.get(info.getStorageType());
+ if (storageTypeStats == null) {
+ storageTypeStats = new StorageTypeStats();
+ storageTypeStatsMap.put(info.getStorageType(), storageTypeStats);
+ }
+ storageTypeStats.addStorage(info, node);
+ }
+
+ private void subtractStorage(final DatanodeStorageInfo info,
+ final DatanodeDescriptor node) {
+ StorageTypeStats storageTypeStats =
+ storageTypeStatsMap.get(info.getStorageType());
+ if (storageTypeStats != null) {
+ storageTypeStats.subtractStorage(info, node);
+ }
+ }
+
+ private void subtractNode(StorageType storageType,
+ final DatanodeDescriptor node) {
+ StorageTypeStats storageTypeStats =
+ storageTypeStatsMap.get(storageType);
+ if (storageTypeStats != null) {
+ storageTypeStats.subtractNode(node);
+ }
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3fed8e6/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/StorageTypeStats.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/StorageTypeStats.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/StorageTypeStats.java
new file mode 100644
index 0000000..45dcc8d
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/StorageTypeStats.java
@@ -0,0 +1,115 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs.server.blockmanagement;
+
+import java.beans.ConstructorProperties;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * Statistics per StorageType.
+ *
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class StorageTypeStats {
+ private long capacityTotal = 0L;
+ private long capacityUsed = 0L;
+ private long capacityRemaining = 0L;
+ private long blockPoolUsed = 0L;
+ private int nodesInService = 0;
+
+ @ConstructorProperties({"capacityTotal",
+ "capacityUsed", "capacityRemaining", "blockPoolUsed", "nodesInService"})
+ public StorageTypeStats(long capacityTotal, long capacityUsed,
+ long capacityRemaining, long blockPoolUsed, int nodesInService) {
+ this.capacityTotal = capacityTotal;
+ this.capacityUsed = capacityUsed;
+ this.capacityRemaining = capacityRemaining;
+ this.blockPoolUsed = blockPoolUsed;
+ this.nodesInService = nodesInService;
+ }
+
+ public long getCapacityTotal() {
+ return capacityTotal;
+ }
+
+ public long getCapacityUsed() {
+ return capacityUsed;
+ }
+
+ public long getCapacityRemaining() {
+ return capacityRemaining;
+ }
+
+ public long getBlockPoolUsed() {
+ return blockPoolUsed;
+ }
+
+ public int getNodesInService() {
+ return nodesInService;
+ }
+
+ StorageTypeStats() {}
+
+ StorageTypeStats(StorageTypeStats other) {
+ capacityTotal = other.capacityTotal;
+ capacityUsed = other.capacityUsed;
+ capacityRemaining = other.capacityRemaining;
+ blockPoolUsed = other.blockPoolUsed;
+ nodesInService = other.nodesInService;
+ }
+
+ void addStorage(final DatanodeStorageInfo info,
+ final DatanodeDescriptor node) {
+ capacityUsed += info.getDfsUsed();
+ blockPoolUsed += info.getBlockPoolUsed();
+ if (!(node.isDecommissionInProgress() || node.isDecommissioned())) {
+ capacityTotal += info.getCapacity();
+ capacityRemaining += info.getRemaining();
+ } else {
+ capacityTotal += info.getDfsUsed();
+ }
+ }
+
+ void addNode(final DatanodeDescriptor node) {
+ if (!(node.isDecommissionInProgress() || node.isDecommissioned())) {
+ nodesInService++;
+ }
+ }
+
+ void subtractStorage(final DatanodeStorageInfo info,
+ final DatanodeDescriptor node) {
+ capacityUsed -= info.getDfsUsed();
+ blockPoolUsed -= info.getBlockPoolUsed();
+ if (!(node.isDecommissionInProgress() || node.isDecommissioned())) {
+ capacityTotal -= info.getCapacity();
+ capacityRemaining -= info.getRemaining();
+ } else {
+ capacityTotal -= info.getDfsUsed();
+ }
+ }
+
+ void subtractNode(final DatanodeDescriptor node) {
+ if (!(node.isDecommissionInProgress() || node.isDecommissioned())) {
+ nodesInService--;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d3fed8e6/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockStatsMXBean.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockStatsMXBean.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockStatsMXBean.java
new file mode 100644
index 0000000..43d983d
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestBlockStatsMXBean.java
@@ -0,0 +1,146 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.blockmanagement;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.net.URL;
+import java.util.HashSet;
+import java.util.Map;
+import java.util.Set;
+
+import org.apache.hadoop.fs.StorageType;
+import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.mortbay.util.ajax.JSON;
+
+/**
+ * Class for testing {@link BlockStatsMXBean} implementation
+ */
+public class TestBlockStatsMXBean {
+
+ private MiniDFSCluster cluster;
+
+ @Before
+ public void setup() throws IOException {
+ HdfsConfiguration conf = new HdfsConfiguration();
+ cluster = null;
+ StorageType[][] types = new StorageType[6][];
+ for (int i=0; i<3; i++) {
+ types[i] = new StorageType[] {StorageType.RAM_DISK, StorageType.DISK};
+ }
+ for (int i=3; i< 5; i++) {
+ types[i] = new StorageType[] {StorageType.RAM_DISK, StorageType.ARCHIVE};
+ }
+ types[5] = new StorageType[] {StorageType.RAM_DISK, StorageType.ARCHIVE,
+ StorageType.ARCHIVE};
+
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(6).
+ storageTypes(types).storagesPerDatanode(3).build();
+ cluster.waitActive();
+ }
+
+ @After
+ public void tearDown() {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+
+ @Test
+ public void testStorageTypeStats() throws Exception {
+ Map<StorageType, StorageTypeStats> storageTypeStatsMap =
+ cluster.getNamesystem().getBlockManager().getStorageTypeStats();
+ assertTrue(storageTypeStatsMap.containsKey(StorageType.RAM_DISK));
+ assertTrue(storageTypeStatsMap.containsKey(StorageType.DISK));
+ assertTrue(storageTypeStatsMap.containsKey(StorageType.ARCHIVE));
+
+ StorageTypeStats storageTypeStats =
+ storageTypeStatsMap.get(StorageType.RAM_DISK);
+ assertEquals(6, storageTypeStats.getNodesInService());
+
+ storageTypeStats = storageTypeStatsMap.get(StorageType.DISK);
+ assertEquals(3, storageTypeStats.getNodesInService());
+
+ storageTypeStats = storageTypeStatsMap.get(StorageType.ARCHIVE);
+ assertEquals(3, storageTypeStats.getNodesInService());
+ }
+
+ protected static String readOutput(URL url) throws IOException {
+ StringBuilder out = new StringBuilder();
+ InputStream in = url.openConnection().getInputStream();
+ byte[] buffer = new byte[64 * 1024];
+ int len = in.read(buffer);
+ while (len > 0) {
+ out.append(new String(buffer, 0, len));
+ len = in.read(buffer);
+ }
+ return out.toString();
+ }
+
+ @Test
+ @SuppressWarnings("unchecked")
+ public void testStorageTypeStatsJMX() throws Exception {
+ URL baseUrl = new URL (cluster.getHttpUri(0));
+ String result = readOutput(new URL(baseUrl, "/jmx"));
+ System.out.println(result);
+
+ Map<String, Object> stat = (Map<String, Object>) JSON.parse(result);
+ Object[] beans =(Object[]) stat.get("beans");
+ Map<String, Object> blockStats = null;
+ for (Object bean : beans) {
+ Map<String, Object> map = (Map<String, Object>) bean;
+ if (map.get("name").equals("Hadoop:service=NameNode,name=BlockStats")) {
+ blockStats = map;
+ }
+ }
+ assertNotNull(blockStats);
+ Object[] storageTypeStatsList =
+ (Object[])blockStats.get("StorageTypeStats");
+ assertNotNull(storageTypeStatsList);
+ assertEquals (3, storageTypeStatsList.length);
+
+ Set<String> typesPresent = new HashSet<> ();
+ for (Object obj : storageTypeStatsList) {
+ Map<String, Object> entry = (Map<String, Object>)obj;
+ String storageType = (String)entry.get("key");
+ Map<String,Object> storageTypeStats = (Map<String,Object>)entry.get("value");
+ typesPresent.add(storageType);
+ if (storageType.equals("ARCHIVE") || storageType.equals("DISK") ) {
+ assertEquals(3l, storageTypeStats.get("nodesInService"));
+ } else if (storageType.equals("RAM_DISK")) {
+ assertEquals(6l, storageTypeStats.get("nodesInService"));
+ }
+ else {
+ fail();
+ }
+ }
+
+ assertTrue(typesPresent.contains("ARCHIVE"));
+ assertTrue(typesPresent.contains("DISK"));
+ assertTrue(typesPresent.contains("RAM_DISK"));
+ }
+}
[25/46] hadoop git commit: HDFS-8627. NPE thrown if unable to fetch
token from Namenode (Contributed by J.Andreina)
Posted by ar...@apache.org.
HDFS-8627. NPE thrown if unable to fetch token from Namenode (Contributed by J.Andreina)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7ba5bbac
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7ba5bbac
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7ba5bbac
Branch: refs/heads/HDFS-7240
Commit: 7ba5bbac02b688f68a8d23671a1e869234b4cebe
Parents: 6d99017
Author: Vinayakumar B <vi...@apache.org>
Authored: Tue Jun 30 15:42:59 2015 +0530
Committer: Vinayakumar B <vi...@apache.org>
Committed: Tue Jun 30 15:42:59 2015 +0530
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
.../hdfs/tools/DelegationTokenFetcher.java | 19 +++++++++++--------
.../hdfs/tools/TestDelegationTokenFetcher.java | 15 +++++++++++++++
3 files changed, 29 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7ba5bbac/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index ea67ea1..b065f98 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -338,6 +338,9 @@ Trunk (Unreleased)
HDFS-8412. Fix the test failures in HTTPFS. (umamahesh)
+ HDFS-8627. NPE thrown if unable to fetch token from Namenode
+ (J.Andreina via vinayakumarb)
+
Release 2.8.0 - UNRELEASED
NEW FEATURES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7ba5bbac/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java
index 6376243..803402d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/DelegationTokenFetcher.java
@@ -176,14 +176,17 @@ public class DelegationTokenFetcher {
final String renewer, final Path tokenFile)
throws IOException {
Token<?> token = fs.getDelegationToken(renewer);
-
- Credentials cred = new Credentials();
- cred.addToken(token.getKind(), token);
- cred.writeTokenStorageFile(tokenFile, conf);
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("Fetched token " + fs.getUri() + " for " + token.getService()
- + " into " + tokenFile);
+ if (null != token) {
+ Credentials cred = new Credentials();
+ cred.addToken(token.getKind(), token);
+ cred.writeTokenStorageFile(tokenFile, conf);
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Fetched token " + fs.getUri() + " for " +
+ token.getService() + " into " + tokenFile);
+ }
+ } else {
+ System.err.println("ERROR: Failed to fetch token from " + fs.getUri());
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7ba5bbac/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDelegationTokenFetcher.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDelegationTokenFetcher.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDelegationTokenFetcher.java
index b4cf1c0..ab3933b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDelegationTokenFetcher.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/TestDelegationTokenFetcher.java
@@ -90,4 +90,19 @@ public class TestDelegationTokenFetcher {
DelegationTokenFetcher.cancelTokens(conf, p);
Assert.assertEquals(testToken, FakeRenewer.getLastCanceled());
}
+
+ /**
+ * If token returned is null, saveDelegationToken should not
+ * throw nullPointerException
+ */
+ @Test
+ public void testReturnedTokenIsNull() throws Exception {
+ WebHdfsFileSystem fs = mock(WebHdfsFileSystem.class);
+ doReturn(null).when(fs).getDelegationToken(anyString());
+ Path p = new Path(f.getRoot().getAbsolutePath(), tokenFile);
+ DelegationTokenFetcher.saveDelegationToken(conf, fs, null, p);
+ // When Token returned is null, TokenFile should not exist
+ Assert.assertFalse(p.getFileSystem(conf).exists(p));
+
+ }
}
[08/46] hadoop git commit: HDFS-8586. Dead Datanode is allocated for
write when client is from deadnode (Contributed by Brahma Reddy Battula)
Posted by ar...@apache.org.
HDFS-8586. Dead Datanode is allocated for write when client is from deadnode (Contributed by Brahma Reddy Battula)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/88ceb382
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/88ceb382
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/88ceb382
Branch: refs/heads/HDFS-7240
Commit: 88ceb382ef45bd09cf004cf44aedbabaf3976759
Parents: a95d39f
Author: Vinayakumar B <vi...@apache.org>
Authored: Mon Jun 29 15:25:03 2015 +0530
Committer: Vinayakumar B <vi...@apache.org>
Committed: Mon Jun 29 15:25:03 2015 +0530
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++
.../BlockPlacementPolicyDefault.java | 3 +-
.../hdfs/server/namenode/TestDeadDatanode.java | 42 ++++++++++++++++++++
3 files changed, 47 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/88ceb382/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index b89e10c..3800184 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -968,6 +968,9 @@ Release 2.8.0 - UNRELEASED
HDFS-8546. Prune cached replicas from DatanodeDescriptor state on replica
invalidation. (wang)
+ HDFS-8586. Dead Datanode is allocated for write when client is from deadnode
+ (Brahma Reddy Battula via vinayakumarb)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/88ceb382/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
index 21ad01d..9023e0a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/BlockPlacementPolicyDefault.java
@@ -454,7 +454,8 @@ public class BlockPlacementPolicyDefault extends BlockPlacementPolicy {
return chooseRandom(NodeBase.ROOT, excludedNodes, blocksize,
maxNodesPerRack, results, avoidStaleNodes, storageTypes);
}
- if (preferLocalNode && localMachine instanceof DatanodeDescriptor) {
+ if (preferLocalNode && localMachine instanceof DatanodeDescriptor
+ && clusterMap.contains(localMachine)) {
DatanodeDescriptor localDatanode = (DatanodeDescriptor) localMachine;
// otherwise try local machine first
if (excludedNodes.add(localMachine)) { // was not in the excluded list
http://git-wip-us.apache.org/repos/asf/hadoop/blob/88ceb382/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeadDatanode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeadDatanode.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeadDatanode.java
index ff70c3f..c5262d4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeadDatanode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestDeadDatanode.java
@@ -18,9 +18,11 @@
package org.apache.hadoop.hdfs.server.namenode;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
import static org.junit.Assert.fail;
import java.io.IOException;
+import java.util.HashSet;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -31,6 +33,9 @@ import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.protocol.Block;
import org.apache.hadoop.hdfs.protocol.BlockListAsLongs;
+import org.apache.hadoop.hdfs.server.blockmanagement.BlockManager;
+import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeManager;
+import org.apache.hadoop.hdfs.server.blockmanagement.DatanodeStorageInfo;
import org.apache.hadoop.hdfs.server.datanode.DataNode;
import org.apache.hadoop.hdfs.server.datanode.DataNodeTestUtils;
import org.apache.hadoop.hdfs.server.protocol.BlockReportContext;
@@ -43,6 +48,7 @@ import org.apache.hadoop.hdfs.server.protocol.RegisterCommand;
import org.apache.hadoop.hdfs.server.protocol.StorageBlockReport;
import org.apache.hadoop.hdfs.server.protocol.StorageReceivedDeletedBlocks;
import org.apache.hadoop.hdfs.server.protocol.StorageReport;
+import org.apache.hadoop.net.Node;
import org.junit.After;
import org.junit.Test;
@@ -126,4 +132,40 @@ public class TestDeadDatanode {
assertEquals(cmd[0].getAction(), RegisterCommand.REGISTER
.getAction());
}
+
+ @Test
+ public void testDeadNodeAsBlockTarget() throws Exception {
+ Configuration conf = new HdfsConfiguration();
+ conf.setInt(DFSConfigKeys.DFS_NAMENODE_HEARTBEAT_RECHECK_INTERVAL_KEY, 500);
+ conf.setLong(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1L);
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(3).build();
+ cluster.waitActive();
+
+ String poolId = cluster.getNamesystem().getBlockPoolId();
+ // wait for datanode to be marked live
+ DataNode dn = cluster.getDataNodes().get(0);
+ DatanodeRegistration reg = DataNodeTestUtils.getDNRegistrationForBP(cluster
+ .getDataNodes().get(0), poolId);
+ // Get the updated datanode descriptor
+ BlockManager bm = cluster.getNamesystem().getBlockManager();
+ DatanodeManager dm = bm.getDatanodeManager();
+ Node clientNode = dm.getDatanode(reg);
+
+ DFSTestUtil.waitForDatanodeState(cluster, reg.getDatanodeUuid(), true,
+ 20000);
+
+ // Shutdown and wait for datanode to be marked dead
+ dn.shutdown();
+ DFSTestUtil.waitForDatanodeState(cluster, reg.getDatanodeUuid(), false,
+ 20000);
+ // Get the updated datanode descriptor available in DNM
+ // choose the targets, but local node should not get selected as this is not
+ // part of the cluster anymore
+ DatanodeStorageInfo[] results = bm.chooseTarget4NewBlock("/hello", 3,
+ clientNode, new HashSet<Node>(), 256 * 1024 * 1024L, null, (byte) 7);
+ for (DatanodeStorageInfo datanodeStorageInfo : results) {
+ assertFalse("Dead node should not be choosen", datanodeStorageInfo
+ .getDatanodeDescriptor().equals(clientNode));
+ }
+ }
}
[07/46] hadoop git commit: YARN-3860. rmadmin -transitionToActive
should check the state of non-target node. (Contributed by Masatake Iwasaki)
Posted by ar...@apache.org.
YARN-3860. rmadmin -transitionToActive should check the state of non-target node. (Contributed by Masatake Iwasaki)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/a95d39f9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/a95d39f9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/a95d39f9
Branch: refs/heads/HDFS-7240
Commit: a95d39f9d08b3b215a1b33e77e9ab8a2ee59b3a9
Parents: aad6a7d
Author: Junping Du <ju...@apache.org>
Authored: Sun Jun 28 22:26:47 2015 -0700
Committer: Junping Du <ju...@apache.org>
Committed: Sun Jun 28 22:26:47 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +++
.../org/apache/hadoop/yarn/client/cli/RMAdminCLI.java | 10 +++++++++-
.../org/apache/hadoop/yarn/client/cli/TestRMAdminCLI.java | 4 ++++
3 files changed, 16 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a95d39f9/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 714bfdc..3ebf799 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -565,6 +565,9 @@ Release 2.8.0 - UNRELEASED
YARN-3859. LeafQueue doesn't print user properly for application add.
(Varun Saxena via devaraj)
+ YARN-3860. rmadmin -transitionToActive should check the state of non-target node.
+ (Masatake Iwasaki via junping_du)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a95d39f9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java
index 932b62d..36d29d6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/main/java/org/apache/hadoop/yarn/client/cli/RMAdminCLI.java
@@ -797,7 +797,15 @@ public class RMAdminCLI extends HAAdmin {
"Could not connect to RM HA Admin for node " + rmId);
}
}
-
+
+ /**
+ * returns the list of all resourcemanager ids for the given configuration.
+ */
+ @Override
+ protected Collection<String> getTargetIds(String targetNodeToActivate) {
+ return HAUtil.getRMHAIds(getConf());
+ }
+
@Override
protected String getUsageString() {
return "Usage: rmadmin";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/a95d39f9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRMAdminCLI.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRMAdminCLI.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRMAdminCLI.java
index a95412c..6dc326a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRMAdminCLI.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/cli/TestRMAdminCLI.java
@@ -27,6 +27,7 @@ import static org.mockito.Matchers.argThat;
import static org.mockito.Matchers.eq;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
@@ -116,6 +117,7 @@ public class TestRMAdminCLI {
YarnConfiguration conf = new YarnConfiguration();
conf.setBoolean(YarnConfiguration.RM_HA_ENABLED, true);
+ conf.set(YarnConfiguration.RM_HA_IDS, "rm1,rm2");
rmAdminCLIWithHAEnabled = new RMAdminCLI(conf) {
@Override
@@ -259,6 +261,8 @@ public class TestRMAdminCLI {
assertEquals(0, rmAdminCLIWithHAEnabled.run(args));
verify(haadmin).transitionToActive(
any(HAServiceProtocol.StateChangeRequestInfo.class));
+ // HAAdmin#isOtherTargetNodeActive should check state of non-target node.
+ verify(haadmin, times(1)).getServiceStatus();
}
@Test(timeout = 500)
[31/46] hadoop git commit: HDFS-8635. Migrate HDFS native build to
new CMake framework (Alan Burlison via Colin P. McCabe)
Posted by ar...@apache.org.
HDFS-8635. Migrate HDFS native build to new CMake framework (Alan Burlison via Colin P. McCabe)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/29df73dc
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/29df73dc
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/29df73dc
Branch: refs/heads/HDFS-7240
Commit: 29df73dc0572cf8e450568f1d0301e8d55d64b48
Parents: 9ee7b6e
Author: Colin Patrick Mccabe <cm...@cloudera.com>
Authored: Tue Jun 30 15:54:38 2015 -0700
Committer: Colin Patrick Mccabe <cm...@cloudera.com>
Committed: Tue Jun 30 15:54:38 2015 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
.../hadoop-hdfs/src/CMakeLists.txt | 129 ++++++-------------
.../src/contrib/libwebhdfs/CMakeLists.txt | 8 +-
.../libwebhdfs/resources/FindJansson.cmake | 9 +-
.../src/main/native/fuse-dfs/CMakeLists.txt | 91 +++++++------
5 files changed, 100 insertions(+), 140 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/29df73dc/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index b065f98..65f286d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -691,6 +691,9 @@ Release 2.8.0 - UNRELEASED
HDFS-8493. Consolidate truncate() related implementation in a single class.
(Rakesh R via wheat9)
+ HDFS-8635. Migrate HDFS native build to new CMake framework (Alan Burlison
+ via Colin P. McCabe)
+
OPTIMIZATIONS
HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than
http://git-wip-us.apache.org/repos/asf/hadoop/blob/29df73dc/hadoop-hdfs-project/hadoop-hdfs/src/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs/src/CMakeLists.txt
index 563727b..b4a3b40 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/CMakeLists.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/CMakeLists.txt
@@ -18,106 +18,59 @@
cmake_minimum_required(VERSION 2.6 FATAL_ERROR)
-# Default to release builds
-set(CMAKE_BUILD_TYPE, Release)
+list(APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/../../../hadoop-common-project/hadoop-common)
+include(HadoopCommon)
-include(../../../hadoop-common-project/hadoop-common/src/JNIFlags.cmake NO_POLICY_SCOPE)
-
-# Compile a library with both shared and static variants
-function(add_dual_library LIBNAME)
- add_library(${LIBNAME} SHARED ${ARGN})
- add_library(${LIBNAME}_static STATIC ${ARGN})
- # Linux builds traditionally ship a libhdfs.a (static linking) and libhdfs.so
- # (dynamic linking). On Windows, we cannot use the same base name for both
- # static and dynamic, because Windows does not use distinct file extensions
- # for a statically linked library vs. a DLL import library. Both use the
- # .lib extension. On Windows, we'll build the static library as
- # hdfs_static.lib.
- if (NOT WIN32)
- set_target_properties(${LIBNAME}_static PROPERTIES OUTPUT_NAME ${LIBNAME})
- endif (NOT WIN32)
-endfunction(add_dual_library)
-
-# Link both a static and a dynamic target against some libraries
-function(target_link_dual_libraries LIBNAME)
- target_link_libraries(${LIBNAME} ${ARGN})
- target_link_libraries(${LIBNAME}_static ${ARGN})
-endfunction(target_link_dual_libraries)
-
-function(output_directory TGT DIR)
- SET_TARGET_PROPERTIES(${TGT} PROPERTIES
- RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${DIR}")
- SET_TARGET_PROPERTIES(${TGT} PROPERTIES
- ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${DIR}")
- SET_TARGET_PROPERTIES(${TGT} PROPERTIES
- LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${DIR}")
-endfunction(output_directory TGT DIR)
-
-function(dual_output_directory TGT DIR)
- output_directory(${TGT} "${DIR}")
- output_directory(${TGT}_static "${DIR}")
-endfunction(dual_output_directory TGT DIR)
+#
+# Main configuration
+#
-# Flatten a list into a string.
-function(FLATTEN_LIST INPUT SEPARATOR OUTPUT)
- string (REPLACE ";" "${SEPARATOR}" _TMPS "${INPUT}")
- set (${OUTPUT} "${_TMPS}" PARENT_SCOPE)
-endfunction()
+# The caller must specify where the generated headers have been placed.
+if(NOT GENERATED_JAVAH)
+ message(FATAL_ERROR "You must set the CMake variable GENERATED_JAVAH")
+endif()
# Check to see if our compiler and linker support the __thread attribute.
-# On Linux and some other operating systems, this is a more efficient
-# alternative to POSIX thread local storage.
-INCLUDE(CheckCSourceCompiles)
-CHECK_C_SOURCE_COMPILES("int main(void) { static __thread int i = 0; return 0; }" HAVE_BETTER_TLS)
+# On Linux and some other operating systems, this is a more efficient
+# alternative to POSIX thread local storage.
+include(CheckCSourceCompiles)
+check_c_source_compiles("int main(void) { static __thread int i = 0; return 0; }" HAVE_BETTER_TLS)
# Check to see if we have Intel SSE intrinsics.
-CHECK_C_SOURCE_COMPILES("#include <emmintrin.h>\nint main(void) { __m128d sum0 = _mm_set_pd(0.0,0.0); return 0; }" HAVE_INTEL_SSE_INTRINSICS)
+check_c_source_compileS("#include <emmintrin.h>\nint main(void) { __m128d sum0 = _mm_set_pd(0.0,0.0); return 0; }" HAVE_INTEL_SSE_INTRINSICS)
# Check if we need to link dl library to get dlopen.
# dlopen on Linux is in separate library but on FreeBSD its in libc
-INCLUDE(CheckLibraryExists)
-CHECK_LIBRARY_EXISTS(dl dlopen "" NEED_LINK_DL)
+include(CheckLibraryExists)
+check_library_exists(dl dlopen "" NEED_LINK_DL)
-find_package(JNI REQUIRED)
-if (NOT GENERATED_JAVAH)
- # Must identify where the generated headers have been placed
- MESSAGE(FATAL_ERROR "You must set the CMake variable GENERATED_JAVAH")
-endif (NOT GENERATED_JAVAH)
-
-if (WIN32)
+if(WIN32)
+ # Set the optimizer level.
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /O2")
-
# Set warning level 4.
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /W4")
-
# Skip "unreferenced formal parameter".
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /wd4100")
-
# Skip "conditional expression is constant".
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} /wd4127")
-
# Skip deprecated POSIX function warnings.
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D_CRT_NONSTDC_NO_DEPRECATE")
-
# Skip CRT non-secure function warnings. If we can convert usage of
# strerror, getenv and ctime to their secure CRT equivalents, then we can
# re-enable the CRT non-secure function warnings.
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D_CRT_SECURE_NO_WARNINGS")
-
# Omit unneeded headers.
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -DWIN32_LEAN_AND_MEAN")
-
set(OS_DIR main/native/libhdfs/os/windows)
set(OUT_DIR target/bin)
-else (WIN32)
- set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -g -Wall -O2")
- set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D_REENTRANT -D_GNU_SOURCE")
- set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -D_LARGEFILE_SOURCE -D_FILE_OFFSET_BITS=64")
+else()
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -fvisibility=hidden")
set(OS_DIR main/native/libhdfs/os/posix)
- set(OS_LINK_LIBRARIES pthread)
set(OUT_DIR target/usr/local/lib)
-endif (WIN32)
+endif()
+
+# Configure JNI.
+include(HadoopJNI)
add_definitions(-DLIBHDFS_DLL_EXPORT)
@@ -132,9 +85,9 @@ include_directories(
)
set(_FUSE_DFS_VERSION 0.1.0)
-CONFIGURE_FILE(${CMAKE_SOURCE_DIR}/config.h.cmake ${CMAKE_BINARY_DIR}/config.h)
+configure_file(${CMAKE_SOURCE_DIR}/config.h.cmake ${CMAKE_BINARY_DIR}/config.h)
-add_dual_library(hdfs
+hadoop_add_dual_library(hdfs
main/native/libhdfs/exception.c
main/native/libhdfs/jni_helper.c
main/native/libhdfs/hdfs.c
@@ -142,17 +95,17 @@ add_dual_library(hdfs
${OS_DIR}/mutexes.c
${OS_DIR}/thread_local_storage.c
)
-if (NEED_LINK_DL)
+if(NEED_LINK_DL)
set(LIB_DL dl)
-endif(NEED_LINK_DL)
+endif()
-target_link_dual_libraries(hdfs
+hadoop_target_link_dual_libraries(hdfs
${JAVA_JVM_LIBRARY}
${LIB_DL}
${OS_LINK_LIBRARIES}
)
-dual_output_directory(hdfs ${OUT_DIR})
+hadoop_dual_output_directory(hdfs ${OUT_DIR})
set(LIBHDFS_VERSION "0.0.0")
set_target_properties(hdfs PROPERTIES
SOVERSION ${LIBHDFS_VERSION})
@@ -232,25 +185,21 @@ target_link_libraries(test_htable
# Skip vecsum on Windows. This could be made to work in the future by
# introducing an abstraction layer over the sys/mman.h functions.
-if (NOT WIN32)
- add_executable(test_libhdfs_vecsum
- main/native/libhdfs/test/vecsum.c
- )
- if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
+if(NOT WIN32)
+ add_executable(test_libhdfs_vecsum main/native/libhdfs/test/vecsum.c)
+ if(CMAKE_SYSTEM_NAME MATCHES "Darwin")
target_link_libraries(test_libhdfs_vecsum
hdfs
- pthread
- )
- else (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
+ pthread)
+ else()
target_link_libraries(test_libhdfs_vecsum
hdfs
pthread
- rt
- )
- endif (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
-endif(NOT WIN32)
+ rt)
+ endif()
+endif()
-IF(REQUIRE_LIBWEBHDFS)
+if(REQUIRE_LIBWEBHDFS)
add_subdirectory(contrib/libwebhdfs)
-ENDIF(REQUIRE_LIBWEBHDFS)
+endif()
add_subdirectory(main/native/fuse-dfs)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/29df73dc/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/CMakeLists.txt
index 44f0c42..dc74feb 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/CMakeLists.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/CMakeLists.txt
@@ -19,12 +19,12 @@
find_package(CURL REQUIRED)
set(CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH}
-"${CMAKE_SOURCE_DIR}/contrib/libwebhdfs/resources/")
+ "${CMAKE_SOURCE_DIR}/contrib/libwebhdfs/resources/")
find_package(Jansson REQUIRED)
include_directories(${JANSSON_INCLUDE_DIR})
-add_dual_library(webhdfs
+hadoop_add_dual_library(webhdfs
src/hdfs_web.c
src/hdfs_http_client.c
src/hdfs_http_query.c
@@ -35,13 +35,13 @@ add_dual_library(webhdfs
../../${OS_DIR}/thread_local_storage.c
../../main/native/libhdfs/common/htable.c
)
-target_link_dual_libraries(webhdfs
+hadoop_target_link_dual_libraries(webhdfs
${JAVA_JVM_LIBRARY}
${CURL_LIBRARY}
${JANSSON_LIBRARY}
pthread
)
-dual_output_directory(webhdfs target)
+hadoop_dual_output_directory(webhdfs target)
set(LIBWEBHDFS_VERSION "0.0.0")
set_target_properties(webhdfs PROPERTIES
SOVERSION ${LIBWEBHDFS_VERSION})
http://git-wip-us.apache.org/repos/asf/hadoop/blob/29df73dc/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/resources/FindJansson.cmake
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/resources/FindJansson.cmake b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/resources/FindJansson.cmake
index cb8c8db..b8c67ea 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/resources/FindJansson.cmake
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/contrib/libwebhdfs/resources/FindJansson.cmake
@@ -5,9 +5,9 @@
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License. You may obtain a copy of the License at
-#
+#
# http://www.apache.org/licenses/LICENSE-2.0
-#
+#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
@@ -24,7 +24,8 @@
# JANSSON_DEFINITIONS - Compiler switches required for using Jansson
find_path(JANSSON_INCLUDE_DIR jansson.h
- /usr/incluce
+ /usr/include
+ /usr/include/jansson
/usr/local/include )
find_library(JANSSON_LIBRARY NAMES jansson
@@ -39,4 +40,4 @@ include(FindPackageHandleStandardArgs)
find_package_handle_standard_args(Jansson DEFAULT_MSG
JANSSON_LIBRARY JANSSON_INCLUDE_DIR)
-mark_as_advanced(JANSSON_INCLUDE_DIR JANSSON_LIBRARY )
\ No newline at end of file
+mark_as_advanced(JANSSON_INCLUDE_DIR JANSSON_LIBRARY )
http://git-wip-us.apache.org/repos/asf/hadoop/blob/29df73dc/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/CMakeLists.txt b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/CMakeLists.txt
index 8a8ea5b..7a5e5a1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/CMakeLists.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/native/fuse-dfs/CMakeLists.txt
@@ -18,57 +18,63 @@
set(CMAKE_SKIP_RPATH TRUE)
+# Flatten a list into a string.
+function(flatten_list INPUT SEPARATOR OUTPUT)
+ string (REPLACE ";" "${SEPARATOR}" _TMPS "${INPUT}")
+ set (${OUTPUT} "${_TMPS}" PARENT_SCOPE)
+endfunction()
+
# Find Linux FUSE
-IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux")
+if(${CMAKE_SYSTEM_NAME} MATCHES "Linux")
find_package(PkgConfig REQUIRED)
pkg_check_modules(FUSE fuse)
- IF(FUSE_FOUND)
- FLATTEN_LIST("${FUSE_CFLAGS}" " " FUSE_CFLAGS)
- FLATTEN_LIST("${FUSE_LDFLAGS}" " " FUSE_LDFLAGS)
+ if(FUSE_FOUND)
+ flatten_list("${FUSE_CFLAGS}" " " FUSE_CFLAGS)
+ flatten_list("${FUSE_LDFLAGS}" " " FUSE_LDFLAGS)
set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${FUSE_CFLAGS}")
set(CMAKE_LD_FLAGS "${CMAKE_LD_FLAGS} ${FUSE_LDFLAGS}")
- MESSAGE(STATUS "Building Linux FUSE client.")
+ message(STATUS "Building Linux FUSE client.")
include_directories(${FUSE_INCLUDE_DIRS})
- ELSE(FUSE_FOUND)
- MESSAGE(STATUS "Failed to find Linux FUSE libraries or include files. Will not build FUSE client.")
- ENDIF(FUSE_FOUND)
-ELSE (${CMAKE_SYSTEM_NAME} MATCHES "Linux")
- MESSAGE(STATUS "Non-Linux system detected. Will not build FUSE client.")
-ENDIF (${CMAKE_SYSTEM_NAME} MATCHES "Linux")
+ else()
+ message(STATUS "Failed to find Linux FUSE libraries or include files. Will not build FUSE client.")
+ endif(FUSE_FOUND)
+else()
+ message(STATUS "Non-Linux system detected. Will not build FUSE client.")
+endif()
-IF(FUSE_FOUND)
+if(FUSE_FOUND)
add_library(posix_util
../util/posix_util.c
)
add_executable(fuse_dfs
fuse_dfs.c
- fuse_options.c
- fuse_connect.c
- fuse_impls_access.c
- fuse_impls_chmod.c
- fuse_impls_chown.c
- fuse_impls_create.c
- fuse_impls_flush.c
- fuse_impls_getattr.c
- fuse_impls_mkdir.c
- fuse_impls_mknod.c
- fuse_impls_open.c
- fuse_impls_read.c
- fuse_impls_readdir.c
- fuse_impls_release.c
- fuse_impls_rename.c
- fuse_impls_rmdir.c
- fuse_impls_statfs.c
- fuse_impls_symlink.c
- fuse_impls_truncate.c
- fuse_impls_unlink.c
- fuse_impls_utimens.c
+ fuse_options.c
+ fuse_connect.c
+ fuse_impls_access.c
+ fuse_impls_chmod.c
+ fuse_impls_chown.c
+ fuse_impls_create.c
+ fuse_impls_flush.c
+ fuse_impls_getattr.c
+ fuse_impls_mkdir.c
+ fuse_impls_mknod.c
+ fuse_impls_open.c
+ fuse_impls_read.c
+ fuse_impls_readdir.c
+ fuse_impls_release.c
+ fuse_impls_rename.c
+ fuse_impls_rmdir.c
+ fuse_impls_statfs.c
+ fuse_impls_symlink.c
+ fuse_impls_truncate.c
+ fuse_impls_unlink.c
+ fuse_impls_utimens.c
fuse_impls_write.c
- fuse_init.c
- fuse_stat_struct.c
- fuse_trash.c
- fuse_users.c
+ fuse_init.c
+ fuse_stat_struct.c
+ fuse_trash.c
+ fuse_users.c
)
target_link_libraries(fuse_dfs
${FUSE_LIBRARIES}
@@ -88,8 +94,9 @@ IF(FUSE_FOUND)
posix_util
pthread
)
-ELSE(FUSE_FOUND)
- IF(REQUIRE_FUSE)
- MESSAGE(FATAL_ERROR "Required component fuse_dfs could not be built.")
- ENDIF(REQUIRE_FUSE)
-ENDIF(FUSE_FOUND)
+ set(REQUIRE_FUSE ${REQUIRE_FUSE}) # Prevent unused variable warning
+else()
+ if(REQUIRE_FUSE)
+ message(FATAL_ERROR "Required component fuse_dfs could not be built.")
+ endif()
+endif()
[12/46] hadoop git commit: HDFS-8653. Code cleanup for
DatanodeManager,
DatanodeDescriptor and DatanodeStorageInfo. Contributed by Zhe Zhang.
Posted by ar...@apache.org.
HDFS-8653. Code cleanup for DatanodeManager, DatanodeDescriptor and DatanodeStorageInfo. Contributed by Zhe Zhang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2ffd8427
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2ffd8427
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2ffd8427
Branch: refs/heads/HDFS-7240
Commit: 2ffd84273ac490724fe7e7825664bb6d09ef0e99
Parents: d3fed8e
Author: Andrew Wang <wa...@apache.org>
Authored: Mon Jun 29 12:12:34 2015 -0700
Committer: Andrew Wang <wa...@apache.org>
Committed: Mon Jun 29 12:12:41 2015 -0700
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 ++
.../CacheReplicationMonitor.java | 4 +--
.../blockmanagement/DatanodeDescriptor.java | 33 ++++++++++----------
.../server/blockmanagement/DatanodeManager.java | 26 +++++++--------
.../blockmanagement/DatanodeStorageInfo.java | 13 ++++++--
5 files changed, 45 insertions(+), 34 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ffd8427/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index e55f340..0c56f2b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -682,6 +682,9 @@ Release 2.8.0 - UNRELEASED
HDFS-8623. Refactor NameNode handling of invalid, corrupt, and under-recovery
blocks. (Zhe Zhang via jing9)
+ HDFS-8653. Code cleanup for DatanodeManager, DatanodeDescriptor and
+ DatanodeStorageInfo. (Zhe Zhang via wang)
+
OPTIMIZATIONS
HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ffd8427/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CacheReplicationMonitor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CacheReplicationMonitor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CacheReplicationMonitor.java
index a0f3503..2f81ddf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CacheReplicationMonitor.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/CacheReplicationMonitor.java
@@ -452,8 +452,8 @@ public class CacheReplicationMonitor extends Thread implements Closeable {
file.getFullPathName(), cachedTotal, neededTotal);
}
- private String findReasonForNotCaching(CachedBlock cblock,
- BlockInfo blockInfo) {
+ private String findReasonForNotCaching(CachedBlock cblock,
+ BlockInfo blockInfo) {
if (blockInfo == null) {
// Somehow, a cache report with the block arrived, but the block
// reports from the DataNode haven't (yet?) described such a block.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ffd8427/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
index dd7b301..99def6b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
@@ -64,7 +64,8 @@ public class DatanodeDescriptor extends DatanodeInfo {
// Stores status of decommissioning.
// If node is not decommissioning, do not use this object for anything.
- public final DecommissioningStatus decommissioningStatus = new DecommissioningStatus();
+ public final DecommissioningStatus decommissioningStatus =
+ new DecommissioningStatus();
private long curBlockReportId = 0;
@@ -115,7 +116,7 @@ public class DatanodeDescriptor extends DatanodeInfo {
return null;
}
- List<E> results = new ArrayList<E>();
+ List<E> results = new ArrayList<>();
for(; !blockq.isEmpty() && numBlocks > 0; numBlocks--) {
results.add(blockq.poll());
}
@@ -135,7 +136,7 @@ public class DatanodeDescriptor extends DatanodeInfo {
}
private final Map<String, DatanodeStorageInfo> storageMap =
- new HashMap<String, DatanodeStorageInfo>();
+ new HashMap<>();
/**
* A list of CachedBlock objects on this datanode.
@@ -217,12 +218,14 @@ public class DatanodeDescriptor extends DatanodeInfo {
private long bandwidth;
/** A queue of blocks to be replicated by this datanode */
- private final BlockQueue<BlockTargetPair> replicateBlocks = new BlockQueue<BlockTargetPair>();
+ private final BlockQueue<BlockTargetPair> replicateBlocks =
+ new BlockQueue<>();
/** A queue of blocks to be recovered by this datanode */
private final BlockQueue<BlockInfoUnderConstruction> recoverBlocks =
- new BlockQueue<BlockInfoUnderConstruction>();
+ new BlockQueue<>();
/** A set of blocks to be invalidated by this datanode */
- private final LightWeightHashSet<Block> invalidateBlocks = new LightWeightHashSet<Block>();
+ private final LightWeightHashSet<Block> invalidateBlocks =
+ new LightWeightHashSet<>();
/* Variables for maintaining number of blocks scheduled to be written to
* this storage. This count is approximate and might be slightly bigger
@@ -230,9 +233,9 @@ public class DatanodeDescriptor extends DatanodeInfo {
* while writing the block).
*/
private EnumCounters<StorageType> currApproxBlocksScheduled
- = new EnumCounters<StorageType>(StorageType.class);
+ = new EnumCounters<>(StorageType.class);
private EnumCounters<StorageType> prevApproxBlocksScheduled
- = new EnumCounters<StorageType>(StorageType.class);
+ = new EnumCounters<>(StorageType.class);
private long lastBlocksScheduledRollTime = 0;
private static final int BLOCKS_SCHEDULED_ROLL_INTERVAL = 600*1000; //10min
private int volumeFailures = 0;
@@ -276,6 +279,7 @@ public class DatanodeDescriptor extends DatanodeInfo {
return storageMap.get(storageID);
}
}
+
DatanodeStorageInfo[] getStorageInfos() {
synchronized (storageMap) {
final Collection<DatanodeStorageInfo> storages = storageMap.values();
@@ -321,7 +325,7 @@ public class DatanodeDescriptor extends DatanodeInfo {
Long.toHexString(curBlockReportId));
iter.remove();
if (zombies == null) {
- zombies = new LinkedList<DatanodeStorageInfo>();
+ zombies = new LinkedList<>();
}
zombies.add(storageInfo);
}
@@ -350,10 +354,7 @@ public class DatanodeDescriptor extends DatanodeInfo {
*/
boolean removeBlock(String storageID, BlockInfo b) {
DatanodeStorageInfo s = getStorageInfo(storageID);
- if (s != null) {
- return s.removeBlock(b);
- }
- return false;
+ return s != null && s.removeBlock(b);
}
public void resetBlocks() {
@@ -449,7 +450,7 @@ public class DatanodeDescriptor extends DatanodeInfo {
+ this.volumeFailures + " to " + volFailures);
synchronized (storageMap) {
failedStorageInfos =
- new HashSet<DatanodeStorageInfo>(storageMap.values());
+ new HashSet<>(storageMap.values());
}
}
@@ -505,7 +506,7 @@ public class DatanodeDescriptor extends DatanodeInfo {
HashMap<String, DatanodeStorageInfo> excessStorages;
// Init excessStorages with all known storages.
- excessStorages = new HashMap<String, DatanodeStorageInfo>(storageMap);
+ excessStorages = new HashMap<>(storageMap);
// Remove storages that the DN reported in the heartbeat.
for (final StorageReport report : reports) {
@@ -542,7 +543,7 @@ public class DatanodeDescriptor extends DatanodeInfo {
private final List<Iterator<BlockInfo>> iterators;
private BlockIterator(final DatanodeStorageInfo... storages) {
- List<Iterator<BlockInfo>> iterators = new ArrayList<Iterator<BlockInfo>>();
+ List<Iterator<BlockInfo>> iterators = new ArrayList<>();
for (DatanodeStorageInfo e : storages) {
iterators.add(e.getBlockIterator());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ffd8427/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
index 8143fb4..4266004 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
@@ -85,7 +85,7 @@ public class DatanodeManager {
* Mapping: StorageID -> DatanodeDescriptor
*/
private final Map<String, DatanodeDescriptor> datanodeMap
- = new HashMap<String, DatanodeDescriptor>();
+ = new HashMap<>();
/** Cluster network topology */
private final NetworkTopology networktopology;
@@ -162,7 +162,7 @@ public class DatanodeManager {
* Software version -> Number of datanodes with this version
*/
private HashMap<String, Integer> datanodesSoftwareVersions =
- new HashMap<String, Integer>(4, 0.75f);
+ new HashMap<>(4, 0.75f);
/**
* The minimum time between resending caching directives to Datanodes,
@@ -217,7 +217,7 @@ public class DatanodeManager {
// locations of those hosts in the include list and store the mapping
// in the cache; so future calls to resolve will be fast.
if (dnsToSwitchMapping instanceof CachedDNSToSwitchMapping) {
- final ArrayList<String> locations = new ArrayList<String>();
+ final ArrayList<String> locations = new ArrayList<>();
for (InetSocketAddress addr : hostFileManager.getIncludes()) {
locations.add(addr.getAddress().getHostAddress());
}
@@ -370,7 +370,7 @@ public class DatanodeManager {
// here we should get node but not datanode only .
Node client = getDatanodeByHost(targethost);
if (client == null) {
- List<String> hosts = new ArrayList<String> (1);
+ List<String> hosts = new ArrayList<> (1);
hosts.add(targethost);
List<String> resolvedHosts = dnsToSwitchMapping.resolve(hosts);
if (resolvedHosts != null && !resolvedHosts.isEmpty()) {
@@ -522,7 +522,7 @@ public class DatanodeManager {
void datanodeDump(final PrintWriter out) {
synchronized (datanodeMap) {
Map<String,DatanodeDescriptor> sortedDatanodeMap =
- new TreeMap<String,DatanodeDescriptor>(datanodeMap);
+ new TreeMap<>(datanodeMap);
out.println("Metasave: Number of datanodes: " + datanodeMap.size());
for (DatanodeDescriptor node : sortedDatanodeMap.values()) {
out.println(node.dumpDatanode());
@@ -660,7 +660,7 @@ public class DatanodeManager {
private void countSoftwareVersions() {
synchronized(datanodeMap) {
- HashMap<String, Integer> versionCount = new HashMap<String, Integer>();
+ HashMap<String, Integer> versionCount = new HashMap<>();
for(DatanodeDescriptor dn: datanodeMap.values()) {
// Check isAlive too because right after removeDatanode(),
// isDatanodeDead() is still true
@@ -677,7 +677,7 @@ public class DatanodeManager {
public HashMap<String, Integer> getDatanodesSoftwareVersions() {
synchronized(datanodeMap) {
- return new HashMap<String, Integer> (this.datanodesSoftwareVersions);
+ return new HashMap<> (this.datanodesSoftwareVersions);
}
}
@@ -710,7 +710,7 @@ public class DatanodeManager {
*/
private String resolveNetworkLocation (DatanodeID node)
throws UnresolvedTopologyException {
- List<String> names = new ArrayList<String>(1);
+ List<String> names = new ArrayList<>(1);
if (dnsToSwitchMapping instanceof CachedDNSToSwitchMapping) {
names.add(node.getIpAddr());
} else {
@@ -1000,7 +1000,7 @@ public class DatanodeManager {
// If the network location is invalid, clear the cached mappings
// so that we have a chance to re-add this DataNode with the
// correct network location later.
- List<String> invalidNodeNames = new ArrayList<String>(3);
+ List<String> invalidNodeNames = new ArrayList<>(3);
// clear cache for nodes in IP or Hostname
invalidNodeNames.add(nodeReg.getIpAddr());
invalidNodeNames.add(nodeReg.getHostName());
@@ -1275,7 +1275,7 @@ public class DatanodeManager {
final HostFileManager.HostSet excludedNodes = hostFileManager.getExcludes();
synchronized(datanodeMap) {
- nodes = new ArrayList<DatanodeDescriptor>(datanodeMap.size());
+ nodes = new ArrayList<>(datanodeMap.size());
for (DatanodeDescriptor dn : datanodeMap.values()) {
final boolean isDead = isDatanodeDead(dn);
final boolean isDecommissioning = dn.isDecommissionInProgress();
@@ -1351,7 +1351,7 @@ public class DatanodeManager {
VolumeFailureSummary volumeFailureSummary) throws IOException {
synchronized (heartbeatManager) {
synchronized (datanodeMap) {
- DatanodeDescriptor nodeinfo = null;
+ DatanodeDescriptor nodeinfo;
try {
nodeinfo = getDatanode(nodeReg);
} catch(UnregisteredNodeException e) {
@@ -1389,7 +1389,7 @@ public class DatanodeManager {
final DatanodeStorageInfo[] storages = b.getExpectedStorageLocations();
// Skip stale nodes during recovery - not heart beated for some time (30s by default).
final List<DatanodeStorageInfo> recoveryLocations =
- new ArrayList<DatanodeStorageInfo>(storages.length);
+ new ArrayList<>(storages.length);
for (int i = 0; i < storages.length; i++) {
if (!storages[i].getDatanodeDescriptor().isStale(staleInterval)) {
recoveryLocations.add(storages[i]);
@@ -1431,7 +1431,7 @@ public class DatanodeManager {
return new DatanodeCommand[] { brCommand };
}
- final List<DatanodeCommand> cmds = new ArrayList<DatanodeCommand>();
+ final List<DatanodeCommand> cmds = new ArrayList<>();
//check pending replication
List<BlockTargetPair> pendingList = nodeinfo.getReplicationCommand(
maxTransfers);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ffd8427/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
index 65b83e1..92841a6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeStorageInfo.java
@@ -37,8 +37,9 @@ import org.apache.hadoop.hdfs.server.protocol.StorageReport;
public class DatanodeStorageInfo {
public static final DatanodeStorageInfo[] EMPTY_ARRAY = {};
- public static DatanodeInfo[] toDatanodeInfos(DatanodeStorageInfo[] storages) {
- return toDatanodeInfos(Arrays.asList(storages));
+ public static DatanodeInfo[] toDatanodeInfos(
+ DatanodeStorageInfo[] storages) {
+ return storages == null ? null: toDatanodeInfos(Arrays.asList(storages));
}
static DatanodeInfo[] toDatanodeInfos(List<DatanodeStorageInfo> storages) {
final DatanodeInfo[] datanodes = new DatanodeInfo[storages.size()];
@@ -58,6 +59,9 @@ public class DatanodeStorageInfo {
}
public static String[] toStorageIDs(DatanodeStorageInfo[] storages) {
+ if (storages == null) {
+ return null;
+ }
String[] storageIDs = new String[storages.length];
for(int i = 0; i < storageIDs.length; i++) {
storageIDs[i] = storages[i].getStorageID();
@@ -66,6 +70,9 @@ public class DatanodeStorageInfo {
}
public static StorageType[] toStorageTypes(DatanodeStorageInfo[] storages) {
+ if (storages == null) {
+ return null;
+ }
StorageType[] storageTypes = new StorageType[storages.length];
for(int i = 0; i < storageTypes.length; i++) {
storageTypes[i] = storages[i].getStorageType();
@@ -380,6 +387,6 @@ public class DatanodeStorageInfo {
}
static enum AddBlockResult {
- ADDED, REPLACED, ALREADY_EXIST;
+ ADDED, REPLACED, ALREADY_EXIST
}
}
[29/46] hadoop git commit: YARN-3768. ArrayIndexOutOfBoundsException
with empty environment variables. (Zhihai Xu via gera)
Posted by ar...@apache.org.
YARN-3768. ArrayIndexOutOfBoundsException with empty environment variables. (Zhihai Xu via gera)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/6f2a41e3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/6f2a41e3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/6f2a41e3
Branch: refs/heads/HDFS-7240
Commit: 6f2a41e37d0b36cdafcfff75125165f212c612a6
Parents: b6ba564
Author: Gera Shegalov <ge...@apache.org>
Authored: Tue Jun 30 14:44:48 2015 -0700
Committer: Gera Shegalov <ge...@apache.org>
Committed: Tue Jun 30 14:59:33 2015 -0700
----------------------------------------------------------------------
.../main/java/org/apache/hadoop/util/Shell.java | 6 +-
hadoop-yarn-project/CHANGES.txt | 3 +
.../java/org/apache/hadoop/yarn/util/Apps.java | 19 ++++--
.../org/apache/hadoop/yarn/util/TestApps.java | 61 ++++++++++++++++++++
4 files changed, 81 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f2a41e3/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
index 45c1588..ed83e8d 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/Shell.java
@@ -221,10 +221,12 @@ abstract public class Shell {
new String[] { "kill", "-" + code, isSetsidAvailable ? "-" + pid : pid };
}
+ public static final String ENV_NAME_REGEX = "[A-Za-z_][A-Za-z0-9_]*";
/** Return a regular expression string that match environment variables */
public static String getEnvironmentVariableRegex() {
- return (WINDOWS) ? "%([A-Za-z_][A-Za-z0-9_]*?)%" :
- "\\$([A-Za-z_][A-Za-z0-9_]*)";
+ return (WINDOWS)
+ ? "%(" + ENV_NAME_REGEX + "?)%"
+ : "\\$(" + ENV_NAME_REGEX + ")";
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f2a41e3/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 8461f69..94bc8fc 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -574,6 +574,9 @@ Release 2.8.0 - UNRELEASED
YARN-3770. SerializedException should also handle java.lang.Error on
de-serialization. (Lavkesh Lahngir via jianhe)
+ YARN-3768. ArrayIndexOutOfBoundsException with empty environment variables.
+ (Zhihai Xu via gera)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f2a41e3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java
index cf3940f..9235e7d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Apps.java
@@ -44,6 +44,14 @@ import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
public class Apps {
public static final String APP = "application";
public static final String ID = "ID";
+ private static final Pattern VAR_SUBBER =
+ Pattern.compile(Shell.getEnvironmentVariableRegex());
+ private static final Pattern VARVAL_SPLITTER = Pattern.compile(
+ "(?<=^|,)" // preceded by ',' or line begin
+ + '(' + Shell.ENV_NAME_REGEX + ')' // var group
+ + '='
+ + "([^,]*)" // val group
+ );
public static ApplicationId toAppID(String aid) {
Iterator<String> it = _split(aid).iterator();
@@ -73,11 +81,10 @@ public class Apps {
public static void setEnvFromInputString(Map<String, String> env,
String envString, String classPathSeparator) {
if (envString != null && envString.length() > 0) {
- String childEnvs[] = envString.split(",");
- Pattern p = Pattern.compile(Shell.getEnvironmentVariableRegex());
- for (String cEnv : childEnvs) {
- String[] parts = cEnv.split("="); // split on '='
- Matcher m = p.matcher(parts[1]);
+ Matcher varValMatcher = VARVAL_SPLITTER.matcher(envString);
+ while (varValMatcher.find()) {
+ String envVar = varValMatcher.group(1);
+ Matcher m = VAR_SUBBER.matcher(varValMatcher.group(2));
StringBuffer sb = new StringBuffer();
while (m.find()) {
String var = m.group(1);
@@ -93,7 +100,7 @@ public class Apps {
m.appendReplacement(sb, Matcher.quoteReplacement(replace));
}
m.appendTail(sb);
- addToEnvironment(env, parts[0], sb.toString(), classPathSeparator);
+ addToEnvironment(env, envVar, sb.toString(), classPathSeparator);
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/6f2a41e3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestApps.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestApps.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestApps.java
new file mode 100644
index 0000000..66d2d23
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/util/TestApps.java
@@ -0,0 +1,61 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.hadoop.yarn.util;
+
+import org.apache.hadoop.util.Shell;
+import org.junit.Test;
+
+import java.io.File;
+import java.util.HashMap;
+import java.util.Map;
+
+import static org.junit.Assert.assertEquals;
+
+public class TestApps {
+ @Test
+ public void testSetEnvFromInputString() {
+ Map<String, String> environment = new HashMap<String, String>();
+ environment.put("JAVA_HOME", "/path/jdk");
+ String goodEnv = "a1=1,b_2=2,_c=3,d=4,e=,f_win=%JAVA_HOME%"
+ + ",g_nix=$JAVA_HOME";
+ Apps.setEnvFromInputString(environment, goodEnv, File.pathSeparator);
+ assertEquals("1", environment.get("a1"));
+ assertEquals("2", environment.get("b_2"));
+ assertEquals("3", environment.get("_c"));
+ assertEquals("4", environment.get("d"));
+ assertEquals("", environment.get("e"));
+ if (Shell.WINDOWS) {
+ assertEquals("$JAVA_HOME", environment.get("g_nix"));
+ assertEquals("/path/jdk", environment.get("f_win"));
+ } else {
+ assertEquals("/path/jdk", environment.get("g_nix"));
+ assertEquals("%JAVA_HOME%", environment.get("f_win"));
+ }
+ String badEnv = "1,,2=a=b,3=a=,4==,5==a,==,c-3=3,=";
+ environment.clear();
+ Apps.setEnvFromInputString(environment, badEnv, File.pathSeparator);
+ assertEquals(environment.size(), 0);
+
+ // Test "=" in the value part
+ environment.clear();
+ Apps.setEnvFromInputString(environment, "b1,e1==,e2=a1=a2,b2",
+ File.pathSeparator);
+ assertEquals("=", environment.get("e1"));
+ assertEquals("a1=a2", environment.get("e2"));
+ }
+}
[42/46] hadoop git commit: MAPREDUCE-6376. Add avro binary support
for jhist files. Contributed by Ray Chiang
Posted by ar...@apache.org.
MAPREDUCE-6376. Add avro binary support for jhist files. Contributed by Ray Chiang
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2ac87df5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2ac87df5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2ac87df5
Branch: refs/heads/HDFS-7240
Commit: 2ac87df578accb6e612f70ded76271cb5082ee10
Parents: 80a68d6
Author: Jason Lowe <jl...@apache.org>
Authored: Wed Jul 1 16:00:03 2015 +0000
Committer: Jason Lowe <jl...@apache.org>
Committed: Wed Jul 1 16:00:03 2015 +0000
----------------------------------------------------------------------
hadoop-mapreduce-project/CHANGES.txt | 3 ++
.../jobhistory/JobHistoryEventHandler.java | 19 +++++++++++--
.../hadoop/mapreduce/jobhistory/TestEvents.java | 3 +-
.../mapreduce/v2/jobhistory/JHAdminConfig.java | 7 +++++
.../mapreduce/jobhistory/EventReader.java | 12 ++++----
.../mapreduce/jobhistory/EventWriter.java | 29 ++++++++++++++++----
.../src/main/resources/mapred-default.xml | 9 ++++++
7 files changed, 69 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ac87df5/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index 0baecf8..409f074 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -372,6 +372,9 @@ Release 2.8.0 - UNRELEASED
OPTIMIZATIONS
+ MAPREDUCE-6376. Add avro binary support for jhist files (Ray Chiang via
+ jlowe)
+
BUG FIXES
MAPREDUCE-6314. TestPipeApplication fails on trunk.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ac87df5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
index 35556a6..0457cc5 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/jobhistory/JobHistoryEventHandler.java
@@ -105,7 +105,8 @@ public class JobHistoryEventHandler extends AbstractService
private int numUnflushedCompletionEvents = 0;
private boolean isTimerActive;
-
+ private EventWriter.WriteMode jhistMode =
+ EventWriter.WriteMode.JSON;
protected BlockingQueue<JobHistoryEvent> eventQueue =
new LinkedBlockingQueue<JobHistoryEvent>();
@@ -260,6 +261,20 @@ public class JobHistoryEventHandler extends AbstractService
LOG.info("Emitting job history data to the timeline server is not enabled");
}
+ // Flag for setting
+ String jhistFormat = conf.get(JHAdminConfig.MR_HS_JHIST_FORMAT,
+ JHAdminConfig.DEFAULT_MR_HS_JHIST_FORMAT);
+ if (jhistFormat.equals("json")) {
+ jhistMode = EventWriter.WriteMode.JSON;
+ } else if (jhistFormat.equals("binary")) {
+ jhistMode = EventWriter.WriteMode.BINARY;
+ } else {
+ LOG.warn("Unrecognized value '" + jhistFormat + "' for property " +
+ JHAdminConfig.MR_HS_JHIST_FORMAT + ". Valid values are " +
+ "'json' or 'binary'. Falling back to default value '" +
+ JHAdminConfig.DEFAULT_MR_HS_JHIST_FORMAT + "'.");
+ }
+
super.serviceInit(conf);
}
@@ -418,7 +433,7 @@ public class JobHistoryEventHandler extends AbstractService
protected EventWriter createEventWriter(Path historyFilePath)
throws IOException {
FSDataOutputStream out = stagingDirFS.create(historyFilePath, true);
- return new EventWriter(out);
+ return new EventWriter(out, this.jhistMode);
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ac87df5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestEvents.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestEvents.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestEvents.java
index 597f7a0..7612ceb 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestEvents.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/jobhistory/TestEvents.java
@@ -190,7 +190,8 @@ public class TestEvents {
ByteArrayOutputStream output = new ByteArrayOutputStream();
FSDataOutputStream fsOutput = new FSDataOutputStream(output,
new FileSystem.Statistics("scheme"));
- EventWriter writer = new EventWriter(fsOutput);
+ EventWriter writer = new EventWriter(fsOutput,
+ EventWriter.WriteMode.JSON);
writer.write(getJobPriorityChangedEvent());
writer.write(getJobStatusChangedEvent());
writer.write(getTaskUpdatedEvent());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ac87df5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java
index a97c2ca..86dfad3 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/src/main/java/org/apache/hadoop/mapreduce/v2/jobhistory/JHAdminConfig.java
@@ -221,4 +221,11 @@ public class JHAdminConfig {
+ "jobname.limit";
public static final int DEFAULT_MR_HS_JOBNAME_LIMIT = 50;
+ /**
+ * Settings for .jhist file format.
+ */
+ public static final String MR_HS_JHIST_FORMAT =
+ MR_HISTORY_PREFIX + "jhist.format";
+ public static final String DEFAULT_MR_HS_JHIST_FORMAT =
+ "json";
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ac87df5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/EventReader.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/EventReader.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/EventReader.java
index e08a929..9898c2d 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/EventReader.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/EventReader.java
@@ -66,16 +66,18 @@ public class EventReader implements Closeable {
public EventReader(DataInputStream in) throws IOException {
this.in = in;
this.version = in.readLine();
-
- if (!EventWriter.VERSION.equals(version)) {
- throw new IOException("Incompatible event log version: "+version);
- }
Schema myschema = new SpecificData(Event.class.getClassLoader()).getSchema(Event.class);
Schema.Parser parser = new Schema.Parser();
this.schema = parser.parse(in.readLine());
this.reader = new SpecificDatumReader(schema, myschema);
- this.decoder = DecoderFactory.get().jsonDecoder(schema, in);
+ if (EventWriter.VERSION.equals(version)) {
+ this.decoder = DecoderFactory.get().jsonDecoder(schema, in);
+ } else if (EventWriter.VERSION_BINARY.equals(version)) {
+ this.decoder = DecoderFactory.get().binaryDecoder(in, null);
+ } else {
+ throw new IOException("Incompatible event log version: " + version);
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ac87df5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/EventWriter.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/EventWriter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/EventWriter.java
index a548dfe..29489a5 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/EventWriter.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/EventWriter.java
@@ -43,20 +43,37 @@ import org.apache.hadoop.mapreduce.Counters;
*/
class EventWriter {
static final String VERSION = "Avro-Json";
+ static final String VERSION_BINARY = "Avro-Binary";
private FSDataOutputStream out;
private DatumWriter<Event> writer =
new SpecificDatumWriter<Event>(Event.class);
private Encoder encoder;
private static final Log LOG = LogFactory.getLog(EventWriter.class);
-
- EventWriter(FSDataOutputStream out) throws IOException {
+ public enum WriteMode { JSON, BINARY }
+ private final WriteMode writeMode;
+ private final boolean jsonOutput; // Cache value while we have 2 modes
+
+ EventWriter(FSDataOutputStream out, WriteMode mode) throws IOException {
this.out = out;
- out.writeBytes(VERSION);
+ this.writeMode = mode;
+ if (this.writeMode==WriteMode.JSON) {
+ this.jsonOutput = true;
+ out.writeBytes(VERSION);
+ } else if (this.writeMode==WriteMode.BINARY) {
+ this.jsonOutput = false;
+ out.writeBytes(VERSION_BINARY);
+ } else {
+ throw new IOException("Unknown mode: " + mode);
+ }
out.writeBytes("\n");
out.writeBytes(Event.SCHEMA$.toString());
out.writeBytes("\n");
- this.encoder = EncoderFactory.get().jsonEncoder(Event.SCHEMA$, out);
+ if (!this.jsonOutput) {
+ this.encoder = EncoderFactory.get().binaryEncoder(out, null);
+ } else {
+ this.encoder = EncoderFactory.get().jsonEncoder(Event.SCHEMA$, out);
+ }
}
synchronized void write(HistoryEvent event) throws IOException {
@@ -65,7 +82,9 @@ class EventWriter {
wrapper.setEvent(event.getDatum());
writer.write(wrapper, encoder);
encoder.flush();
- out.writeBytes("\n");
+ if (this.jsonOutput) {
+ out.writeBytes("\n");
+ }
}
void flush() throws IOException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2ac87df5/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
index ba63c02..ddcd2df 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/resources/mapred-default.xml
@@ -1714,6 +1714,15 @@
</property>
<property>
+ <description>
+ File format the AM will use when generating the .jhist file. Valid
+ values are "json" for text output and "binary" for faster parsing.
+ </description>
+ <name>mapreduce.jobhistory.jhist.format</name>
+ <value>json</value>
+</property>
+
+<property>
<name>mapreduce.job.heap.memory-mb.ratio</name>
<value>0.8</value>
<description>The ratio of heap-size to container-size. If no -Xmx is
[32/46] hadoop git commit: MAPREDUCE-6407. Migrate MAPREDUCE native
build to new CMake framework (Alan Burlison via Colin P. McCabe)
Posted by ar...@apache.org.
MAPREDUCE-6407. Migrate MAPREDUCE native build to new CMake framework (Alan Burlison via Colin P. McCabe)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9c63825f
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9c63825f
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9c63825f
Branch: refs/heads/HDFS-7240
Commit: 9c63825f10363a68e7eb34005fa4f4b7ddb27a03
Parents: 29df73d
Author: Colin Patrick Mccabe <cm...@cloudera.com>
Authored: Tue Jun 30 16:06:46 2015 -0700
Committer: Colin Patrick Mccabe <cm...@cloudera.com>
Committed: Tue Jun 30 16:17:06 2015 -0700
----------------------------------------------------------------------
hadoop-mapreduce-project/CHANGES.txt | 3 +
.../src/CMakeLists.txt | 337 ++++++++-----------
.../src/JNIFlags.cmake | 118 -------
3 files changed, 137 insertions(+), 321 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9c63825f/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index 5a4d826..64b04cc 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -115,6 +115,9 @@ Trunk (Unreleased)
MAPREDUCE-2632. Avoid calling the partitioner when the numReduceTasks is 1.
(Ravi Teja Ch N V and Sunil G via kasha)
+ MAPREDUCE-6407. Migrate MAPREDUCE nativetask build to new CMake framework
+ (Alan Burlison via Colin P. McCabe)
+
BUG FIXES
MAPREDUCE-6191. Improve clearing stale state of Java serialization
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9c63825f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt
index 80036e8..f878a94 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/CMakeLists.txt
@@ -18,236 +18,170 @@
cmake_minimum_required(VERSION 2.6 FATAL_ERROR)
-# Default to release builds
-set(CMAKE_BUILD_TYPE, Release)
-
-include(JNIFlags.cmake NO_POLICY_SCOPE)
-
-# Compile a library with both shared and static variants
-function(add_dual_library LIBNAME)
- add_library(${LIBNAME} SHARED ${ARGN})
- add_library(${LIBNAME}_static STATIC ${ARGN})
- set_target_properties(${LIBNAME}_static PROPERTIES OUTPUT_NAME ${LIBNAME})
-endfunction(add_dual_library)
-
-# Link both a static and a dynamic target against some libraries
-function(target_link_dual_libraries LIBNAME)
- target_link_libraries(${LIBNAME} ${ARGN})
- target_link_libraries(${LIBNAME}_static ${ARGN})
-endfunction(target_link_dual_libraries)
-
-function(output_directory TGT DIR)
- SET_TARGET_PROPERTIES(${TGT} PROPERTIES
- RUNTIME_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${DIR}")
- SET_TARGET_PROPERTIES(${TGT} PROPERTIES
- ARCHIVE_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${DIR}")
- SET_TARGET_PROPERTIES(${TGT} PROPERTIES
- LIBRARY_OUTPUT_DIRECTORY "${CMAKE_BINARY_DIR}/${DIR}")
-endfunction(output_directory TGT DIR)
-
-function(dual_output_directory TGT DIR)
- output_directory(${TGT} "${DIR}")
- output_directory(${TGT}_static "${DIR}")
-endfunction(dual_output_directory TGT DIR)
-
-#
-# This macro alters the behavior of find_package and find_library.
-# It does this by setting the CMAKE_FIND_LIBRARY_SUFFIXES global variable.
-# You should save that variable before calling this function and restore it
-# after you have accomplished your goal.
-#
-# The behavior is altered in two ways:
-# 1. We always find shared libraries, never static;
-# 2. We find shared libraries with the given version number.
-#
-# On Windows this function is a no-op. Windows does not encode
-# version number information information into library path names.
-#
-macro(set_find_shared_library_version LVERS)
- IF(${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
- # Mac OS uses .dylib
- SET(CMAKE_FIND_LIBRARY_SUFFIXES ".${LVERS}.dylib")
- ELSEIF(${CMAKE_SYSTEM_NAME} MATCHES "FreeBSD")
- # FreeBSD has always .so installed.
- SET(CMAKE_FIND_LIBRARY_SUFFIXES ".so")
- ELSEIF(${CMAKE_SYSTEM_NAME} MATCHES "Windows")
- # Windows doesn't support finding shared libraries by version.
- ELSE()
- # Most UNIX variants use .so
- SET(CMAKE_FIND_LIBRARY_SUFFIXES ".so.${LVERS}")
- ENDIF()
-endmacro(set_find_shared_library_version LVERS)
-
-if (NOT GENERATED_JAVAH)
- #Must identify where the generated headers have been placed
- MESSAGE(FATAL_ERROR "You must set the cmake variable GENERATED_JAVAH")
-endif (NOT GENERATED_JAVAH)
-find_package(JNI REQUIRED)
-
-SET(STORED_CMAKE_FIND_LIBRARY_SUFFIXES CMAKE_FIND_LIBRARY_SUFFIXES)
-set_find_shared_library_version("1")
-SET(CMAKE_FIND_LIBRARY_SUFFIXES STORED_CMAKE_FIND_LIBRARY_SUFFIXES)
-
-# primitive configs
-set(PRFLAGS "-DSIMPLE_MEMCPY")
-set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} ${PRFLAGS} -fno-strict-aliasing -Wall -Wno-sign-compare")
-set(CMAKE_LD_FLAGS "${CMAKE_LD_FLAGS} -no-undefined -version-info 0:1:0
- -L${_JAVA_HOME}/jre/lib/amd64/server -ljvm")
-set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} ${CMAKE_C_FLAGS} -g -O2 -DNDEBUG -fPIC")
-set(D main/native/)
-
-SET(STORED_CMAKE_FIND_LIBRARY_SUFFIXES CMAKE_FIND_LIBRARY_SUFFIXES)
-set_find_shared_library_version("1")
-SET(CMAKE_FIND_LIBRARY_SUFFIXES STORED_CMAKE_FIND_LIBRARY_SUFFIXES)
-
-INCLUDE(CheckFunctionExists)
-INCLUDE(CheckCSourceCompiles)
-#INCLUDE(CheckLibraryExists)
-INCLUDE(CheckIncludeFiles)
-#CHECK_FUNCTION_EXISTS(sync_file_range HAVE_SYNC_FILE_RANGE)
-#CHECK_FUNCTION_EXISTS(posix_fadvise HAVE_POSIX_FADVISE)
-#CHECK_LIBRARY_EXISTS(dl dlopen "" NEED_LINK_DL)
-CHECK_INCLUDE_FILES(fcntl.h HAVE_FCNTL_H)
-CHECK_INCLUDE_FILES(malloc.h HAVE_MALLOC_H)
-CHECK_INCLUDE_FILES(mach/mach.h HAVE_MACH_MACH_H)
-CHECK_INCLUDE_FILES(memory.h HAVE_MEMORY_H)
-CHECK_INCLUDE_FILES(stddef.h HAVE_STDDEF_H)
-CHECK_INCLUDE_FILES(stdint.h HAVE_STDINT_H)
-CHECK_INCLUDE_FILES(stdlib.h HAVE_STDLIB_H)
-CHECK_INCLUDE_FILES(string.h HAVE_STRING_H)
-CHECK_INCLUDE_FILES(unistd.h HAVE_UNITSTD_H)
-CHECK_FUNCTION_EXISTS(clock_gettime HAVE_CLOCK_GETTIME)
-CHECK_FUNCTION_EXISTS(localtime_r HAVE_LOCALTIME_R)
-CHECK_FUNCTION_EXISTS(memset HAVE_MEMSET)
-CHECK_FUNCTION_EXISTS(strchr HAVE_STRCHR)
-CHECK_FUNCTION_EXISTS(strtoul HAVE_STRTOUL)
-
-SET(STORED_CMAKE_FIND_LIBRARY_SUFFIXES CMAKE_FIND_LIBRARY_SUFFIXES)
-set_find_shared_library_version("1")
+list(APPEND CMAKE_MODULE_PATH ${CMAKE_SOURCE_DIR}/../../../../hadoop-common-project/hadoop-common/)
+include(HadoopCommon)
+
+# Add extra compiler and linker flags.
+# -Wno-sign-compare
+hadoop_add_compiler_flags("-DNDEBUG -DSIMPLE_MEMCPY -fno-strict-aliasing")
+
+# Source location.
+set(SRC main/native)
+
+# The caller must specify where the generated headers have been placed.
+if(NOT GENERATED_JAVAH)
+ message(FATAL_ERROR "You must set the CMake variable GENERATED_JAVAH")
+endif()
+
+# Configure JNI.
+include(HadoopJNI)
+
+# Probe for headers and functions.
+include(CheckFunctionExists)
+include(CheckIncludeFiles)
+
+check_include_files(fcntl.h HAVE_FCNTL_H)
+check_include_files(malloc.h HAVE_MALLOC_H)
+check_include_files(mach/mach.h HAVE_MACH_MACH_H)
+check_include_files(memory.h HAVE_MEMORY_H)
+check_include_files(stddef.h HAVE_STDDEF_H)
+check_include_files(stdint.h HAVE_STDINT_H)
+check_include_files(stdlib.h HAVE_STDLIB_H)
+Check_include_files(string.h HAVE_STRING_H)
+check_include_files(unistd.h HAVE_UNITSTD_H)
+check_function_exists(clock_gettime HAVE_CLOCK_GETTIME)
+check_function_exists(localtime_r HAVE_LOCALTIME_R)
+check_function_exists(memset HAVE_MEMSET)
+check_function_exists(strchr HAVE_STRCHR)
+check_function_exists(strtoul HAVE_STRTOUL)
+
+# Require snappy.
+set(STORED_CMAKE_FIND_LIBRARY_SUFFIXES CMAKE_FIND_LIBRARY_SUFFIXES)
+hadoop_set_find_shared_library_version("1")
find_library(SNAPPY_LIBRARY
NAMES snappy
PATHS ${CUSTOM_SNAPPY_PREFIX} ${CUSTOM_SNAPPY_PREFIX}/lib
${CUSTOM_SNAPPY_PREFIX}/lib64 ${CUSTOM_SNAPPY_LIB})
-SET(CMAKE_FIND_LIBRARY_SUFFIXES STORED_CMAKE_FIND_LIBRARY_SUFFIXES)
+set(CMAKE_FIND_LIBRARY_SUFFIXES STORED_CMAKE_FIND_LIBRARY_SUFFIXES)
find_path(SNAPPY_INCLUDE_DIR
NAMES snappy.h
PATHS ${CUSTOM_SNAPPY_PREFIX} ${CUSTOM_SNAPPY_PREFIX}/include
${CUSTOM_SNAPPY_INCLUDE})
-if (SNAPPY_LIBRARY AND SNAPPY_INCLUDE_DIR)
+if(SNAPPY_LIBRARY AND SNAPPY_INCLUDE_DIR)
GET_FILENAME_COMPONENT(HADOOP_SNAPPY_LIBRARY ${SNAPPY_LIBRARY} NAME)
set(SNAPPY_SOURCE_FILES
- "${D}/src/codec/SnappyCodec.cc")
-else (SNAPPY_LIBRARY AND SNAPPY_INCLUDE_DIR)
+ "${SRC}/src/codec/SnappyCodec.cc")
+ set(REQUIRE_SNAPPY ${REQUIRE_SNAPPY}) # Stop warning about unused variable.
+ message(STATUS "Found Snappy: ${SNAPPY_LIBRARY}")
+else()
set(SNAPPY_LIBRARY "")
set(SNAPPY_INCLUDE_DIR "")
set(SNAPPY_SOURCE_FILES "")
- IF(REQUIRE_SNAPPY)
- MESSAGE(FATAL_ERROR "Required snappy library could not be found. SNAPPY_LIBRARY=${SNAPPY_LIBRARY}, SNAPPY_INCLUDE_DIR=${SNAPPY_INCLUDE_DIR}, CUSTOM_SNAPPY_INCLUDE_DIR=${CUSTOM_SNAPPY_INCLUDE_DIR}, CUSTOM_SNAPPY_PREFIX=${CUSTOM_SNAPPY_PREFIX}, CUSTOM_SNAPPY_INCLUDE=${CUSTOM_SNAPPY_INCLUDE}")
- ENDIF(REQUIRE_SNAPPY)
-endif (SNAPPY_LIBRARY AND SNAPPY_INCLUDE_DIR)
+ if(REQUIRE_SNAPPY)
+ message(FATAL_ERROR "Required snappy library could not be found. SNAPPY_LIBRARY=${SNAPPY_LIBRARY}, SNAPPY_INCLUDE_DIR=${SNAPPY_INCLUDE_DIR}, CUSTOM_SNAPPY_INCLUDE_DIR=${CUSTOM_SNAPPY_INCLUDE_DIR}, CUSTOM_SNAPPY_PREFIX=${CUSTOM_SNAPPY_PREFIX}, CUSTOM_SNAPPY_INCLUDE=${CUSTOM_SNAPPY_INCLUDE}")
+ endif()
+endif()
-CONFIGURE_FILE(${CMAKE_SOURCE_DIR}/config.h.cmake ${CMAKE_BINARY_DIR}/config.h)
+configure_file(${CMAKE_SOURCE_DIR}/config.h.cmake ${CMAKE_BINARY_DIR}/config.h)
include_directories(
${GENERATED_JAVAH}
- ${D}/src
- ${D}/src/util
- ${D}/src/lib
- ${D}/test
+ ${SRC}/src
+ ${SRC}/src/util
+ ${SRC}/src/lib
+ ${SRC}/test
${CMAKE_CURRENT_SOURCE_DIR}
${CMAKE_BINARY_DIR}
${JNI_INCLUDE_DIRS}
${SNAPPY_INCLUDE_DIR}
)
# add gtest as system library to suppress gcc warnings
-include_directories(SYSTEM ${D}/gtest/include)
-
+include_directories(SYSTEM ${SRC}/gtest/include)
-SET(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE)
+set(CMAKE_BUILD_WITH_INSTALL_RPATH TRUE)
-if (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
+if(CMAKE_SYSTEM_NAME MATCHES "Darwin")
# macosx does not have -lrt
set(NT_DEPEND_LIBRARY dl pthread z ${SNAPPY_LIBRARY} ${JAVA_JVM_LIBRARY})
set(SYSTEM_MAC TRUE)
-else (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
+else()
set(NT_DEPEND_LIBRARY dl rt pthread z ${SNAPPY_LIBRARY} ${JAVA_JVM_LIBRARY})
set(SYSTEM_MAC FALSE)
-endif (${CMAKE_SYSTEM_NAME} MATCHES "Darwin")
+endif()
configure_file(main/native/test.sh test/test.sh)
-add_dual_library(nativetask
+hadoop_add_dual_library(nativetask
${CMAKE_BINARY_DIR}/lz4.c
- ${D}/src/codec/BlockCodec.cc
- ${D}/src/codec/GzipCodec.cc
- ${D}/src/codec/Lz4Codec.cc
+ ${SRC}/src/codec/BlockCodec.cc
+ ${SRC}/src/codec/GzipCodec.cc
+ ${SRC}/src/codec/Lz4Codec.cc
${SNAPPY_SOURCE_FILES}
- ${D}/src/handler/BatchHandler.cc
- ${D}/src/handler/MCollectorOutputHandler.cc
- ${D}/src/handler/AbstractMapHandler.cc
- ${D}/src/handler/CombineHandler.cc
- ${D}/src/lib/Buffers.cc
- ${D}/src/lib/BufferStream.cc
- ${D}/src/lib/Compressions.cc
- ${D}/src/lib/PartitionBucket.cc
- ${D}/src/lib/PartitionBucketIterator.cc
- ${D}/src/lib/FileSystem.cc
- ${D}/src/lib/IFile.cc
- ${D}/src/lib/jniutils.cc
- ${D}/src/lib/Log.cc
- ${D}/src/lib/MapOutputCollector.cc
- ${D}/src/lib/MapOutputSpec.cc
- ${D}/src/lib/MemoryBlock.cc
- ${D}/src/lib/Merge.cc
- ${D}/src/lib/NativeLibrary.cc
- ${D}/src/lib/Iterator.cc
- ${D}/src/lib/NativeObjectFactory.cc
- ${D}/src/lib/NativeRuntimeJniImpl.cc
- ${D}/src/lib/NativeTask.cc
- ${D}/src/lib/SpillInfo.cc
- ${D}/src/lib/Path.cc
- ${D}/src/lib/Streams.cc
- ${D}/src/lib/TaskCounters.cc
- ${D}/src/util/Checksum.cc
- ${D}/src/util/Random.cc
- ${D}/src/util/StringUtil.cc
- ${D}/src/util/SyncUtils.cc
- ${D}/src/util/Timer.cc
- ${D}/src/util/WritableUtils.cc
+ ${SRC}/src/handler/BatchHandler.cc
+ ${SRC}/src/handler/MCollectorOutputHandler.cc
+ ${SRC}/src/handler/AbstractMapHandler.cc
+ ${SRC}/src/handler/CombineHandler.cc
+ ${SRC}/src/lib/Buffers.cc
+ ${SRC}/src/lib/BufferStream.cc
+ ${SRC}/src/lib/Compressions.cc
+ ${SRC}/src/lib/PartitionBucket.cc
+ ${SRC}/src/lib/PartitionBucketIterator.cc
+ ${SRC}/src/lib/FileSystem.cc
+ ${SRC}/src/lib/IFile.cc
+ ${SRC}/src/lib/jniutils.cc
+ ${SRC}/src/lib/Log.cc
+ ${SRC}/src/lib/MapOutputCollector.cc
+ ${SRC}/src/lib/MapOutputSpec.cc
+ ${SRC}/src/lib/MemoryBlock.cc
+ ${SRC}/src/lib/Merge.cc
+ ${SRC}/src/lib/NativeLibrary.cc
+ ${SRC}/src/lib/Iterator.cc
+ ${SRC}/src/lib/NativeObjectFactory.cc
+ ${SRC}/src/lib/NativeRuntimeJniImpl.cc
+ ${SRC}/src/lib/NativeTask.cc
+ ${SRC}/src/lib/SpillInfo.cc
+ ${SRC}/src/lib/Path.cc
+ ${SRC}/src/lib/Streams.cc
+ ${SRC}/src/lib/TaskCounters.cc
+ ${SRC}/src/util/Checksum.cc
+ ${SRC}/src/util/Random.cc
+ ${SRC}/src/util/StringUtil.cc
+ ${SRC}/src/util/SyncUtils.cc
+ ${SRC}/src/util/Timer.cc
+ ${SRC}/src/util/WritableUtils.cc
)
target_link_libraries(nativetask ${NT_DEPEND_LIBRARY})
-add_library(gtest ${D}/gtest/gtest-all.cc)
+add_library(gtest ${SRC}/gtest/gtest-all.cc)
set_target_properties(gtest PROPERTIES COMPILE_FLAGS "-w")
add_executable(nttest
- ${D}/test/lib/TestByteArray.cc
- ${D}/test/lib/TestByteBuffer.cc
- ${D}/test/lib/TestComparatorForDualPivotQuickSort.cc
- ${D}/test/lib/TestComparatorForStdSort.cc
- ${D}/test/lib/TestFixSizeContainer.cc
- ${D}/test/lib/TestMemoryPool.cc
- ${D}/test/lib/TestIterator.cc
- ${D}/test/lib/TestKVBuffer.cc
- ${D}/test/lib/TestMemBlockIterator.cc
- ${D}/test/lib/TestMemoryBlock.cc
- ${D}/test/lib/TestPartitionBucket.cc
- ${D}/test/lib/TestReadBuffer.cc
- ${D}/test/lib/TestReadWriteBuffer.cc
- ${D}/test/util/TestChecksum.cc
- ${D}/test/util/TestStringUtil.cc
- ${D}/test/util/TestWritableUtils.cc
- ${D}/test/TestCommand.cc
- ${D}/test/TestConfig.cc
- ${D}/test/TestCounter.cc
- ${D}/test/TestCompressions.cc
- ${D}/test/TestFileSystem.cc
- ${D}/test/TestIFile.cc
- ${D}/test/TestPrimitives.cc
- ${D}/test/TestSort.cc
- ${D}/test/TestMain.cc
- ${D}/test/test_commons.cc)
-
+ ${SRC}/test/lib/TestByteArray.cc
+ ${SRC}/test/lib/TestByteBuffer.cc
+ ${SRC}/test/lib/TestComparatorForDualPivotQuickSort.cc
+ ${SRC}/test/lib/TestComparatorForStdSort.cc
+ ${SRC}/test/lib/TestFixSizeContainer.cc
+ ${SRC}/test/lib/TestMemoryPool.cc
+ ${SRC}/test/lib/TestIterator.cc
+ ${SRC}/test/lib/TestKVBuffer.cc
+ ${SRC}/test/lib/TestMemBlockIterator.cc
+ ${SRC}/test/lib/TestMemoryBlock.cc
+ ${SRC}/test/lib/TestPartitionBucket.cc
+ ${SRC}/test/lib/TestReadBuffer.cc
+ ${SRC}/test/lib/TestReadWriteBuffer.cc
+ ${SRC}/test/util/TestChecksum.cc
+ ${SRC}/test/util/TestStringUtil.cc
+ ${SRC}/test/util/TestWritableUtils.cc
+ ${SRC}/test/TestCommand.cc
+ ${SRC}/test/TestConfig.cc
+ ${SRC}/test/TestCounter.cc
+ ${SRC}/test/TestCompressions.cc
+ ${SRC}/test/TestFileSystem.cc
+ ${SRC}/test/TestIFile.cc
+ ${SRC}/test/TestPrimitives.cc
+ ${SRC}/test/TestSort.cc
+ ${SRC}/test/TestMain.cc
+ ${SRC}/test/test_commons.cc)
target_link_libraries(nttest
nativetask_static
@@ -255,17 +189,14 @@ target_link_libraries(nttest
${NT_DEPEND_LIBRARY}
)
-IF (${CMAKE_SYSTEM_NAME} MATCHES "Linux")
- #
- # By embedding '$ORIGIN' into the RPATH of libnativetask.so,
- # dlopen will look in the directory containing libnativetask.so.
- # However, $ORIGIN is not supported by all operating systems.
- #
- SET_TARGET_PROPERTIES(nativetask
- PROPERTIES INSTALL_RPATH "\$ORIGIN/")
-ENDIF()
-
-SET(LIBNATIVETASK_VERSION "1.0.0")
-SET_TARGET_PROPERTIES(nativetask PROPERTIES SOVERSION ${LIBNATIVETASK_VERSION})
-dual_output_directory(nativetask target/usr/local/lib)
-output_directory(nttest test)
+# By embedding '$ORIGIN' into the RPATH of libnativetask.so, dlopen will look in
+# the directory containing libnativetask.so. However, $ORIGIN is not supported by
+# all operating systems.
+if(CMAKE_SYSTEM_NAME MATCHES "Linux|SunOS")
+ set_target_properties(nativetask PROPERTIES INSTALL_RPATH "\$ORIGIN/")
+endif()
+
+set(LIBNATIVETASK_VERSION "1.0.0")
+set_target_properties(nativetask PROPERTIES SOVERSION ${LIBNATIVETASK_VERSION})
+hadoop_dual_output_directory(nativetask target/usr/local/lib)
+hadoop_output_directory(nttest test)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9c63825f/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/JNIFlags.cmake
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/JNIFlags.cmake b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/JNIFlags.cmake
deleted file mode 100644
index 0100b06..0000000
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-nativetask/src/JNIFlags.cmake
+++ /dev/null
@@ -1,118 +0,0 @@
-#
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-cmake_minimum_required(VERSION 2.6 FATAL_ERROR)
-
-# If JVM_ARCH_DATA_MODEL is 32, compile all binaries as 32-bit.
-# This variable is set by maven.
-if (JVM_ARCH_DATA_MODEL EQUAL 32)
- # Force 32-bit code generation on amd64/x86_64, ppc64, sparc64
- if (CMAKE_COMPILER_IS_GNUCC AND CMAKE_SYSTEM_PROCESSOR MATCHES ".*64")
- set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -m32")
- set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -m32")
- set(CMAKE_LD_FLAGS "${CMAKE_LD_FLAGS} -m32")
- endif ()
- if (CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" OR CMAKE_SYSTEM_PROCESSOR STREQUAL "amd64")
- # Set CMAKE_SYSTEM_PROCESSOR to ensure that find_package(JNI) will use
- # the 32-bit version of libjvm.so.
- set(CMAKE_SYSTEM_PROCESSOR "i686")
- endif ()
-endif (JVM_ARCH_DATA_MODEL EQUAL 32)
-
-# Determine float ABI of JVM on ARM Linux
-if (CMAKE_SYSTEM_PROCESSOR MATCHES "^arm" AND CMAKE_SYSTEM_NAME STREQUAL "Linux")
- find_program(READELF readelf)
- if (READELF MATCHES "NOTFOUND")
- message(WARNING "readelf not found; JVM float ABI detection disabled")
- else (READELF MATCHES "NOTFOUND")
- execute_process(
- COMMAND ${READELF} -A ${JAVA_JVM_LIBRARY}
- OUTPUT_VARIABLE JVM_ELF_ARCH
- ERROR_QUIET)
- if (NOT JVM_ELF_ARCH MATCHES "Tag_ABI_VFP_args: VFP registers")
- message("Soft-float JVM detected")
-
- # Test compilation with -mfloat-abi=softfp using an arbitrary libc function
- # (typically fails with "fatal error: bits/predefs.h: No such file or directory"
- # if soft-float dev libraries are not installed)
- include(CMakePushCheckState)
- cmake_push_check_state()
- set(CMAKE_REQUIRED_FLAGS "${CMAKE_REQUIRED_FLAGS} -mfloat-abi=softfp")
- include(CheckSymbolExists)
- check_symbol_exists(exit stdlib.h SOFTFP_AVAILABLE)
- if (NOT SOFTFP_AVAILABLE)
- message(FATAL_ERROR "Soft-float dev libraries required (e.g. 'apt-get install libc6-dev-armel' on Debian/Ubuntu)")
- endif (NOT SOFTFP_AVAILABLE)
- cmake_pop_check_state()
-
- set(CMAKE_C_FLAGS "${CMAKE_C_FLAGS} -mfloat-abi=softfp")
- endif ()
- endif (READELF MATCHES "NOTFOUND")
-endif (CMAKE_SYSTEM_PROCESSOR MATCHES "^arm" AND CMAKE_SYSTEM_NAME STREQUAL "Linux")
-
-IF("${CMAKE_SYSTEM}" MATCHES "Linux")
- #
- # Locate JNI_INCLUDE_DIRS and JNI_LIBRARIES.
- # Since we were invoked from Maven, we know that the JAVA_HOME environment
- # variable is valid. So we ignore system paths here and just use JAVA_HOME.
- #
- FILE(TO_CMAKE_PATH "$ENV{JAVA_HOME}" _JAVA_HOME)
- IF(CMAKE_SYSTEM_PROCESSOR MATCHES "^i.86$")
- SET(_java_libarch "i386")
- ELSEIF (CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" OR CMAKE_SYSTEM_PROCESSOR STREQUAL "amd64")
- SET(_java_libarch "amd64")
- ELSEIF (CMAKE_SYSTEM_PROCESSOR MATCHES "^arm")
- SET(_java_libarch "arm")
- ELSE()
- SET(_java_libarch ${CMAKE_SYSTEM_PROCESSOR})
- ENDIF()
- SET(_JDK_DIRS "${_JAVA_HOME}/jre/lib/${_java_libarch}/*"
- "${_JAVA_HOME}/jre/lib/${_java_libarch}"
- "${_JAVA_HOME}/jre/lib/*"
- "${_JAVA_HOME}/jre/lib"
- "${_JAVA_HOME}/lib/*"
- "${_JAVA_HOME}/lib"
- "${_JAVA_HOME}/include/*"
- "${_JAVA_HOME}/include"
- "${_JAVA_HOME}"
- )
- FIND_PATH(JAVA_INCLUDE_PATH
- NAMES jni.h
- PATHS ${_JDK_DIRS}
- NO_DEFAULT_PATH)
- #In IBM java, it's jniport.h instead of jni_md.h
- FIND_PATH(JAVA_INCLUDE_PATH2
- NAMES jni_md.h jniport.h
- PATHS ${_JDK_DIRS}
- NO_DEFAULT_PATH)
- SET(JNI_INCLUDE_DIRS ${JAVA_INCLUDE_PATH} ${JAVA_INCLUDE_PATH2})
- FIND_LIBRARY(JAVA_JVM_LIBRARY
- NAMES rt jvm
- PATHS ${_JDK_DIRS}
- NO_DEFAULT_PATH)
- SET(JNI_LIBRARIES ${JAVA_JVM_LIBRARY})
- MESSAGE("JAVA_HOME=${JAVA_HOME}, JAVA_JVM_LIBRARY=${JAVA_JVM_LIBRARY}")
- MESSAGE("JAVA_INCLUDE_PATH=${JAVA_INCLUDE_PATH}, JAVA_INCLUDE_PATH2=${JAVA_INCLUDE_PATH2}")
- IF(JAVA_JVM_LIBRARY AND JAVA_INCLUDE_PATH AND JAVA_INCLUDE_PATH2)
- MESSAGE("Located all JNI components successfully.")
- ELSE()
- MESSAGE(FATAL_ERROR "Failed to find a viable JVM installation under JAVA_HOME.")
- ENDIF()
-ELSE()
- find_package(JNI REQUIRED)
-ENDIF()
[36/46] hadoop git commit: MAPREDUCE-6121.
JobResourceUpdater#compareFs() doesn't handle HA namespaces (rchiang via
rkanter)
Posted by ar...@apache.org.
MAPREDUCE-6121. JobResourceUpdater#compareFs() doesn't handle HA namespaces (rchiang via rkanter)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3a72bfd0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3a72bfd0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3a72bfd0
Branch: refs/heads/HDFS-7240
Commit: 3a72bfd08281fd271bef4f41289125d39c41928c
Parents: aaafa0b
Author: Robert Kanter <rk...@apache.org>
Authored: Tue Jun 30 16:46:41 2015 -0700
Committer: Robert Kanter <rk...@apache.org>
Committed: Tue Jun 30 16:49:27 2015 -0700
----------------------------------------------------------------------
hadoop-mapreduce-project/CHANGES.txt | 3 ++
.../hadoop/mapreduce/JobResourceUploader.java | 38 +-------------------
2 files changed, 4 insertions(+), 37 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a72bfd0/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index 64b04cc..0baecf8 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -518,6 +518,9 @@ Release 2.8.0 - UNRELEASED
MAPREDUCE-6413. TestLocalJobSubmission is failing with unknown host
(zhihai xu via jlowe)
+ MAPREDUCE-6121. JobResourceUpdater#compareFs() doesn't handle HA namespaces
+ (rchiang via rkanter)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3a72bfd0/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobResourceUploader.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobResourceUploader.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobResourceUploader.java
index 134de35..f3e4d2f 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobResourceUploader.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobResourceUploader.java
@@ -194,7 +194,7 @@ class JobResourceUploader {
FileSystem remoteFs = null;
remoteFs = originalPath.getFileSystem(conf);
- if (compareFs(remoteFs, jtFs)) {
+ if (FileUtil.compareFs(remoteFs, jtFs)) {
return originalPath;
}
// this might have name collisions. copy will throw an exception
@@ -205,42 +205,6 @@ class JobResourceUploader {
return newPath;
}
- /*
- * see if two file systems are the same or not.
- */
- private boolean compareFs(FileSystem srcFs, FileSystem destFs) {
- URI srcUri = srcFs.getUri();
- URI dstUri = destFs.getUri();
- if (srcUri.getScheme() == null) {
- return false;
- }
- if (!srcUri.getScheme().equals(dstUri.getScheme())) {
- return false;
- }
- String srcHost = srcUri.getHost();
- String dstHost = dstUri.getHost();
- if ((srcHost != null) && (dstHost != null)) {
- try {
- srcHost = InetAddress.getByName(srcHost).getCanonicalHostName();
- dstHost = InetAddress.getByName(dstHost).getCanonicalHostName();
- } catch (UnknownHostException ue) {
- return false;
- }
- if (!srcHost.equals(dstHost)) {
- return false;
- }
- } else if (srcHost == null && dstHost != null) {
- return false;
- } else if (srcHost != null && dstHost == null) {
- return false;
- }
- // check for ports
- if (srcUri.getPort() != dstUri.getPort()) {
- return false;
- }
- return true;
- }
-
private void copyJar(Path originalJarPath, Path submitJarFile,
short replication) throws IOException {
jtFs.copyFromLocalFile(originalJarPath, submitJarFile);
[17/46] hadoop git commit: YARN-3770. SerializedException should also
handle java.lang.Error on de-serialization. Contributed by Lavkesh Lahngir
Posted by ar...@apache.org.
YARN-3770. SerializedException should also handle java.lang.Error on de-serialization. Contributed by Lavkesh Lahngir
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4672315e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4672315e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4672315e
Branch: refs/heads/HDFS-7240
Commit: 4672315e2d6abe1cee0210cf7d3e8ab114ba933c
Parents: 460e98f
Author: Jian He <ji...@apache.org>
Authored: Mon Jun 29 14:31:32 2015 -0700
Committer: Jian He <ji...@apache.org>
Committed: Mon Jun 29 14:31:49 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +++
.../api/records/impl/pb/SerializedExceptionPBImpl.java | 2 +-
.../records/impl/pb/TestSerializedExceptionPBImpl.java | 12 ++++++++++--
3 files changed, 14 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4672315e/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 2fdcc9d..8461f69 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -571,6 +571,9 @@ Release 2.8.0 - UNRELEASED
YARN-3695. ServerProxy (NMProxy, etc.) shouldn't retry forever for non
network exception. (Raju Bairishetti via jianhe)
+ YARN-3770. SerializedException should also handle java.lang.Error on
+ de-serialization. (Lavkesh Lahngir via jianhe)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4672315e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/SerializedExceptionPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/SerializedExceptionPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/SerializedExceptionPBImpl.java
index fd9e170..2ec232e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/SerializedExceptionPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/SerializedExceptionPBImpl.java
@@ -101,7 +101,7 @@ public class SerializedExceptionPBImpl extends SerializedException {
} else if (RuntimeException.class.isAssignableFrom(realClass)) {
classType = RuntimeException.class;
} else {
- classType = Exception.class;
+ classType = Throwable.class;
}
return instantiateException(realClass.asSubclass(classType), getMessage(),
cause == null ? null : cause.deSerialize());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4672315e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/impl/pb/TestSerializedExceptionPBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/impl/pb/TestSerializedExceptionPBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/impl/pb/TestSerializedExceptionPBImpl.java
index ac7e40e..ecfa63e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/impl/pb/TestSerializedExceptionPBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/test/java/org/apache/hadoop/yarn/api/records/impl/pb/TestSerializedExceptionPBImpl.java
@@ -20,10 +20,9 @@ package org.apache.hadoop.yarn.api.records.impl.pb;
import java.nio.channels.ClosedChannelException;
-import org.junit.Assert;
-import org.apache.hadoop.yarn.api.records.impl.pb.SerializedExceptionPBImpl;
import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.proto.YarnProtos.SerializedExceptionProto;
+import org.junit.Assert;
import org.junit.Test;
public class TestSerializedExceptionPBImpl {
@@ -79,4 +78,13 @@ public class TestSerializedExceptionPBImpl {
SerializedExceptionPBImpl pb3 = new SerializedExceptionPBImpl();
Assert.assertEquals(defaultProto.getTrace(), pb3.getRemoteTrace());
}
+
+ @Test
+ public void testThrowableDeserialization() {
+ // java.lang.Error should also be serializable
+ Error ex = new Error();
+ SerializedExceptionPBImpl pb = new SerializedExceptionPBImpl();
+ pb.init(ex);
+ Assert.assertEquals(ex.getClass(), pb.deSerialize().getClass());
+ }
}
[44/46] hadoop git commit: MAPREDUCE-6420. Interrupted Exception in
LocalContainerLauncher should be logged in warn/info level. Contributed by
Chang Li
Posted by ar...@apache.org.
MAPREDUCE-6420. Interrupted Exception in LocalContainerLauncher should be logged in warn/info level. Contributed by Chang Li
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/82671562
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/82671562
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/82671562
Branch: refs/heads/HDFS-7240
Commit: 826715622e6937887a4e20b3ce327d7e2fd89009
Parents: ec0dc71
Author: Jason Lowe <jl...@apache.org>
Authored: Wed Jul 1 17:50:02 2015 +0000
Committer: Jason Lowe <jl...@apache.org>
Committed: Wed Jul 1 17:50:02 2015 +0000
----------------------------------------------------------------------
hadoop-mapreduce-project/CHANGES.txt | 3 +++
.../java/org/apache/hadoop/mapred/LocalContainerLauncher.java | 2 +-
2 files changed, 4 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82671562/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index 409f074..dd9da7e 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -524,6 +524,9 @@ Release 2.8.0 - UNRELEASED
MAPREDUCE-6121. JobResourceUpdater#compareFs() doesn't handle HA namespaces
(rchiang via rkanter)
+ MAPREDUCE-6420. Interrupted Exception in LocalContainerLauncher should be
+ logged in warn/info level (Chang Li via jlowe)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/82671562/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java
index 9d8b4a5..1a0d5fb 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapred/LocalContainerLauncher.java
@@ -233,7 +233,7 @@ public class LocalContainerLauncher extends AbstractService implements
try {
event = eventQueue.take();
} catch (InterruptedException e) { // mostly via T_KILL? JOB_KILL?
- LOG.error("Returning, interrupted : " + e);
+ LOG.warn("Returning, interrupted : " + e);
break;
}
[15/46] hadoop git commit: YARN-3695. ServerProxy (NMProxy,
etc.) shouldn't retry forever for non network exception. Contributed
by Raju Bairishetti
Posted by ar...@apache.org.
YARN-3695. ServerProxy (NMProxy, etc.) shouldn't retry forever for non network exception. Contributed by Raju Bairishetti
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/62e583c7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/62e583c7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/62e583c7
Branch: refs/heads/HDFS-7240
Commit: 62e583c7dcbb30d95d8b32a4978fbdb3b98d67cc
Parents: fad291e
Author: Jian He <ji...@apache.org>
Authored: Mon Jun 29 13:37:32 2015 -0700
Committer: Jian He <ji...@apache.org>
Committed: Mon Jun 29 13:37:32 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../apache/hadoop/yarn/client/ServerProxy.java | 21 ++---
.../containermanager/TestNMProxy.java | 81 +++++++++++++-------
3 files changed, 70 insertions(+), 35 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62e583c7/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 3ebf799..2fdcc9d 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -568,6 +568,9 @@ Release 2.8.0 - UNRELEASED
YARN-3860. rmadmin -transitionToActive should check the state of non-target node.
(Masatake Iwasaki via junping_du)
+ YARN-3695. ServerProxy (NMProxy, etc.) shouldn't retry forever for non
+ network exception. (Raju Bairishetti via jianhe)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62e583c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/ServerProxy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/ServerProxy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/ServerProxy.java
index e9bcf8d..de7fc7d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/ServerProxy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/client/ServerProxy.java
@@ -53,19 +53,22 @@ public class ServerProxy {
long maxWaitTime = conf.getLong(maxWaitTimeStr, defMaxWaitTime);
long retryIntervalMS =
conf.getLong(connectRetryIntervalStr, defRetryInterval);
- if (maxWaitTime == -1) {
- // wait forever.
- return RetryPolicies.RETRY_FOREVER;
- }
- Preconditions.checkArgument(maxWaitTime > 0, "Invalid Configuration. "
- + maxWaitTimeStr + " should be a positive value.");
+ Preconditions.checkArgument((maxWaitTime == -1 || maxWaitTime > 0),
+ "Invalid Configuration. " + maxWaitTimeStr + " should be either"
+ + " positive value or -1.");
Preconditions.checkArgument(retryIntervalMS > 0, "Invalid Configuration. "
+ connectRetryIntervalStr + "should be a positive value.");
- RetryPolicy retryPolicy =
- RetryPolicies.retryUpToMaximumTimeWithFixedSleep(maxWaitTime,
- retryIntervalMS, TimeUnit.MILLISECONDS);
+ RetryPolicy retryPolicy = null;
+ if (maxWaitTime == -1) {
+ // wait forever.
+ retryPolicy = RetryPolicies.RETRY_FOREVER;
+ } else {
+ retryPolicy =
+ RetryPolicies.retryUpToMaximumTimeWithFixedSleep(maxWaitTime,
+ retryIntervalMS, TimeUnit.MILLISECONDS);
+ }
Map<Class<? extends Exception>, RetryPolicy> exceptionToPolicyMap =
new HashMap<Class<? extends Exception>, RetryPolicy>();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/62e583c7/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestNMProxy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestNMProxy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestNMProxy.java
index 0b372be..102c9c6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestNMProxy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestNMProxy.java
@@ -22,6 +22,7 @@ import java.io.IOException;
import java.net.InetSocketAddress;
import org.apache.hadoop.fs.UnsupportedFileSystemException;
+import org.apache.hadoop.io.retry.UnreliableInterface;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.token.Token;
@@ -58,8 +59,8 @@ public class TestNMProxy extends BaseContainerManagerTest {
@Before
public void setUp() throws Exception {
- conf.setLong(YarnConfiguration.CLIENT_NM_CONNECT_MAX_WAIT_MS, 10000);
- conf.setLong(YarnConfiguration.CLIENT_NM_CONNECT_RETRY_INTERVAL_MS, 100);
+ containerManager.start();
+ containerManager.setBlockNewContainerRequests(false);
}
@Override
@@ -77,7 +78,13 @@ public class TestNMProxy extends BaseContainerManagerTest {
// This causes super to throw an NMNotYetReadyException
containerManager.setBlockNewContainerRequests(true);
} else {
- throw new java.net.ConnectException("start container exception");
+ if (isRetryPolicyRetryForEver()) {
+ // Throw non network exception
+ throw new IOException(
+ new UnreliableInterface.UnreliableException());
+ } else {
+ throw new java.net.ConnectException("start container exception");
+ }
}
} else {
// This stops super from throwing an NMNotYetReadyException
@@ -86,6 +93,11 @@ public class TestNMProxy extends BaseContainerManagerTest {
return super.startContainers(requests);
}
+ private boolean isRetryPolicyRetryForEver() {
+ return conf.getLong(
+ YarnConfiguration.CLIENT_NM_CONNECT_MAX_WAIT_MS, 1000) == -1;
+ }
+
@Override
public StopContainersResponse stopContainers(
StopContainersRequest requests) throws YarnException, IOException {
@@ -110,30 +122,13 @@ public class TestNMProxy extends BaseContainerManagerTest {
}
@Test(timeout = 20000)
- public void testNMProxyRetry() throws Exception {
- containerManager.start();
- containerManager.setBlockNewContainerRequests(false);
- StartContainersRequest allRequests =
- Records.newRecord(StartContainersRequest.class);
- ApplicationId appId = ApplicationId.newInstance(1, 1);
- ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(appId, 1);
+ public void testNMProxyRetry() throws Exception {
+ conf.setLong(YarnConfiguration.CLIENT_NM_CONNECT_MAX_WAIT_MS, 10000);
+ conf.setLong(YarnConfiguration.CLIENT_NM_CONNECT_RETRY_INTERVAL_MS, 100);
+ StartContainersRequest allRequests =
+ Records.newRecord(StartContainersRequest.class);
- org.apache.hadoop.yarn.api.records.Token nmToken =
- context.getNMTokenSecretManager().createNMToken(attemptId,
- context.getNodeId(), user);
- final InetSocketAddress address =
- conf.getSocketAddr(YarnConfiguration.NM_BIND_HOST,
- YarnConfiguration.NM_ADDRESS, YarnConfiguration.DEFAULT_NM_ADDRESS,
- YarnConfiguration.DEFAULT_NM_PORT);
- Token<NMTokenIdentifier> token =
- ConverterUtils.convertFromYarn(nmToken,
- SecurityUtil.buildTokenService(address));
- UserGroupInformation ugi = UserGroupInformation.createRemoteUser(user);
- ugi.addToken(token);
-
- ContainerManagementProtocol proxy =
- NMProxy.createNMProxy(conf, ContainerManagementProtocol.class, ugi,
- YarnRPC.create(conf), address);
+ ContainerManagementProtocol proxy = getNMProxy();
retryCount = 0;
shouldThrowNMNotYetReadyException = false;
@@ -156,4 +151,38 @@ public class TestNMProxy extends BaseContainerManagerTest {
proxy.startContainers(allRequests);
Assert.assertEquals(5, retryCount);
}
+
+ @Test(timeout = 20000, expected = IOException.class)
+ public void testShouldNotRetryForeverForNonNetworkExceptionsOnNMConnections()
+ throws Exception {
+ conf.setLong(YarnConfiguration.CLIENT_NM_CONNECT_MAX_WAIT_MS, -1);
+ StartContainersRequest allRequests =
+ Records.newRecord(StartContainersRequest.class);
+
+ ContainerManagementProtocol proxy = getNMProxy();
+
+ shouldThrowNMNotYetReadyException = false;
+ retryCount = 0;
+ proxy.startContainers(allRequests);
+ }
+
+ private ContainerManagementProtocol getNMProxy() {
+ ApplicationId appId = ApplicationId.newInstance(1, 1);
+ ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(appId, 1);
+
+ org.apache.hadoop.yarn.api.records.Token nmToken =
+ context.getNMTokenSecretManager().createNMToken(attemptId,
+ context.getNodeId(), user);
+ final InetSocketAddress address =
+ conf.getSocketAddr(YarnConfiguration.NM_BIND_HOST,
+ YarnConfiguration.NM_ADDRESS, YarnConfiguration.DEFAULT_NM_ADDRESS,
+ YarnConfiguration.DEFAULT_NM_PORT);
+ Token<NMTokenIdentifier> token =
+ ConverterUtils.convertFromYarn(nmToken,
+ SecurityUtil.buildTokenService(address));
+ UserGroupInformation ugi = UserGroupInformation.createRemoteUser(user);
+ ugi.addToken(token);
+ return NMProxy.createNMProxy(conf, ContainerManagementProtocol.class, ugi,
+ YarnRPC.create(conf), address);
+ }
}
[23/46] hadoop git commit: HDFS-8579. Update HDFS usage with missing
options (Contributed by J.Andreina)
Posted by ar...@apache.org.
HDFS-8579. Update HDFS usage with missing options (Contributed by J.Andreina)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/77588e1d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/77588e1d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/77588e1d
Branch: refs/heads/HDFS-7240
Commit: 77588e1d32cc465e6b0699bb3564a4c7c6df16fa
Parents: d3797f9
Author: Vinayakumar B <vi...@apache.org>
Authored: Tue Jun 30 15:00:22 2015 +0530
Committer: Vinayakumar B <vi...@apache.org>
Committed: Tue Jun 30 15:00:22 2015 +0530
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +++
hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs | 1 +
2 files changed, 4 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/77588e1d/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 3535f90..ea67ea1 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -983,6 +983,9 @@ Release 2.8.0 - UNRELEASED
HDFS-8687. Remove the duplicate usage message from Dfsck.java. (Brahma
Reddy Battula via Arpit Agarwal)
+ HDFS-8579. Update HDFS usage with missing options
+ (J.Andreina via vinayakumarb)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/77588e1d/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
index f464261..2e09cef 100755
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/bin/hdfs
@@ -26,6 +26,7 @@ function hadoop_usage
echo " Hadoop jar and the required libraries"
echo " crypto configure HDFS encryption zones"
echo " datanode run a DFS datanode"
+ echo " debug run a Debug Admin to execute HDFS debug commands"
echo " dfs run a filesystem command on the file system"
echo " dfsadmin run a DFS admin client"
echo " fetchdt fetch a delegation token from the NameNode"
[45/46] hadoop git commit: MAPREDUCE-6418. MRApp should not shutdown
LogManager during shutdown. Contributed by Chang Li
Posted by ar...@apache.org.
MAPREDUCE-6418. MRApp should not shutdown LogManager during shutdown. Contributed by Chang Li
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/eac1d189
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/eac1d189
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/eac1d189
Branch: refs/heads/HDFS-7240
Commit: eac1d1894354e90d314087af8e7fb168ddef9a3d
Parents: 8267156
Author: Jason Lowe <jl...@apache.org>
Authored: Wed Jul 1 18:01:22 2015 +0000
Committer: Jason Lowe <jl...@apache.org>
Committed: Wed Jul 1 18:01:22 2015 +0000
----------------------------------------------------------------------
hadoop-mapreduce-project/CHANGES.txt | 3 +++
.../apache/hadoop/mapreduce/v2/app/MRAppMaster.java | 14 +++++++++++---
.../org/apache/hadoop/mapreduce/v2/app/MRApp.java | 15 +++++++++++++++
3 files changed, 29 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eac1d189/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index dd9da7e..2f80615 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -527,6 +527,9 @@ Release 2.8.0 - UNRELEASED
MAPREDUCE-6420. Interrupted Exception in LocalContainerLauncher should be
logged in warn/info level (Chang Li via jlowe)
+ MAPREDUCE-6418. MRApp should not shutdown LogManager during shutdown
+ (Chang Li via jlowe)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eac1d189/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
index 0d3519b..f199ecb 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/main/java/org/apache/hadoop/mapreduce/v2/app/MRAppMaster.java
@@ -1197,11 +1197,15 @@ public class MRAppMaster extends CompositeService {
startJobs();
}
}
-
+
+ protected void shutdownTaskLog() {
+ TaskLog.syncLogsShutdown(logSyncer);
+ }
+
@Override
public void stop() {
super.stop();
- TaskLog.syncLogsShutdown(logSyncer);
+ shutdownTaskLog();
}
private boolean isRecoverySupported() throws IOException {
@@ -1705,10 +1709,14 @@ public class MRAppMaster extends CompositeService {
T call(Configuration conf) throws Exception;
}
+ protected void shutdownLogManager() {
+ LogManager.shutdown();
+ }
+
@Override
protected void serviceStop() throws Exception {
super.serviceStop();
- LogManager.shutdown();
+ shutdownLogManager();
}
public ClientService getClientService() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/eac1d189/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java
index 4fe4c44..b51adf2 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-app/src/test/java/org/apache/hadoop/mapreduce/v2/app/MRApp.java
@@ -804,5 +804,20 @@ public class MRApp extends MRAppMaster {
new Text(containerToken.getService()));
return token.decodeIdentifier();
}
+
+ @Override
+ protected void shutdownTaskLog() {
+ // Avoid closing the logging system during unit tests,
+ // otherwise subsequent MRApp instances in the same test
+ // will fail to log anything.
+ }
+
+ @Override
+ protected void shutdownLogManager() {
+ // Avoid closing the logging system during unit tests,
+ // otherwise subsequent MRApp instances in the same test
+ // will fail to log anything.
+ }
+
}
[41/46] hadoop git commit: YARN-3830.
AbstractYarnScheduler.createReleaseCache may try to clean a null attempt.
Contributed by nijel.
Posted by ar...@apache.org.
YARN-3830. AbstractYarnScheduler.createReleaseCache may try to clean a
null attempt. Contributed by nijel.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/80a68d60
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/80a68d60
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/80a68d60
Branch: refs/heads/HDFS-7240
Commit: 80a68d60560e505b5f8e01969dc3c168a1e5a7f3
Parents: 7405c59
Author: Devaraj K <de...@apache.org>
Authored: Wed Jul 1 19:03:44 2015 +0530
Committer: Devaraj K <de...@apache.org>
Committed: Wed Jul 1 19:03:44 2015 +0530
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +
.../scheduler/AbstractYarnScheduler.java | 34 +++++----
.../scheduler/TestAbstractYarnScheduler.java | 74 ++++++++++++++++++--
3 files changed, 91 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/80a68d60/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index 8c07e48..4389e27 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -583,6 +583,9 @@ Release 2.8.0 - UNRELEASED
YARN-3823. Fix mismatch in default values for
yarn.scheduler.maximum-allocation-vcores property. (Ray Chiang via devaraj)
+ YARN-3830. AbstractYarnScheduler.createReleaseCache may try to clean a null
+ attempt. (nijel via devaraj)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/80a68d60/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
index ae927f1..aad76fd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
@@ -66,6 +66,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeCleanContainerEvent;
import org.apache.hadoop.yarn.util.resource.Resources;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.util.concurrent.SettableFuture;
@@ -451,25 +452,30 @@ public abstract class AbstractYarnScheduler
new Timer().schedule(new TimerTask() {
@Override
public void run() {
- for (SchedulerApplication<T> app : applications.values()) {
+ clearPendingContainerCache();
+ LOG.info("Release request cache is cleaned up");
+ }
+ }, nmExpireInterval);
+ }
- T attempt = app.getCurrentAppAttempt();
- synchronized (attempt) {
- for (ContainerId containerId : attempt.getPendingRelease()) {
- RMAuditLogger.logFailure(
- app.getUser(),
+ @VisibleForTesting
+ public void clearPendingContainerCache() {
+ for (SchedulerApplication<T> app : applications.values()) {
+ T attempt = app.getCurrentAppAttempt();
+ if (attempt != null) {
+ synchronized (attempt) {
+ for (ContainerId containerId : attempt.getPendingRelease()) {
+ RMAuditLogger.logFailure(app.getUser(),
AuditConstants.RELEASE_CONTAINER,
- "Unauthorized access or invalid container",
- "Scheduler",
- "Trying to release container not owned by app or with invalid id.",
- attempt.getApplicationId(), containerId);
- }
- attempt.getPendingRelease().clear();
+ "Unauthorized access or invalid container", "Scheduler",
+ "Trying to release container not owned by app "
+ + "or with invalid id.", attempt.getApplicationId(),
+ containerId);
}
+ attempt.getPendingRelease().clear();
}
- LOG.info("Release request cache is cleaned up");
}
- }, nmExpireInterval);
+ }
}
// clean up a completed container
http://git-wip-us.apache.org/repos/asf/hadoop/blob/80a68d60/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAbstractYarnScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAbstractYarnScheduler.java
index 48ce822..91dd249 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAbstractYarnScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/TestAbstractYarnScheduler.java
@@ -18,26 +18,36 @@
package org.apache.hadoop.yarn.server.resourcemanager.scheduler;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceOption;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.server.resourcemanager.MockAM;
+import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
import org.apache.hadoop.yarn.server.resourcemanager.MockNodes;
import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
import org.apache.hadoop.yarn.server.resourcemanager.ParameterizedSchedulerTestBase;
+import org.apache.hadoop.yarn.server.resourcemanager.recovery.MemoryRMStateStore;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.MockRMApp;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppState;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeRemovedSchedulerEvent;
import org.apache.hadoop.yarn.util.resource.Resources;
import org.junit.Assert;
import org.junit.Test;
-import java.io.IOException;
-import java.util.HashMap;
-
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-
@SuppressWarnings("unchecked")
public class TestAbstractYarnScheduler extends ParameterizedSchedulerTestBase {
@@ -341,6 +351,58 @@ public class TestAbstractYarnScheduler extends ParameterizedSchedulerTestBase {
}
}
+ /*
+ * This test case is to test the pending containers are cleared from the
+ * attempt even if one of the application in the list have current attempt as
+ * null (no attempt).
+ */
+ @SuppressWarnings({ "rawtypes" })
+ @Test(timeout = 10000)
+ public void testReleasedContainerIfAppAttemptisNull() throws Exception {
+ YarnConfiguration conf=getConf();
+ conf.set(YarnConfiguration.RM_STORE, MemoryRMStateStore.class.getName());
+ MemoryRMStateStore memStore = new MemoryRMStateStore();
+ memStore.init(conf);
+ MockRM rm1 = new MockRM(conf, memStore);
+ try {
+ rm1.start();
+ MockNM nm1 =
+ new MockNM("127.0.0.1:1234", 8192, rm1.getResourceTrackerService());
+ nm1.registerNode();
+
+ AbstractYarnScheduler scheduler =
+ (AbstractYarnScheduler) rm1.getResourceScheduler();
+ // Mock App without attempt
+ RMApp mockAPp =
+ new MockRMApp(125, System.currentTimeMillis(), RMAppState.NEW);
+ SchedulerApplication<FiCaSchedulerApp> application =
+ new SchedulerApplication<FiCaSchedulerApp>(null, mockAPp.getUser());
+
+ // Second app with one app attempt
+ RMApp app = rm1.submitApp(200);
+ MockAM am1 = MockRM.launchAndRegisterAM(app, rm1, nm1);
+ final ContainerId runningContainer =
+ ContainerId.newContainerId(am1.getApplicationAttemptId(), 2);
+ am1.allocate(null, Arrays.asList(runningContainer));
+
+ Map schedulerApplications = scheduler.getSchedulerApplications();
+ SchedulerApplication schedulerApp =
+ (SchedulerApplication) scheduler.getSchedulerApplications().get(
+ app.getApplicationId());
+ schedulerApplications.put(mockAPp.getApplicationId(), application);
+
+ scheduler.clearPendingContainerCache();
+
+ Assert.assertEquals("Pending containers are not released "
+ + "when one of the application attempt is null !", schedulerApp
+ .getCurrentAppAttempt().getPendingRelease().size(), 0);
+ } finally {
+ if (rm1 != null) {
+ rm1.stop();
+ }
+ }
+ }
+
private void verifyMaximumResourceCapability(
Resource expectedMaximumResource, AbstractYarnScheduler scheduler) {
[19/46] hadoop git commit: Merge branch 'trunk' of
https://git-wip-us.apache.org/repos/asf/hadoop into trunk
Posted by ar...@apache.org.
Merge branch 'trunk' of https://git-wip-us.apache.org/repos/asf/hadoop into trunk
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/34ee0b9b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/34ee0b9b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/34ee0b9b
Branch: refs/heads/HDFS-7240
Commit: 34ee0b9b4797093f5aeb0d55f32cf1b74b02e1c2
Parents: 8e1bdc1 4672315
Author: Ming Ma <mi...@apache.org>
Authored: Mon Jun 29 14:37:44 2015 -0700
Committer: Ming Ma <mi...@apache.org>
Committed: Mon Jun 29 14:37:44 2015 -0700
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +++
.../api/records/impl/pb/SerializedExceptionPBImpl.java | 2 +-
.../records/impl/pb/TestSerializedExceptionPBImpl.java | 12 ++++++++++--
3 files changed, 14 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
[26/46] hadoop git commit: Revert "HADOOP-12009 Clarify
FileSystem.listStatus() sorting order & fix
FileSystemContractBaseTest:testListStatus. (J.Andreina via stevel)"
Posted by ar...@apache.org.
Revert "HADOOP-12009 Clarify FileSystem.listStatus() sorting order & fix FileSystemContractBaseTest:testListStatus. (J.Andreina via stevel)"
This reverts commit 3dfa8161f9412bcb040f3c29c471344f25f24337.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/076948d9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/076948d9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/076948d9
Branch: refs/heads/HDFS-7240
Commit: 076948d9a4053cc8be1927005c797273bae85e99
Parents: 7ba5bba
Author: Steve Loughran <st...@apache.org>
Authored: Tue Jun 30 11:54:12 2015 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Jun 30 11:54:12 2015 +0100
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 4 ----
.../main/java/org/apache/hadoop/fs/FileSystem.java | 17 +----------------
.../src/site/markdown/filesystem/filesystem.md | 4 ----
.../hadoop/fs/FileSystemContractBaseTest.java | 11 +++--------
4 files changed, 4 insertions(+), 32 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/076948d9/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 2cf9509..c010ff1 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -664,10 +664,6 @@ Release 2.8.0 - UNRELEASED
HADOOP-11958. MetricsSystemImpl fails to show backtrace when an error
occurs (Jason Lowe via jeagles)
-
- HADOOP-12009 Clarify FileSystem.listStatus() sorting order & fix
- FileSystemContractBaseTest:testListStatus. (J.Andreina via stevel)
-
OPTIMIZATIONS
HADOOP-11785. Reduce the number of listStatus operation in distcp
http://git-wip-us.apache.org/repos/asf/hadoop/blob/076948d9/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
index c98074a..c73caf7 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileSystem.java
@@ -1498,9 +1498,7 @@ public abstract class FileSystem extends Configured implements Closeable {
/**
* List the statuses of the files/directories in the given path if the path is
* a directory.
- * <p>
- * Does not guarantee to return the List of files/directories status in a
- * sorted order.
+ *
* @param f given path
* @return the statuses of the files/directories in the given patch
* @throws FileNotFoundException when the path does not exist;
@@ -1542,9 +1540,6 @@ public abstract class FileSystem extends Configured implements Closeable {
/**
* Filter files/directories in the given path using the user-supplied path
* filter.
- * <p>
- * Does not guarantee to return the List of files/directories status in a
- * sorted order.
*
* @param f
* a path name
@@ -1565,9 +1560,6 @@ public abstract class FileSystem extends Configured implements Closeable {
/**
* Filter files/directories in the given list of paths using default
* path filter.
- * <p>
- * Does not guarantee to return the List of files/directories status in a
- * sorted order.
*
* @param files
* a list of paths
@@ -1584,9 +1576,6 @@ public abstract class FileSystem extends Configured implements Closeable {
/**
* Filter files/directories in the given list of paths using user-supplied
* path filter.
- * <p>
- * Does not guarantee to return the List of files/directories status in a
- * sorted order.
*
* @param files
* a list of paths
@@ -1747,8 +1736,6 @@ public abstract class FileSystem extends Configured implements Closeable {
* while consuming the entries. Each file system implementation should
* override this method and provide a more efficient implementation, if
* possible.
- * Does not guarantee to return the iterator that traverses statuses
- * of the files in a sorted order.
*
* @param p target path
* @return remote iterator
@@ -1776,8 +1763,6 @@ public abstract class FileSystem extends Configured implements Closeable {
/**
* List the statuses and block locations of the files in the given path.
- * Does not guarantee to return the iterator that traverses statuses
- * of the files in a sorted order.
*
* If the path is a directory,
* if recursive is false, returns files in the directory;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/076948d9/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md
index f323374..84e3755 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md
@@ -183,10 +183,6 @@ to the same path:
forall fs in listStatus(Path) :
fs == getFileStatus(fs.path)
-**Ordering of results**: there is no guarantee of ordering of the listed entries.
-While HDFS currently returns an alphanumerically sorted list, neither the Posix `readdir()`
-nor Java's `File.listFiles()` API calls define any ordering of returned values. Applications
-which require a uniform sort order on the results must perform the sorting themselves.
### Atomicity and Consistency
http://git-wip-us.apache.org/repos/asf/hadoop/blob/076948d9/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemContractBaseTest.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemContractBaseTest.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemContractBaseTest.java
index 495af00..2ca81e9 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemContractBaseTest.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/FileSystemContractBaseTest.java
@@ -20,7 +20,6 @@ package org.apache.hadoop.fs;
import java.io.FileNotFoundException;
import java.io.IOException;
-import java.util.ArrayList;
import junit.framework.TestCase;
@@ -225,13 +224,9 @@ public abstract class FileSystemContractBaseTest extends TestCase {
paths = fs.listStatus(path("/test/hadoop"));
assertEquals(3, paths.length);
- ArrayList<String> list = new ArrayList<String>();
- for (FileStatus fileState : paths) {
- list.add(fileState.getPath().toString());
- }
- assertTrue(list.contains(path("/test/hadoop/a")));
- assertTrue(list.contains(path("/test/hadoop/b")));
- assertTrue(list.contains(path("/test/hadoop/c")));
+ assertEquals(path("/test/hadoop/a"), paths[0].getPath());
+ assertEquals(path("/test/hadoop/b"), paths[1].getPath());
+ assertEquals(path("/test/hadoop/c"), paths[2].getPath());
paths = fs.listStatus(path("/test/hadoop/a"));
assertEquals(0, paths.length);
[38/46] hadoop git commit: HADOOP-12116. Fix unrecommended syntax
usages in hadoop/hdfs/yarn script for cygwin in branch-2. Contributed by Li
Lu.
Posted by ar...@apache.org.
HADOOP-12116. Fix unrecommended syntax usages in hadoop/hdfs/yarn script for cygwin in branch-2. Contributed by Li Lu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b8e792cb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b8e792cb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b8e792cb
Branch: refs/heads/HDFS-7240
Commit: b8e792cba257fdb0ca266ecb2f60f3f10c3a0c3b
Parents: 3a72bfd
Author: cnauroth <cn...@apache.org>
Authored: Tue Jun 30 16:54:30 2015 -0700
Committer: cnauroth <cn...@apache.org>
Committed: Tue Jun 30 16:54:30 2015 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +++
1 file changed, 3 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b8e792cb/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 4e5f6bb..74d428f 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -913,6 +913,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-12159. Move DistCpUtils#compareFs() to org.apache.hadoop.fs.FileUtil
and fix for HA namespaces (rchiang via rkanter)
+ HADOOP-12116. Fix unrecommended syntax usages in hadoop/hdfs/yarn script for
+ cygwin in branch-2. (Li Lu via cnauroth)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
[03/46] hadoop git commit: YARN-3859. LeafQueue doesn't print user
properly for application add. Contributed by Varun Saxena.
Posted by ar...@apache.org.
YARN-3859. LeafQueue doesn't print user properly for application add.
Contributed by Varun Saxena.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b543d1a3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b543d1a3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b543d1a3
Branch: refs/heads/HDFS-7240
Commit: b543d1a390a67e5e92fea67d3a2635058c29e9da
Parents: 79ed0f9
Author: Devaraj K <de...@apache.org>
Authored: Sun Jun 28 10:04:50 2015 +0530
Committer: Devaraj K <de...@apache.org>
Committed: Sun Jun 28 10:04:50 2015 +0530
----------------------------------------------------------------------
hadoop-yarn-project/CHANGES.txt | 3 +++
.../yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java | 3 ++-
2 files changed, 5 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b543d1a3/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index e2bf706..cb6deb1 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -562,6 +562,9 @@ Release 2.8.0 - UNRELEASED
YARN-2871. TestRMRestart#testRMRestartGetApplicationList sometime fails in trunk.
(zhihai xu via xgong)
+ YARN-3859. LeafQueue doesn't print user properly for application add.
+ (Varun Saxena via devaraj)
+
Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b543d1a3/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
index 8e39133..e5b44a6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
@@ -676,7 +676,8 @@ public class LeafQueue extends AbstractCSQueue {
LOG.info("Application added -" +
" appId: " + application.getApplicationId() +
- " user: " + user + "," + " leaf-queue: " + getQueueName() +
+ " user: " + application.getUser() + "," +
+ " leaf-queue: " + getQueueName() +
" #user-pending-applications: " + user.getPendingApplications() +
" #user-active-applications: " + user.getActiveApplications() +
" #queue-pending-applications: " + getNumPendingApplications() +
[37/46] hadoop git commit: HADOOP-12159. Move DistCpUtils#compareFs()
to org.apache.hadoop.fs.FileUtil and fix for HA namespaces (rchiang via
rkanter)
Posted by ar...@apache.org.
HADOOP-12159. Move DistCpUtils#compareFs() to org.apache.hadoop.fs.FileUtil and fix for HA namespaces (rchiang via rkanter)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/aaafa0b2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/aaafa0b2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/aaafa0b2
Branch: refs/heads/HDFS-7240
Commit: aaafa0b2ee64f6cfb7fdc717500e1c483b9df8cc
Parents: ad60807
Author: Robert Kanter <rk...@apache.org>
Authored: Tue Jun 30 16:42:59 2015 -0700
Committer: Robert Kanter <rk...@apache.org>
Committed: Tue Jun 30 16:49:27 2015 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 3 +
.../java/org/apache/hadoop/fs/FileUtil.java | 42 +++++++
.../java/org/apache/hadoop/fs/TestFileUtil.java | 112 ++++++++++++++++++-
.../java/org/apache/hadoop/tools/DistCp.java | 3 +-
.../apache/hadoop/tools/util/DistCpUtils.java | 39 -------
5 files changed, 155 insertions(+), 44 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aaafa0b2/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 19ba620..4e5f6bb 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -910,6 +910,9 @@ Release 2.8.0 - UNRELEASED
HADOOP-10798. globStatus() should always return a sorted list of files
(cmccabe)
+ HADOOP-12159. Move DistCpUtils#compareFs() to org.apache.hadoop.fs.FileUtil
+ and fix for HA namespaces (rchiang via rkanter)
+
Release 2.7.2 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aaafa0b2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
index 9b9e213..8abb4eb 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
@@ -19,6 +19,9 @@
package org.apache.hadoop.fs;
import java.io.*;
+import java.net.InetAddress;
+import java.net.URI;
+import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Enumeration;
@@ -1329,4 +1332,43 @@ public class FileUtil {
unexpandedWildcardClasspath.toString()};
return jarCp;
}
+
+ public static boolean compareFs(FileSystem srcFs, FileSystem destFs) {
+ if (srcFs==null || destFs==null) {
+ return false;
+ }
+ URI srcUri = srcFs.getUri();
+ URI dstUri = destFs.getUri();
+ if (srcUri.getScheme()==null) {
+ return false;
+ }
+ if (!srcUri.getScheme().equals(dstUri.getScheme())) {
+ return false;
+ }
+ String srcHost = srcUri.getHost();
+ String dstHost = dstUri.getHost();
+ if ((srcHost!=null) && (dstHost!=null)) {
+ if (srcHost.equals(dstHost)) {
+ return srcUri.getPort()==dstUri.getPort();
+ }
+ try {
+ srcHost = InetAddress.getByName(srcHost).getCanonicalHostName();
+ dstHost = InetAddress.getByName(dstHost).getCanonicalHostName();
+ } catch (UnknownHostException ue) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Could not compare file-systems. Unknown host: ", ue);
+ }
+ return false;
+ }
+ if (!srcHost.equals(dstHost)) {
+ return false;
+ }
+ } else if (srcHost==null && dstHost!=null) {
+ return false;
+ } else if (srcHost!=null) {
+ return false;
+ }
+ // check for ports
+ return srcUri.getPort()==dstUri.getPort();
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aaafa0b2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java
index 41794b8..3418ade 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java
@@ -17,7 +17,8 @@
*/
package org.apache.hadoop.fs;
-import org.junit.Before;
+import org.junit.*;
+
import java.io.BufferedReader;
import java.io.File;
import java.io.FileInputStream;
@@ -25,8 +26,11 @@ import java.io.FileOutputStream;
import java.io.FileReader;
import java.io.IOException;
import java.io.OutputStream;
+import java.net.InetAddress;
import java.net.URI;
import java.io.PrintWriter;
+import java.net.URISyntaxException;
+import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
@@ -44,10 +48,12 @@ import org.apache.hadoop.util.Shell;
import org.apache.hadoop.util.StringUtils;
import org.apache.tools.tar.TarEntry;
import org.apache.tools.tar.TarOutputStream;
-import org.junit.After;
-import org.junit.Assert;
-import org.junit.Test;
+
+import javax.print.attribute.URISyntax;
+
import static org.junit.Assert.*;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
public class TestFileUtil {
private static final Log LOG = LogFactory.getLog(TestFileUtil.class);
@@ -64,6 +70,25 @@ public class TestFileUtil {
private final File dir2 = new File(del, DIR + "2");
private final File partitioned = new File(TEST_DIR, "partitioned");
+ private InetAddress inet1;
+ private InetAddress inet2;
+ private InetAddress inet3;
+ private InetAddress inet4;
+ private InetAddress inet5;
+ private InetAddress inet6;
+ private URI uri1;
+ private URI uri2;
+ private URI uri3;
+ private URI uri4;
+ private URI uri5;
+ private URI uri6;
+ private FileSystem fs1;
+ private FileSystem fs2;
+ private FileSystem fs3;
+ private FileSystem fs4;
+ private FileSystem fs5;
+ private FileSystem fs6;
+
/**
* Creates multiple directories for testing.
*
@@ -80,6 +105,7 @@ public class TestFileUtil {
* file: part-r-00000, contents: "foo"
* file: part-r-00001, contents: "bar"
*/
+ @Ignore
private void setupDirs() throws IOException {
Assert.assertFalse(del.exists());
Assert.assertFalse(tmp.exists());
@@ -1096,4 +1122,82 @@ public class TestFileUtil {
}
}
}
+
+ @Ignore
+ public void setupCompareFs() {
+ // Set up Strings
+ String host1 = "1.2.3.4";
+ String host2 = "2.3.4.5";
+ int port1 = 7000;
+ int port2 = 7001;
+ String uris1 = "hdfs://" + host1 + ":" + Integer.toString(port1) + "/tmp/foo";
+ String uris2 = "hdfs://" + host1 + ":" + Integer.toString(port2) + "/tmp/foo";
+ String uris3 = "hdfs://" + host2 + ":" + Integer.toString(port2) + "/tmp/foo";
+ String uris4 = "hdfs://" + host2 + ":" + Integer.toString(port2) + "/tmp/foo";
+ String uris5 = "file:///" + host1 + ":" + Integer.toString(port1) + "/tmp/foo";
+ String uris6 = "hdfs:///" + host1 + "/tmp/foo";
+ // Set up URI objects
+ try {
+ uri1 = new URI(uris1);
+ uri2 = new URI(uris2);
+ uri3 = new URI(uris3);
+ uri4 = new URI(uris4);
+ uri5 = new URI(uris5);
+ uri6 = new URI(uris6);
+ } catch (URISyntaxException use) {
+ }
+ // Set up InetAddress
+ inet1 = mock(InetAddress.class);
+ when(inet1.getCanonicalHostName()).thenReturn(host1);
+ inet2 = mock(InetAddress.class);
+ when(inet2.getCanonicalHostName()).thenReturn(host1);
+ inet3 = mock(InetAddress.class);
+ when(inet3.getCanonicalHostName()).thenReturn(host2);
+ inet4 = mock(InetAddress.class);
+ when(inet4.getCanonicalHostName()).thenReturn(host2);
+ inet5 = mock(InetAddress.class);
+ when(inet5.getCanonicalHostName()).thenReturn(host1);
+ inet6 = mock(InetAddress.class);
+ when(inet6.getCanonicalHostName()).thenReturn(host1);
+
+ // Link of InetAddress to corresponding URI
+ try {
+ when(InetAddress.getByName(uris1)).thenReturn(inet1);
+ when(InetAddress.getByName(uris2)).thenReturn(inet2);
+ when(InetAddress.getByName(uris3)).thenReturn(inet3);
+ when(InetAddress.getByName(uris4)).thenReturn(inet4);
+ when(InetAddress.getByName(uris5)).thenReturn(inet5);
+ } catch (UnknownHostException ue) {
+ }
+
+ fs1 = mock(FileSystem.class);
+ when(fs1.getUri()).thenReturn(uri1);
+ fs2 = mock(FileSystem.class);
+ when(fs2.getUri()).thenReturn(uri2);
+ fs3 = mock(FileSystem.class);
+ when(fs3.getUri()).thenReturn(uri3);
+ fs4 = mock(FileSystem.class);
+ when(fs4.getUri()).thenReturn(uri4);
+ fs5 = mock(FileSystem.class);
+ when(fs5.getUri()).thenReturn(uri5);
+ fs6 = mock(FileSystem.class);
+ when(fs6.getUri()).thenReturn(uri6);
+ }
+
+ @Test
+ public void testCompareFsNull() throws Exception {
+ setupCompareFs();
+ assertEquals(FileUtil.compareFs(null,fs1),false);
+ assertEquals(FileUtil.compareFs(fs1,null),false);
+ }
+
+ @Test
+ public void testCompareFsDirectories() throws Exception {
+ setupCompareFs();
+ assertEquals(FileUtil.compareFs(fs1,fs1),true);
+ assertEquals(FileUtil.compareFs(fs1,fs2),false);
+ assertEquals(FileUtil.compareFs(fs1,fs5),false);
+ assertEquals(FileUtil.compareFs(fs3,fs4),true);
+ assertEquals(FileUtil.compareFs(fs1,fs6),false);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aaafa0b2/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java
index 6921a1e..2385df3 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.mapreduce.Cluster;
@@ -347,7 +348,7 @@ public class DistCp extends Configured implements Tool {
workDir = new Path(workDir, WIP_PREFIX + targetPath.getName()
+ rand.nextInt());
FileSystem workFS = workDir.getFileSystem(configuration);
- if (!DistCpUtils.compareFs(targetFS, workFS)) {
+ if (!FileUtil.compareFs(targetFS, workFS)) {
throw new IllegalArgumentException("Work path " + workDir +
" and target path " + targetPath + " are in different file system");
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aaafa0b2/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/util/DistCpUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/util/DistCpUtils.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/util/DistCpUtils.java
index 5ac5bf1..70d947e 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/util/DistCpUtils.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/util/DistCpUtils.java
@@ -468,43 +468,4 @@ public class DistCpUtils {
return (sourceChecksum == null || targetChecksum == null ||
sourceChecksum.equals(targetChecksum));
}
-
- /* see if two file systems are the same or not
- *
- */
- public static boolean compareFs(FileSystem srcFs, FileSystem destFs) {
- URI srcUri = srcFs.getUri();
- URI dstUri = destFs.getUri();
- if (srcUri.getScheme() == null) {
- return false;
- }
- if (!srcUri.getScheme().equals(dstUri.getScheme())) {
- return false;
- }
- String srcHost = srcUri.getHost();
- String dstHost = dstUri.getHost();
- if ((srcHost != null) && (dstHost != null)) {
- try {
- srcHost = InetAddress.getByName(srcHost).getCanonicalHostName();
- dstHost = InetAddress.getByName(dstHost).getCanonicalHostName();
- } catch(UnknownHostException ue) {
- if (LOG.isDebugEnabled())
- LOG.debug("Could not compare file-systems. Unknown host: ", ue);
- return false;
- }
- if (!srcHost.equals(dstHost)) {
- return false;
- }
- }
- else if (srcHost == null && dstHost != null) {
- return false;
- }
- else if (srcHost != null) {
- return false;
- }
-
- //check for ports
-
- return srcUri.getPort() == dstUri.getPort();
- }
}
[06/46] hadoop git commit: Adding release 2.7.2 to CHANGES.txt.
Posted by ar...@apache.org.
Adding release 2.7.2 to CHANGES.txt.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/aad6a7d5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/aad6a7d5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/aad6a7d5
Branch: refs/heads/HDFS-7240
Commit: aad6a7d5dba5858d6e9845f18c4baec16c91911d
Parents: c6793dd
Author: Vinod Kumar Vavilapalli <vi...@apache.org>
Authored: Sun Jun 28 16:29:12 2015 -0700
Committer: Vinod Kumar Vavilapalli <vi...@apache.org>
Committed: Sun Jun 28 16:29:12 2015 -0700
----------------------------------------------------------------------
hadoop-common-project/hadoop-common/CHANGES.txt | 12 ++++++++++++
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 12 ++++++++++++
hadoop-mapreduce-project/CHANGES.txt | 12 ++++++++++++
hadoop-yarn-project/CHANGES.txt | 12 ++++++++++++
4 files changed, 48 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aad6a7d5/hadoop-common-project/hadoop-common/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/CHANGES.txt b/hadoop-common-project/hadoop-common/CHANGES.txt
index 132e4c0..219ef25 100644
--- a/hadoop-common-project/hadoop-common/CHANGES.txt
+++ b/hadoop-common-project/hadoop-common/CHANGES.txt
@@ -884,6 +884,18 @@ Release 2.8.0 - UNRELEASED
HADOOP-12076. Incomplete Cache Mechanism in CredentialProvider API.
(Larry McCay via cnauroth)
+Release 2.7.2 - UNRELEASED
+
+ INCOMPATIBLE CHANGES
+
+ NEW FEATURES
+
+ IMPROVEMENTS
+
+ OPTIMIZATIONS
+
+ BUG FIXES
+
Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aad6a7d5/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 94477fe..b89e10c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -968,6 +968,18 @@ Release 2.8.0 - UNRELEASED
HDFS-8546. Prune cached replicas from DatanodeDescriptor state on replica
invalidation. (wang)
+Release 2.7.2 - UNRELEASED
+
+ INCOMPATIBLE CHANGES
+
+ NEW FEATURES
+
+ IMPROVEMENTS
+
+ OPTIMIZATIONS
+
+ BUG FIXES
+
Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aad6a7d5/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index 4df3b44..785fce8 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -512,6 +512,18 @@ Release 2.8.0 - UNRELEASED
MAPREDUCE-6413. TestLocalJobSubmission is failing with unknown host
(zhihai xu via jlowe)
+Release 2.7.2 - UNRELEASED
+
+ INCOMPATIBLE CHANGES
+
+ NEW FEATURES
+
+ IMPROVEMENTS
+
+ OPTIMIZATIONS
+
+ BUG FIXES
+
Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/aad6a7d5/hadoop-yarn-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/CHANGES.txt b/hadoop-yarn-project/CHANGES.txt
index cb6deb1..714bfdc 100644
--- a/hadoop-yarn-project/CHANGES.txt
+++ b/hadoop-yarn-project/CHANGES.txt
@@ -565,6 +565,18 @@ Release 2.8.0 - UNRELEASED
YARN-3859. LeafQueue doesn't print user properly for application add.
(Varun Saxena via devaraj)
+Release 2.7.2 - UNRELEASED
+
+ INCOMPATIBLE CHANGES
+
+ NEW FEATURES
+
+ IMPROVEMENTS
+
+ OPTIMIZATIONS
+
+ BUG FIXES
+
Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES