You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by as...@apache.org on 2017/08/24 19:35:54 UTC
[01/50] [abbrv] hadoop git commit: YARN-5146. Support for Fair
Scheduler in new YARN UI. Contributed by Abdullah Yousufi. [Forced Update!]
Repository: hadoop
Updated Branches:
refs/heads/YARN-5972 66b01b343 -> 10eeb8b50 (forced update)
YARN-5146. Support for Fair Scheduler in new YARN UI. Contributed by Abdullah Yousufi.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/dadb0c22
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/dadb0c22
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/dadb0c22
Branch: refs/heads/YARN-5972
Commit: dadb0c2225adef5cb0126610733c285b51f4f43e
Parents: e3ae3e2
Author: Sunil G <su...@apache.org>
Authored: Tue Aug 15 21:58:44 2017 +0530
Committer: Sunil G <su...@apache.org>
Committed: Tue Aug 15 21:58:44 2017 +0530
----------------------------------------------------------------------
.../src/main/webapp/app/adapters/yarn-queue.js | 30 -----
.../app/adapters/yarn-queue/capacity-queue.js | 23 ++++
.../app/adapters/yarn-queue/fair-queue.js | 23 ++++
.../app/adapters/yarn-queue/fifo-queue.js | 23 ++++
.../app/adapters/yarn-queue/yarn-queue.js | 30 +++++
.../main/webapp/app/components/tree-selector.js | 19 ++-
.../src/main/webapp/app/models/yarn-queue.js | 94 --------------
.../app/models/yarn-queue/capacity-queue.js | 95 ++++++++++++++
.../webapp/app/models/yarn-queue/fair-queue.js | 79 ++++++++++++
.../webapp/app/models/yarn-queue/fifo-queue.js | 52 ++++++++
.../webapp/app/models/yarn-queue/yarn-queue.js | 23 ++++
.../main/webapp/app/routes/cluster-overview.js | 4 +-
.../src/main/webapp/app/routes/yarn-queue.js | 26 ++--
.../src/main/webapp/app/routes/yarn-queues.js | 12 +-
.../main/webapp/app/routes/yarn-queues/index.js | 25 ----
.../app/routes/yarn-queues/queues-selector.js | 25 ----
.../main/webapp/app/serializers/yarn-queue.js | 129 -------------------
.../serializers/yarn-queue/capacity-queue.js | 128 ++++++++++++++++++
.../app/serializers/yarn-queue/fair-queue.js | 92 +++++++++++++
.../app/serializers/yarn-queue/fifo-queue.js | 59 +++++++++
.../app/serializers/yarn-queue/yarn-queue.js | 47 +++++++
.../components/queue-configuration-table.hbs | 54 --------
.../templates/components/queue-navigator.hbs | 7 +-
.../yarn-queue/capacity-queue-conf-table.hbs | 54 ++++++++
.../yarn-queue/capacity-queue-info.hbs | 84 ++++++++++++
.../components/yarn-queue/capacity-queue.hbs | 63 +++++++++
.../yarn-queue/fair-queue-conf-table.hbs | 52 ++++++++
.../components/yarn-queue/fair-queue-info.hbs | 66 ++++++++++
.../components/yarn-queue/fair-queue.hbs | 63 +++++++++
.../yarn-queue/fifo-queue-conf-table.hbs | 56 ++++++++
.../components/yarn-queue/fifo-queue-info.hbs | 47 +++++++
.../components/yarn-queue/fifo-queue.hbs | 48 +++++++
.../webapp/app/templates/yarn-queue/info.hbs | 73 +----------
.../main/webapp/app/templates/yarn-queues.hbs | 54 +-------
.../src/main/webapp/app/utils/color-utils.js | 1 -
35 files changed, 1266 insertions(+), 494 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dadb0c22/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-queue.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-queue.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-queue.js
deleted file mode 100644
index f2017df..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-queue.js
+++ /dev/null
@@ -1,30 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import AbstractAdapter from './abstract';
-
-export default AbstractAdapter.extend({
- address: "rmWebAddress",
- restNameSpace: "cluster",
- serverName: "RM",
-
- pathForType(/*modelName*/) {
- return 'scheduler'; // move to some common place, return path by modelname.
- }
-
-});
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dadb0c22/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-queue/capacity-queue.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-queue/capacity-queue.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-queue/capacity-queue.js
new file mode 100644
index 0000000..7eb9f76
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-queue/capacity-queue.js
@@ -0,0 +1,23 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import YarnQueueAdapter from './yarn-queue';
+
+export default YarnQueueAdapter.extend({
+
+});
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dadb0c22/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-queue/fair-queue.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-queue/fair-queue.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-queue/fair-queue.js
new file mode 100644
index 0000000..7eb9f76
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-queue/fair-queue.js
@@ -0,0 +1,23 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import YarnQueueAdapter from './yarn-queue';
+
+export default YarnQueueAdapter.extend({
+
+});
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dadb0c22/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-queue/fifo-queue.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-queue/fifo-queue.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-queue/fifo-queue.js
new file mode 100644
index 0000000..7eb9f76
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-queue/fifo-queue.js
@@ -0,0 +1,23 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import YarnQueueAdapter from './yarn-queue';
+
+export default YarnQueueAdapter.extend({
+
+});
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dadb0c22/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-queue/yarn-queue.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-queue/yarn-queue.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-queue/yarn-queue.js
new file mode 100644
index 0000000..8184c39
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/adapters/yarn-queue/yarn-queue.js
@@ -0,0 +1,30 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import AbstractAdapter from '../abstract';
+
+export default AbstractAdapter.extend({
+ address: "rmWebAddress",
+ restNameSpace: "cluster",
+ serverName: "RM",
+
+ pathForType(/*modelName*/) {
+ return 'scheduler'; // move to some common place, return path by modelname.
+ }
+
+});
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dadb0c22/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/tree-selector.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/tree-selector.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/tree-selector.js
index 3d72b2f..1a81a32 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/tree-selector.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/components/tree-selector.js
@@ -39,6 +39,9 @@ export default Ember.Component.extend({
// mainSvg
mainSvg: undefined,
+ used: undefined,
+ max: undefined,
+
// Init data
initData: function() {
this.map = { };
@@ -52,7 +55,8 @@ export default Ember.Component.extend({
}.bind(this));
// var selected = this.get("selected");
-
+ this.used = this.get("used");
+ this.max = this.get("max");
this.initQueue("root", 1, this.treeData);
},
@@ -81,7 +85,6 @@ export default Ember.Component.extend({
// Queue is not existed
return;
}
-
if (depth > this.maxDepth) {
this.maxDepth = this.maxDepth + 1;
}
@@ -149,7 +152,9 @@ export default Ember.Component.extend({
nodeEnter.append("circle")
.attr("r", 1e-6)
.style("fill", function(d) {
- var usedCap = d.queueData.get("usedCapacity");
+ var maxCap = d.queueData.get(this.max);
+ maxCap = maxCap == undefined ? 100 : maxCap;
+ var usedCap = d.queueData.get(this.used) / maxCap * 100.0;
if (usedCap <= 60.0) {
return "LimeGreen";
} else if (usedCap <= 100.0) {
@@ -157,7 +162,7 @@ export default Ember.Component.extend({
} else {
return "LightCoral";
}
- });
+ }.bind(this));
// append percentage
nodeEnter.append("text")
@@ -166,13 +171,15 @@ export default Ember.Component.extend({
.attr("fill", "white")
.attr("text-anchor", function() { return "middle"; })
.text(function(d) {
- var usedCap = d.queueData.get("usedCapacity");
+ var maxCap = d.queueData.get(this.max);
+ maxCap = maxCap == undefined ? 100 : maxCap;
+ var usedCap = d.queueData.get(this.used) / maxCap * 100.0;
if (usedCap >= 100.0) {
return usedCap.toFixed(0) + "%";
} else {
return usedCap.toFixed(1) + "%";
}
- })
+ }.bind(this))
.style("fill-opacity", 1e-6);
// append queue name
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dadb0c22/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-queue.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-queue.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-queue.js
deleted file mode 100644
index 27c48f7..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-queue.js
+++ /dev/null
@@ -1,94 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import DS from 'ember-data';
-
-export default DS.Model.extend({
- name: DS.attr('string'),
- children: DS.attr('array'),
- parent: DS.attr('string'),
- capacity: DS.attr('number'),
- maxCapacity: DS.attr('number'),
- usedCapacity: DS.attr('number'),
- absCapacity: DS.attr('number'),
- absMaxCapacity: DS.attr('number'),
- absUsedCapacity: DS.attr('number'),
- state: DS.attr('string'),
- userLimit: DS.attr('number'),
- userLimitFactor: DS.attr('number'),
- preemptionDisabled: DS.attr('number'),
- numPendingApplications: DS.attr('number'),
- numActiveApplications: DS.attr('number'),
- users: DS.hasMany('YarnUser'),
-
- isLeafQueue: function() {
- var len = this.get("children.length");
- if (!len) {
- return true;
- }
- return len <= 0;
- }.property("children"),
-
- capacitiesBarChartData: function() {
- return [
- {
- label: "Absolute Capacity",
- value: this.get("name") === "root" ? 100 : this.get("absCapacity")
- },
- {
- label: "Absolute Used",
- value: this.get("name") === "root" ? this.get("usedCapacity") : this.get("absUsedCapacity")
- },
- {
- label: "Absolute Max Capacity",
- value: this.get("name") === "root" ? 100 : this.get("absMaxCapacity")
- }
- ];
- }.property("absCapacity", "absUsedCapacity", "absMaxCapacity"),
-
- userUsagesDonutChartData: function() {
- var data = [];
- if (this.get("users")) {
- this.get("users").forEach(function(o) {
- data.push({
- label: o.get("name"),
- value: o.get("usedMemoryMB")
- });
- });
- }
-
- return data;
- }.property("users"),
-
- hasUserUsages: function() {
- return this.get("userUsagesDonutChartData").length > 0;
- }.property(),
-
- numOfApplicationsDonutChartData: function() {
- return [
- {
- label: "Pending Apps",
- value: this.get("numPendingApplications") || 0 // TODO, fix the REST API so root will return #applications as well.
- },
- {
- label: "Active Apps",
- value: this.get("numActiveApplications") || 0
- }
- ];
- }.property()
-});
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dadb0c22/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-queue/capacity-queue.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-queue/capacity-queue.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-queue/capacity-queue.js
new file mode 100644
index 0000000..1cb07bb
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-queue/capacity-queue.js
@@ -0,0 +1,95 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import DS from 'ember-data';
+
+export default DS.Model.extend({
+ name: DS.attr('string'),
+ children: DS.attr('array'),
+ parent: DS.attr('string'),
+ capacity: DS.attr('number'),
+ maxCapacity: DS.attr('number'),
+ usedCapacity: DS.attr('number'),
+ absCapacity: DS.attr('number'),
+ absMaxCapacity: DS.attr('number'),
+ absUsedCapacity: DS.attr('number'),
+ state: DS.attr('string'),
+ userLimit: DS.attr('number'),
+ userLimitFactor: DS.attr('number'),
+ preemptionDisabled: DS.attr('number'),
+ numPendingApplications: DS.attr('number'),
+ numActiveApplications: DS.attr('number'),
+ users: DS.hasMany('YarnUser'),
+ type: DS.attr('string'),
+
+ isLeafQueue: function() {
+ var len = this.get("children.length");
+ if (!len) {
+ return true;
+ }
+ return len <= 0;
+ }.property("children"),
+
+ capacitiesBarChartData: function() {
+ return [
+ {
+ label: "Absolute Capacity",
+ value: this.get("name") === "root" ? 100 : this.get("absCapacity")
+ },
+ {
+ label: "Absolute Used",
+ value: this.get("name") === "root" ? this.get("usedCapacity") : this.get("absUsedCapacity")
+ },
+ {
+ label: "Absolute Max Capacity",
+ value: this.get("name") === "root" ? 100 : this.get("absMaxCapacity")
+ }
+ ];
+ }.property("absCapacity", "usedCapacity", "absMaxCapacity"),
+
+ userUsagesDonutChartData: function() {
+ var data = [];
+ if (this.get("users")) {
+ this.get("users").forEach(function(o) {
+ data.push({
+ label: o.get("name"),
+ value: o.get("usedMemoryMB")
+ });
+ });
+ }
+
+ return data;
+ }.property("users"),
+
+ hasUserUsages: function() {
+ return this.get("userUsagesDonutChartData").length > 0;
+ }.property(),
+
+ numOfApplicationsDonutChartData: function() {
+ return [
+ {
+ label: "Pending Apps",
+ value: this.get("numPendingApplications") || 0 // TODO, fix the REST API so root will return #applications as well.
+ },
+ {
+ label: "Active Apps",
+ value: this.get("numActiveApplications") || 0
+ }
+ ];
+ }.property("numPendingApplications", "numActiveApplications")
+});
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dadb0c22/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-queue/fair-queue.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-queue/fair-queue.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-queue/fair-queue.js
new file mode 100644
index 0000000..be71362
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-queue/fair-queue.js
@@ -0,0 +1,79 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import DS from 'ember-data';
+
+export default DS.Model.extend({
+ name: DS.attr('string'),
+ children: DS.attr('array'),
+ parent: DS.attr('string'),
+ maxApps: DS.attr('number'),
+ minResources: DS.attr(),
+ maxResources: DS.attr(),
+ usedResources: DS.attr(),
+ demandResources: DS.attr(),
+ steadyFairResources: DS.attr(),
+ fairResources: DS.attr(),
+ clusterResources: DS.attr(),
+ pendingContainers: DS.attr('number'),
+ allocatedContainers: DS.attr('number'),
+ reservedContainers: DS.attr('number'),
+ schedulingPolicy: DS.attr('string'),
+ preemptable: DS.attr('number'),
+ numPendingApplications: DS.attr('number'),
+ numActiveApplications: DS.attr('number'),
+ type: DS.attr('string'),
+
+ isLeafQueue: function() {
+ var len = this.get("children.length");
+ if (!len) {
+ return true;
+ }
+ return len <= 0;
+ }.property("children"),
+
+ capacitiesBarChartData: function() {
+ return [
+ {
+ label: "Steady Fair Memory",
+ value: this.get("steadyFairResources.memory")
+ },
+ {
+ label: "Used Memory",
+ value: this.get("usedResources.memory")
+ },
+ {
+ label: "Maximum Memory",
+ value: this.get("maxResources.memory")
+ }
+ ];
+ }.property("maxResources.memory", "usedResources.memory", "maxResources.memory"),
+
+ numOfApplicationsDonutChartData: function() {
+ return [
+ {
+ label: "Pending Apps",
+ value: this.get("numPendingApplications") || 0 // TODO, fix the REST API so root will return #applications as well.
+ },
+ {
+ label: "Active Apps",
+ value: this.get("numActiveApplications") || 0
+ }
+ ];
+ }.property()
+});
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dadb0c22/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-queue/fifo-queue.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-queue/fifo-queue.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-queue/fifo-queue.js
new file mode 100644
index 0000000..2386dc4
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-queue/fifo-queue.js
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import DS from 'ember-data';
+
+export default DS.Model.extend({
+ name: DS.attr('string'),
+ capacity: DS.attr('number'),
+ usedCapacity: DS.attr('number'),
+ state: DS.attr('string'),
+ minQueueMemoryCapacity: DS.attr('number'),
+ maxQueueMemoryCapacity: DS.attr('number'),
+ numNodes: DS.attr('number'),
+ usedNodeCapacity: DS.attr('number'),
+ availNodeCapacity: DS.attr('number'),
+ totalNodeCapacity: DS.attr('number'),
+ numContainers: DS.attr('number'),
+ type: DS.attr('string'),
+
+ capacitiesBarChartData: function() {
+ return [
+ {
+ label: "Available Capacity",
+ value: this.get("availNodeCapacity")
+ },
+ {
+ label: "Used Capacity",
+ value: this.get("usedNodeCapacity")
+ },
+ {
+ label: "Total Capacity",
+ value: this.get("totalNodeCapacity")
+ }
+ ];
+ }.property("availNodeCapacity", "usedNodeCapacity", "totalNodeCapacity")
+
+});
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dadb0c22/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-queue/yarn-queue.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-queue/yarn-queue.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-queue/yarn-queue.js
new file mode 100644
index 0000000..dcf5f48
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/models/yarn-queue/yarn-queue.js
@@ -0,0 +1,23 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import DS from 'ember-data';
+
+export default DS.Model.extend({
+ type: DS.attr('string')
+});
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dadb0c22/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/cluster-overview.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/cluster-overview.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/cluster-overview.js
index b5db17d..3c6abd4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/cluster-overview.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/cluster-overview.js
@@ -28,7 +28,7 @@ export default AbstractRoute.extend({
{
state: "RUNNING"
}),
- queues: this.store.query('yarn-queue', {}),
+ queues: this.store.query('yarn-queue.yarn-queue', {}),
});
},
@@ -39,6 +39,6 @@ export default AbstractRoute.extend({
unloadAll() {
this.store.unloadAll('ClusterMetric');
this.store.unloadAll('yarn-app');
- this.store.unloadAll('yarn-queue');
+ this.store.unloadAll('yarn-queue.yarn-queue');
}
});
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dadb0c22/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-queue.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-queue.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-queue.js
index 1c4546c..cd4ed09 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-queue.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-queue.js
@@ -22,22 +22,28 @@ import AbstractRoute from './abstract';
export default AbstractRoute.extend({
model(param) {
- return Ember.RSVP.hash({
- selected : param.queue_name,
- queues: this.store.query('yarn-queue', {}),
- selectedQueue : undefined,
- apps: this.store.query('yarn-app', {
- queue: param.queue_name
- })
- });
+ return Ember.RSVP.hash({
+ selected : param.queue_name,
+ queues: this.store.query("yarn-queue.yarn-queue", {}).then((model) => {
+ let type = model.get('firstObject').get('type');
+ return this.store.query("yarn-queue." + type + "-queue", {});
+ }),
+ selectedQueue : undefined,
+ apps: this.store.query('yarn-app', {
+ queue: param.queue_name
+ })
+ });
},
afterModel(model) {
- model.selectedQueue = this.store.peekRecord('yarn-queue', model.selected);
+ var type = model.queues.get('firstObject').constructor.modelName;
+ model.selectedQueue = this.store.peekRecord(type, model.selected);
},
unloadAll() {
- this.store.unloadAll('yarn-queue');
+ this.store.unloadAll('yarn-queue.capacity-queue');
+ this.store.unloadAll('yarn-queue.fair-queue');
+ this.store.unloadAll('yarn-queue.fifo-queue');
this.store.unloadAll('yarn-app');
}
});
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dadb0c22/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-queues.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-queues.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-queues.js
index e4f145d..7d8a200 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-queues.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-queues.js
@@ -30,17 +30,23 @@ export default AbstractRoute.extend({
}
return Ember.RSVP.hash({
selected : queueName,
- queues: this.store.query('yarn-queue', {}),
+ queues: this.store.query("yarn-queue.yarn-queue", {}).then((model) => {
+ let type = model.get('firstObject').get('type');
+ return this.store.query("yarn-queue." + type + "-queue", {});
+ }),
selectedQueue : undefined
});
},
afterModel(model) {
- model.selectedQueue = this.store.peekRecord('yarn-queue', model.selected);
+ var type = model.queues.get('firstObject').constructor.modelName;
+ model.selectedQueue = this.store.peekRecord(type, model.selected);
},
unloadAll() {
- this.store.unloadAll('yarn-queue');
+ this.store.unloadAll('yarn-queue.capacity-queue');
+ this.store.unloadAll('yarn-queue.fair-queue');
+ this.store.unloadAll('yarn-queue.fifo-queue');
this.store.unloadAll('yarn-app');
},
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dadb0c22/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-queues/index.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-queues/index.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-queues/index.js
deleted file mode 100644
index 436c6d8..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-queues/index.js
+++ /dev/null
@@ -1,25 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import Ember from 'ember';
-
-export default Ember.Route.extend({
- beforeModel() {
- this.transitionTo('yarn-queues.root');
- }
-});
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dadb0c22/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-queues/queues-selector.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-queues/queues-selector.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-queues/queues-selector.js
deleted file mode 100644
index 5d14c6f..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/routes/yarn-queues/queues-selector.js
+++ /dev/null
@@ -1,25 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import Ember from 'ember';
-
-export default Ember.Route.extend({
- model() {
- return this.store.findAll('yarn-queue');
- },
-});
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dadb0c22/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-queue.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-queue.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-queue.js
deleted file mode 100644
index 4fc1a29..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-queue.js
+++ /dev/null
@@ -1,129 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-import DS from 'ember-data';
-
-export default DS.JSONAPISerializer.extend({
-
- normalizeSingleResponse(store, primaryModelClass, payload, id,
- requestType) {
- var children = [];
- if (payload.queues) {
- payload.queues.queue.forEach(function(queue) {
- children.push(queue.queueName);
- });
- }
-
- var includedData = [];
- var relationshipUserData = [];
-
- // update user models
- if (payload.users && payload.users.user) {
- payload.users.user.forEach(function(u) {
- includedData.push({
- type: "YarnUser",
- id: u.username + "_" + payload.queueName,
- attributes: {
- name: u.username,
- queueName: payload.queueName,
- usedMemoryMB: u.resourcesUsed.memory || 0,
- usedVCore: u.resourcesUsed.vCores || 0,
- }
- });
-
- relationshipUserData.push({
- type: "YarnUser",
- id: u.username + "_" + payload.queueName,
- });
- });
- }
-
-
- var fixedPayload = {
- id: id,
- type: primaryModelClass.modelName, // yarn-queue
- attributes: {
- name: payload.queueName,
- parent: payload.myParent,
- children: children,
- capacity: payload.capacity,
- usedCapacity: payload.usedCapacity,
- maxCapacity: payload.maxCapacity,
- absCapacity: payload.absoluteCapacity,
- absMaxCapacity: payload.absoluteMaxCapacity,
- absUsedCapacity: payload.absoluteUsedCapacity,
- state: payload.state,
- userLimit: payload.userLimit,
- userLimitFactor: payload.userLimitFactor,
- preemptionDisabled: payload.preemptionDisabled,
- numPendingApplications: payload.numPendingApplications,
- numActiveApplications: payload.numActiveApplications,
- },
- // Relationships
- relationships: {
- users: {
- data: relationshipUserData
- }
- }
- };
-
- return {
- queue: this._super(store, primaryModelClass, fixedPayload, id, requestType),
- includedData: includedData
- };
- },
-
- handleQueue(store, primaryModelClass, payload, id, requestType) {
- var data = [];
- var includedData = [];
- var result = this.normalizeSingleResponse(store, primaryModelClass,
- payload, id, requestType);
-
- data.push(result.queue);
- includedData = includedData.concat(result.includedData);
-
- if (payload.queues) {
- for (var i = 0; i < payload.queues.queue.length; i++) {
- var queue = payload.queues.queue[i];
- queue.myParent = payload.queueName;
- var childResult = this.handleQueue(store, primaryModelClass, queue,
- queue.queueName,
- requestType);
-
- data = data.concat(childResult.data);
- includedData = includedData.concat(childResult.includedData);
- }
- }
-
- return {
- data: data,
- includedData: includedData
- };
- },
-
- normalizeArrayResponse(store, primaryModelClass, payload, id, requestType) {
- var normalizedArrayResponse = {};
- var result = this.handleQueue(store, primaryModelClass,
- payload.scheduler.schedulerInfo, "root", requestType);
-
- normalizedArrayResponse.data = result.data;
- normalizedArrayResponse.included = result.includedData;
-
- return normalizedArrayResponse;
- }
-});
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dadb0c22/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-queue/capacity-queue.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-queue/capacity-queue.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-queue/capacity-queue.js
new file mode 100644
index 0000000..c7350ef
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-queue/capacity-queue.js
@@ -0,0 +1,128 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import DS from 'ember-data';
+
+export default DS.JSONAPISerializer.extend({
+
+ normalizeSingleResponse(store, primaryModelClass, payload, id,
+ requestType) {
+ var children = [];
+ if (payload.queues) {
+ payload.queues.queue.forEach(function(queue) {
+ children.push(queue.queueName);
+ });
+ }
+
+ var includedData = [];
+ var relationshipUserData = [];
+
+ // update user models
+ if (payload.users && payload.users.user) {
+ payload.users.user.forEach(function(u) {
+ includedData.push({
+ type: "YarnUser",
+ id: u.username + "_" + payload.queueName,
+ attributes: {
+ name: u.username,
+ queueName: payload.queueName,
+ usedMemoryMB: u.resourcesUsed.memory || 0,
+ usedVCore: u.resourcesUsed.vCores || 0,
+ }
+ });
+
+ relationshipUserData.push({
+ type: "YarnUser",
+ id: u.username + "_" + payload.queueName,
+ });
+ });
+ }
+
+ var fixedPayload = {
+ id: id,
+ type: primaryModelClass.modelName, // yarn-queue
+ attributes: {
+ name: payload.queueName,
+ parent: payload.myParent,
+ children: children,
+ capacity: payload.capacity,
+ usedCapacity: payload.usedCapacity,
+ maxCapacity: payload.maxCapacity,
+ absCapacity: payload.absoluteCapacity,
+ absMaxCapacity: payload.absoluteMaxCapacity,
+ absUsedCapacity: payload.absoluteUsedCapacity,
+ state: payload.state,
+ userLimit: payload.userLimit,
+ userLimitFactor: payload.userLimitFactor,
+ preemptionDisabled: payload.preemptionDisabled,
+ numPendingApplications: payload.numPendingApplications,
+ numActiveApplications: payload.numActiveApplications,
+ type: "capacity",
+ },
+ // Relationships
+ relationships: {
+ users: {
+ data: relationshipUserData
+ }
+ }
+ };
+ return {
+ queue: this._super(store, primaryModelClass, fixedPayload, id, requestType),
+ includedData: includedData
+ };
+ },
+
+ handleQueue(store, primaryModelClass, payload, id, requestType) {
+ var data = [];
+ var includedData = [];
+ var result = this.normalizeSingleResponse(store, primaryModelClass,
+ payload, id, requestType);
+
+ data.push(result.queue);
+ includedData = includedData.concat(result.includedData);
+
+ if (payload.queues) {
+ for (var i = 0; i < payload.queues.queue.length; i++) {
+ var queue = payload.queues.queue[i];
+ queue.myParent = payload.queueName;
+ var childResult = this.handleQueue(store, primaryModelClass, queue,
+ queue.queueName,
+ requestType);
+
+ data = data.concat(childResult.data);
+ includedData = includedData.concat(childResult.includedData);
+ }
+ }
+
+ return {
+ data: data,
+ includedData: includedData
+ };
+ },
+
+ normalizeArrayResponse(store, primaryModelClass, payload, id, requestType) {
+ var normalizedArrayResponse = {};
+ var result = this.handleQueue(store, primaryModelClass,
+ payload.scheduler.schedulerInfo, "root", requestType);
+
+ normalizedArrayResponse.data = result.data;
+ normalizedArrayResponse.included = result.includedData;
+
+ return normalizedArrayResponse;
+ }
+});
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dadb0c22/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-queue/fair-queue.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-queue/fair-queue.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-queue/fair-queue.js
new file mode 100644
index 0000000..2215d2d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-queue/fair-queue.js
@@ -0,0 +1,92 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import DS from 'ember-data';
+
+export default DS.JSONAPISerializer.extend({
+
+ normalizeSingleResponse(store, primaryModelClass, payload, id,
+ requestType) {
+ var children = [];
+ if (payload.childQueues) {
+ payload.childQueues.queue.forEach(function(queue) {
+ children.push(queue.queueName);
+ });
+ }
+
+ var fixedPayload = {
+ id: id,
+ type: primaryModelClass.modelName,
+ attributes: {
+ name: payload.queueName,
+ parent: payload.myParent,
+ children: children,
+ maxApps: payload.maxApps,
+ minResources: payload.minResources,
+ maxResources: payload.maxResources,
+ usedResources: payload.usedResources,
+ demandResources: payload.demandResources,
+ steadyFairResources: payload.steadyFairResources,
+ fairResources: payload.fairResources,
+ clusterResources: payload.clusterResources,
+ pendingContainers: payload.pendingContainers,
+ allocatedContainers: payload.allocatedContainers,
+ reservedContainers: payload.reservedContainers,
+ schedulingPolicy: payload.schedulingPolicy,
+ preemptable: payload.preemptable,
+ numPendingApplications: payload.numPendingApps,
+ numActiveApplications: payload.numActiveApps,
+ type: "fair",
+ },
+ };
+ return this._super(store, primaryModelClass, fixedPayload, id, requestType);
+ },
+
+ handleQueue(store, primaryModelClass, payload, id, requestType) {
+ var data = [];
+ var includedData = [];
+ if(!payload) return data;
+ var result = this.normalizeSingleResponse(store, primaryModelClass,
+ payload, id, requestType);
+
+ data.push(result);
+
+ if (payload.childQueues) {
+ for (var i = 0; i < payload.childQueues.queue.length; i++) {
+ var queue = payload.childQueues.queue[i];
+ queue.myParent = payload.queueName;
+ var childResult = this.handleQueue(store, primaryModelClass, queue,
+ queue.queueName,
+ requestType);
+
+ data = data.concat(childResult);
+ }
+ }
+
+ return data;
+ },
+
+ normalizeArrayResponse(store, primaryModelClass, payload, id, requestType) {
+ var normalizedArrayResponse = {};
+ var result = this.handleQueue(store, primaryModelClass,
+ payload.scheduler.schedulerInfo.rootQueue, "root", requestType);
+
+ normalizedArrayResponse.data = result;
+ return normalizedArrayResponse;
+ }
+});
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dadb0c22/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-queue/fifo-queue.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-queue/fifo-queue.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-queue/fifo-queue.js
new file mode 100644
index 0000000..297ec18
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-queue/fifo-queue.js
@@ -0,0 +1,59 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import DS from 'ember-data';
+
+export default DS.JSONAPISerializer.extend({
+
+ normalizeSingleResponse(store, primaryModelClass, payload, id,
+ requestType) {
+
+ var fixedPayload = {
+ id: id,
+ type: primaryModelClass.modelName,
+ attributes: {
+ name: id,
+ capacity: payload.capacity * 100,
+ usedCapacity: payload.usedCapacity * 100,
+ usedNodeCapacity: payload.usedNodeCapacity,
+ availNodeCapacity: payload.availNodeCapacity,
+ totalNodeCapacity: payload.totalNodeCapacity,
+ numNodes: payload.numNodes,
+ numContainers: payload.numContainers,
+ state: payload.qstate,
+ minQueueMemoryCapacity: payload.minQueueMemoryCapacity,
+ maxQueueMemoryCapacity: payload.maxQueueMemoryCapacity,
+ type: "fifo",
+ },
+
+ };
+
+ return this._super(store, primaryModelClass, fixedPayload, id,
+ requestType);
+ },
+
+ normalizeArrayResponse(store, primaryModelClass, payload, id, requestType) {
+ var normalizedArrayResponse = {};
+ normalizedArrayResponse.data = [
+ this.normalizeSingleResponse(store, primaryModelClass,
+ payload.scheduler.schedulerInfo, "root", requestType)
+ ];
+
+ return normalizedArrayResponse;
+ }
+});
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dadb0c22/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-queue/yarn-queue.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-queue/yarn-queue.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-queue/yarn-queue.js
new file mode 100644
index 0000000..b2e0f2f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/serializers/yarn-queue/yarn-queue.js
@@ -0,0 +1,47 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+import DS from 'ember-data';
+
+export default DS.JSONAPISerializer.extend({
+
+ normalizeSingleResponse(store, primaryModelClass, payload, id,
+ requestType) {
+
+ var fixedPayload = {
+ id: id,
+ type: primaryModelClass.modelName,
+ attributes: {
+ type: payload.type.split(/(?=[A-Z])/)[0]
+ }
+ };
+ return this._super(store, primaryModelClass, fixedPayload, id,
+ requestType);
+ },
+
+ normalizeArrayResponse(store, primaryModelClass, payload, id, requestType) {
+ var normalizedArrayResponse = {};
+
+ normalizedArrayResponse.data = [
+ this.normalizeSingleResponse(store, primaryModelClass,
+ payload.scheduler.schedulerInfo, "root", requestType)
+ ];
+
+ return normalizedArrayResponse;
+ }
+});
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dadb0c22/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/queue-configuration-table.hbs
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/queue-configuration-table.hbs b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/queue-configuration-table.hbs
deleted file mode 100644
index 17a1e1a..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/queue-configuration-table.hbs
+++ /dev/null
@@ -1,54 +0,0 @@
-{{!
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
-}}
-
-<table id="queue-configuration-table" class="table table-striped table-bordered" cellspacing="0" width="100%" height="100%">
- <thead>
- <tr>
- <td><b>Configurations</b></td>
- <td>Value</td>
- </tr>
- </thead>
- <tbody>
- <tr>
- <td>Configured Capacity</td>
- <td>{{queue.capacity}}</td>
- </tr>
- <tr>
- <td>Configured Max Capacity</td>
- <td>{{queue.maxCapacity}}</td>
- </tr>
- <tr>
- <td>State</td>
- <td>{{queue.state}}</td>
- </tr>
- {{#if queue.isLeafQueue}}
- <tr>
- <td>User Limit Percent</td>
- <td>{{queue.userLimit}}</td>
- </tr>
- <tr>
- <td>User Limit Factor</td>
- <td>{{queue.userLimitFactor}}</td>
- </tr>
- <tr>
- <td>Preemption Disabled</td>
- <td>{{queue.preemptionDisabled}}</td>
- </tr>
- {{/if}}
- </tbody>
-</table>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dadb0c22/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/queue-navigator.hbs
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/queue-navigator.hbs b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/queue-navigator.hbs
index d8dd236..e3b0a90 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/queue-navigator.hbs
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/queue-navigator.hbs
@@ -20,9 +20,12 @@
<div class="row">
<div class="col-md-12 container-fluid">
<div class="panel panel-default" id="tree-selector-container">
- {{tree-selector model=model parentId="tree-selector-container" selected=selected}}
+ <div class="panel-heading">
+ Scheduler: {{model.firstObject.type}}
+ </div>
+ {{tree-selector model=model parentId="tree-selector-container" selected=selected used=used max=max}}
</div>
</div>
</div>
-{{outlet}}
\ No newline at end of file
+{{outlet}}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dadb0c22/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/yarn-queue/capacity-queue-conf-table.hbs
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/yarn-queue/capacity-queue-conf-table.hbs b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/yarn-queue/capacity-queue-conf-table.hbs
new file mode 100644
index 0000000..3f6017f
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/yarn-queue/capacity-queue-conf-table.hbs
@@ -0,0 +1,54 @@
+{{!
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+}}
+
+<table id="queue-configuration-table" class="table table-striped table-bordered" cellspacing="0" width="100%" height="100%">
+ <thead>
+ <tr>
+ <td><b>Configurations</b></td>
+ <td>Value</td>
+ </tr>
+ </thead>
+ <tbody>
+ <tr>
+ <td>Configured Capacity</td>
+ <td>{{queue.capacity}}</td>
+ </tr>
+ <tr>
+ <td>Configured Max Capacity</td>
+ <td>{{queue.maxCapacity}}</td>
+ </tr>
+ <tr>
+ <td>State</td>
+ <td>{{queue.state}}</td>
+ </tr>
+ {{#if queue.isLeafQueue}}
+ <tr>
+ <td>User Limit Percent</td>
+ <td>{{queue.userLimit}}</td>
+ </tr>
+ <tr>
+ <td>User Limit Factor</td>
+ <td>{{queue.userLimitFactor}}</td>
+ </tr>
+ <tr>
+ <td>Preemption Disabled</td>
+ <td>{{queue.preemptionDisabled}}</td>
+ </tr>
+ {{/if}}
+ </tbody>
+</table>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dadb0c22/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/yarn-queue/capacity-queue-info.hbs
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/yarn-queue/capacity-queue-info.hbs b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/yarn-queue/capacity-queue-info.hbs
new file mode 100644
index 0000000..7d44e69
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/yarn-queue/capacity-queue-info.hbs
@@ -0,0 +1,84 @@
+{{!
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+}}
+
+<div class="row">
+
+ <div class="col-lg-6 container-fluid">
+ <div class="panel panel-default">
+ <div class="panel-heading">
+ Queue Capacities: {{model.selected}}
+ </div>
+ <div class="container-fluid" id="capacity-bar-chart">
+ <br/>
+ {{bar-chart data=model.selectedQueue.capacitiesBarChartData
+ title=""
+ parentId="capacity-bar-chart"
+ textWidth=170
+ ratio=0.55
+ maxHeight=350}}
+ </div>
+ </div>
+ </div>
+
+ <div class="col-lg-6 container-fluid">
+ <div class="panel panel-default">
+ <div class="panel-heading">
+ Queue Information: {{model.selected}}
+ </div>
+ {{yarn-queue.capacity-queue-conf-table queue=model.selectedQueue}}
+ </div>
+ </div>
+
+</div>
+
+<div class="row">
+
+ <div class="col-lg-6 container-fluid">
+ <div class="panel panel-default">
+ <div class="panel-heading">
+ Running Apps: {{model.selected}}
+ </div>
+ <div class="container-fluid" id="numapplications-donut-chart">
+ {{donut-chart data=model.selectedQueue.numOfApplicationsDonutChartData
+ showLabels=true
+ parentId="numapplications-donut-chart"
+ ratio=0.6
+ maxHeight=350}}
+ </div>
+ </div>
+ </div>
+
+ {{#if model.selectedQueue.hasUserUsages}}
+ <div class="col-lg-6 container-fluid">
+ <div class="panel panel-default">
+ <div class="panel-heading">
+ User Usages: {{model.selected}}
+ </div>
+ <div class="container-fluid" id="userusage-donut-chart">
+ {{donut-chart data=model.selectedQueue.userUsagesDonutChartData
+ showLabels=true
+ parentId="userusage-donut-chart"
+ type="memory"
+ ratio=0.6
+ maxHeight=350}}
+ </div>
+ </div>
+ </div>
+ {{/if}}
+
+</div>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dadb0c22/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/yarn-queue/capacity-queue.hbs
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/yarn-queue/capacity-queue.hbs b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/yarn-queue/capacity-queue.hbs
new file mode 100644
index 0000000..8b63b66
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/yarn-queue/capacity-queue.hbs
@@ -0,0 +1,63 @@
+{{!
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+}}
+
+{{queue-navigator model=model.queues selected=model.selected
+ used="usedCapacity" max="absMaxCapacity"}}
+
+<div class="row">
+ <div class="col-lg-4 container-fluid">
+ <div class="panel panel-default">
+ <div class="panel-heading">
+ Queue Information: {{model.selected}}
+ </div>
+ {{yarn-queue.capacity-queue-conf-table queue=model.selectedQueue}}
+ </div>
+ </div>
+
+ <div class="col-lg-4 container-fluid">
+ <div class="panel panel-default">
+ <div class="panel-heading">
+ Queue Capacities: {{model.selected}}
+ </div>
+ <div class="container-fluid" id="capacity-bar-chart">
+ <br/>
+ {{bar-chart data=model.selectedQueue.capacitiesBarChartData
+ title=""
+ parentId="capacity-bar-chart"
+ textWidth=175
+ ratio=0.55
+ maxHeight=350}}
+ </div>
+ </div>
+ </div>
+
+ <div class="col-lg-4 container-fluid">
+ <div class="panel panel-default">
+ <div class="panel-heading">
+ Running Apps: {{model.selected}}
+ </div>
+ <div class="container-fluid" id="numapplications-donut-chart">
+ {{donut-chart data=model.selectedQueue.numOfApplicationsDonutChartData
+ showLabels=true
+ parentId="numapplications-donut-chart"
+ ratio=0.6
+ maxHeight=350}}
+ </div>
+ </div>
+ </div>
+</div>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dadb0c22/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/yarn-queue/fair-queue-conf-table.hbs
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/yarn-queue/fair-queue-conf-table.hbs b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/yarn-queue/fair-queue-conf-table.hbs
new file mode 100644
index 0000000..00fabcc
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/yarn-queue/fair-queue-conf-table.hbs
@@ -0,0 +1,52 @@
+{{!
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+}}
+
+<table id="queue-configuration-table" class="table table-striped table-bordered" cellspacing="0" width="100%" height="100%">
+ <thead>
+ <tr>
+ <td><b>Configurations</b></td>
+ <td>Value</td>
+ </tr>
+ </thead>
+ <tbody>
+ <tr>
+ <td>Fair Memory, VCores</td>
+ <td>{{queue.fairResources.memory}} MB, {{queue.fairResources.vCores}}</td>
+ </tr>
+ <tr>
+ <td>Minimum Memory, VCores</td>
+ <td>{{queue.minResources.memory}} MB, {{queue.minResources.vCores}}</td>
+ </tr>
+ <tr>
+ <td>Cluster Memory, VCores</td>
+ <td>{{queue.clusterResources.memory}} MB, {{queue.clusterResources.vCores}}</td>
+ </tr>
+ <tr>
+ <td>Pending, Allocated, Reserved Containers</td>
+ <td>{{queue.pendingContainers}} , {{queue.allocatedContainers}} , {{queue.reservedContainers}}</td>
+ </tr>
+ <tr>
+ <td>Scheduling Policy</td>
+ <td>{{queue.schedulingPolicy}}</td>
+ </tr>
+ <tr>
+ <td>Preemption Enabled</td>
+ <td>{{queue.preemptable}}</td>
+ </tr>
+ </tbody>
+</table>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dadb0c22/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/yarn-queue/fair-queue-info.hbs
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/yarn-queue/fair-queue-info.hbs b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/yarn-queue/fair-queue-info.hbs
new file mode 100644
index 0000000..a770bfe
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/yarn-queue/fair-queue-info.hbs
@@ -0,0 +1,66 @@
+{{!
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+}}
+
+<div class="row">
+
+ <div class="col-lg-6 container-fluid">
+ <div class="panel panel-default">
+ <div class="panel-heading">
+ Queue Capacities: {{model.selected}}
+ </div>
+ <div class="container-fluid" id="capacity-bar-chart">
+ <br/>
+ {{bar-chart data=model.selectedQueue.capacitiesBarChartData
+ title=""
+ parentId="capacity-bar-chart"
+ textWidth=170
+ ratio=0.55
+ maxHeight=350}}
+ </div>
+ </div>
+ </div>
+
+ <div class="col-lg-6 container-fluid">
+ <div class="panel panel-default">
+ <div class="panel-heading">
+ Queue Information: {{model.selected}}
+ </div>
+ {{yarn-queue.fair-queue-conf-table queue=model.selectedQueue}}
+ </div>
+ </div>
+
+</div>
+
+<div class="row">
+
+ <div class="col-lg-6 container-fluid">
+ <div class="panel panel-default">
+ <div class="panel-heading">
+ Running Apps: {{model.selected}}
+ </div>
+ <div class="container-fluid" id="numapplications-donut-chart">
+ {{donut-chart data=model.selectedQueue.numOfApplicationsDonutChartData
+ showLabels=true
+ parentId="numapplications-donut-chart"
+ ratio=0.6
+ maxHeight=350}}
+ </div>
+ </div>
+ </div>
+
+</div>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dadb0c22/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/yarn-queue/fair-queue.hbs
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/yarn-queue/fair-queue.hbs b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/yarn-queue/fair-queue.hbs
new file mode 100644
index 0000000..0341108
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/yarn-queue/fair-queue.hbs
@@ -0,0 +1,63 @@
+{{!
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+}}
+
+{{queue-navigator model=model.queues selected=model.selected
+ used="usedResources.memory" max="clusterResources.memory"}}
+
+<div class="row">
+ <div class="col-lg-4 container-fluid">
+ <div class="panel panel-default">
+ <div class="panel-heading">
+ Queue Information: {{model.selected}}
+ </div>
+ {{yarn-queue.fair-queue-conf-table queue=model.selectedQueue}}
+ </div>
+ </div>
+
+ <div class="col-lg-4 container-fluid">
+ <div class="panel panel-default">
+ <div class="panel-heading">
+ Queue Capacities: {{model.selected}}
+ </div>
+ <div class="container-fluid" id="capacity-bar-chart">
+ <br/>
+ {{bar-chart data=model.selectedQueue.capacitiesBarChartData
+ title=""
+ parentId="capacity-bar-chart"
+ textWidth=150
+ ratio=0.55
+ maxHeight=350}}
+ </div>
+ </div>
+ </div>
+
+ <div class="col-lg-4 container-fluid">
+ <div class="panel panel-default">
+ <div class="panel-heading">
+ Running Apps: {{model.selected}}
+ </div>
+ <div class="container-fluid" id="numapplications-donut-chart">
+ {{donut-chart data=model.selectedQueue.numOfApplicationsDonutChartData
+ showLabels=true
+ parentId="numapplications-donut-chart"
+ ratio=0.6
+ maxHeight=350}}
+ </div>
+ </div>
+ </div>
+</div>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dadb0c22/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/yarn-queue/fifo-queue-conf-table.hbs
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/yarn-queue/fifo-queue-conf-table.hbs b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/yarn-queue/fifo-queue-conf-table.hbs
new file mode 100644
index 0000000..4ced3e7
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/yarn-queue/fifo-queue-conf-table.hbs
@@ -0,0 +1,56 @@
+{{!
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+}}
+
+<table id="queue-configuration-table" class="table table-striped table-bordered" cellspacing="0" width="100%" height="100%">
+ <thead>
+ <tr>
+ <td><b>Configurations</b></td>
+ <td>Value</td>
+ </tr>
+ </thead>
+ <tbody>
+ <tr>
+ <td>Configured Capacity</td>
+ <td>{{queue.capacity}}</td>
+ </tr>
+ <tr>
+ <td>Used Capacity</td>
+ <td>{{queue.usedCapacity}}</td>
+ </tr>
+ <tr>
+ <td>State</td>
+ <td>{{queue.state}}</td>
+ </tr>
+ <tr>
+ <td>Minimum Queue Memory Capacity</td>
+ <td>{{queue.minQueueMemoryCapacity}}</td>
+ </tr>
+ <tr>
+ <td>Maximum Queue Memory Capacity</td>
+ <td>{{queue.maxQueueMemoryCapacity}}</td>
+ </tr>
+ <tr>
+ <td>Number of Nodes</td>
+ <td>{{queue.numNodes}}</td>
+ </tr>
+ <tr>
+ <td>Number of Containers</td>
+ <td>{{queue.numContainers}}</td>
+ </tr>
+ </tbody>
+</table>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dadb0c22/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/yarn-queue/fifo-queue-info.hbs
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/yarn-queue/fifo-queue-info.hbs b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/yarn-queue/fifo-queue-info.hbs
new file mode 100644
index 0000000..7f4e8a7
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/yarn-queue/fifo-queue-info.hbs
@@ -0,0 +1,47 @@
+{{!
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+}}
+
+<div class="row">
+
+ <div class="col-lg-6 container-fluid">
+ <div class="panel panel-default">
+ <div class="panel-heading">
+ Queue Capacities: {{model.selected}}
+ </div>
+ <div class="container-fluid" id="capacity-bar-chart">
+ <br/>
+ {{bar-chart data=model.selectedQueue.capacitiesBarChartData
+ title=""
+ parentId="capacity-bar-chart"
+ textWidth=170
+ ratio=0.55
+ maxHeight=350}}
+ </div>
+ </div>
+ </div>
+
+ <div class="col-lg-6 container-fluid">
+ <div class="panel panel-default">
+ <div class="panel-heading">
+ Queue Information: {{model.selected}}
+ </div>
+ {{yarn-queue.fifo-queue-conf-table queue=model.selectedQueue}}
+ </div>
+ </div>
+
+</div>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dadb0c22/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/yarn-queue/fifo-queue.hbs
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/yarn-queue/fifo-queue.hbs b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/yarn-queue/fifo-queue.hbs
new file mode 100644
index 0000000..46d79f0
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/components/yarn-queue/fifo-queue.hbs
@@ -0,0 +1,48 @@
+{{!
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+}}
+
+{{queue-navigator model=model.queues selected=model.selected
+ used="usedNodeCapacity" max="totalNodeCapacity"}}
+
+<div class="row">
+ <div class="col-lg-6 container-fluid">
+ <div class="panel panel-default">
+ <div class="panel-heading">
+ Queue Information: {{model.selected}}
+ </div>
+ {{yarn-queue.fifo-queue-conf-table queue=model.selectedQueue}}
+ </div>
+ </div>
+
+ <div class="col-lg-6 container-fluid">
+ <div class="panel panel-default">
+ <div class="panel-heading">
+ Queue Capacities: {{model.selected}}
+ </div>
+ <div class="container-fluid" id="capacity-bar-chart">
+ <br/>
+ {{bar-chart data=model.selectedQueue.capacitiesBarChartData
+ title=""
+ parentId="capacity-bar-chart"
+ textWidth=150
+ ratio=0.55
+ maxHeight=350}}
+ </div>
+ </div>
+ </div>
+</div>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dadb0c22/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-queue/info.hbs
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-queue/info.hbs b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-queue/info.hbs
index c112ef9..2f138a7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-queue/info.hbs
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-queue/info.hbs
@@ -16,69 +16,10 @@
* limitations under the License.
}}
-<div class="row">
-
- <div class="col-lg-6 container-fluid">
- <div class="panel panel-default">
- <div class="panel-heading">
- Queue Capacities: {{model.selected}}
- </div>
- <div class="container-fluid" id="capacity-bar-chart">
- <br/>
- {{bar-chart data=model.selectedQueue.capacitiesBarChartData
- title=""
- parentId="capacity-bar-chart"
- textWidth=170
- ratio=0.55
- maxHeight=350}}
- </div>
- </div>
- </div>
-
- <div class="col-lg-6 container-fluid">
- <div class="panel panel-default">
- <div class="panel-heading">
- Queue Information: {{model.selected}}
- </div>
- {{queue-configuration-table queue=model.selectedQueue}}
- </div>
- </div>
-
-</div>
-
-<div class="row">
-
- <div class="col-lg-6 container-fluid">
- <div class="panel panel-default">
- <div class="panel-heading">
- Running Apps: {{model.selected}}
- </div>
- <div class="container-fluid" id="numapplications-donut-chart">
- {{donut-chart data=model.selectedQueue.numOfApplicationsDonutChartData
- showLabels=true
- parentId="numapplications-donut-chart"
- ratio=0.6
- maxHeight=350}}
- </div>
- </div>
- </div>
-
- {{#if model.selectedQueue.hasUserUsages}}
- <div class="col-lg-6 container-fluid">
- <div class="panel panel-default">
- <div class="panel-heading">
- User Usages: {{model.selected}}
- </div>
- <div class="container-fluid" id="userusage-donut-chart">
- {{donut-chart data=model.selectedQueue.userUsagesDonutChartData
- showLabels=true
- parentId="userusage-donut-chart"
- type="memory"
- ratio=0.6
- maxHeight=350}}
- </div>
- </div>
- </div>
- {{/if}}
-
-</div>
+{{#if (eq model.queues.firstObject.type "capacity")}}
+ {{yarn-queue.capacity-queue-info model=model}}
+{{else if (eq model.queues.firstObject.type "fair")}}
+ {{yarn-queue.fair-queue-info model=model}}
+{{else}}
+ {{yarn-queue.fifo-queue-info model=model}}
+{{/if}}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dadb0c22/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-queues.hbs
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-queues.hbs b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-queues.hbs
index 6dfb220..fccdb5b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-queues.hbs
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/templates/yarn-queues.hbs
@@ -17,54 +17,14 @@
}}
{{breadcrumb-bar breadcrumbs=breadcrumbs}}
-
<div class="container-fluid">
- {{queue-navigator model=model.queues selected=model.selected}}
-
- <div class="row">
-
- <div class="col-lg-4 container-fluid">
- <div class="panel panel-default">
- <div class="panel-heading">
- Queue Information: {{model.selected}}
- </div>
- {{queue-configuration-table queue=model.selectedQueue}}
- </div>
- </div>
-
- <div class="col-lg-4 container-fluid">
- <div class="panel panel-default">
- <div class="panel-heading">
- Queue Capacities: {{model.selected}}
- </div>
- <div class="container-fluid" id="capacity-bar-chart">
- <br/>
- {{bar-chart data=model.selectedQueue.capacitiesBarChartData
- title=""
- parentId="capacity-bar-chart"
- textWidth=150
- ratio=0.55
- maxHeight=350}}
- </div>
- </div>
- </div>
-
- <div class="col-lg-4 container-fluid">
- <div class="panel panel-default">
- <div class="panel-heading">
- Running Apps: {{model.selected}}
- </div>
- <div class="container-fluid" id="numapplications-donut-chart">
- {{donut-chart data=model.selectedQueue.numOfApplicationsDonutChartData
- showLabels=true
- parentId="numapplications-donut-chart"
- ratio=0.6
- maxHeight=350}}
- </div>
- </div>
- </div>
-
- </div>
+ {{#if (eq model.queues.firstObject.type "capacity")}}
+ {{yarn-queue.capacity-queue model=model}}
+ {{else if (eq model.queues.firstObject.type "fair")}}
+ {{yarn-queue.fair-queue model=model}}
+ {{else}}
+ {{yarn-queue.fifo-queue model=model}}
+ {{/if}}
</div>
{{outlet}}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dadb0c22/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/utils/color-utils.js
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/utils/color-utils.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/utils/color-utils.js
index 6c0cfee..af0cdf4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/utils/color-utils.js
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/app/utils/color-utils.js
@@ -55,7 +55,6 @@ export default {
}
}
- console.log(colors);
return colors;
},
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[18/50] [abbrv] hadoop git commit: HADOOP-14398. Modify documents for
the FileSystem Builder API. (Lei (Eddy) Xu)
Posted by as...@apache.org.
HADOOP-14398. Modify documents for the FileSystem Builder API. (Lei (Eddy) Xu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/99e558b1
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/99e558b1
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/99e558b1
Branch: refs/heads/YARN-5972
Commit: 99e558b13ba4d5832aea97374e1d07b4e78e5e39
Parents: 4230872
Author: Lei Xu <le...@apache.org>
Authored: Thu Aug 17 18:06:23 2017 -0700
Committer: Lei Xu <le...@apache.org>
Committed: Thu Aug 17 18:06:23 2017 -0700
----------------------------------------------------------------------
.../hadoop/fs/FSDataOutputStreamBuilder.java | 74 ++++++--
.../src/site/markdown/filesystem/filesystem.md | 33 +++-
.../filesystem/fsdataoutputstreambuilder.md | 182 +++++++++++++++++++
.../src/site/markdown/filesystem/index.md | 1 +
4 files changed, 272 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/99e558b1/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataOutputStreamBuilder.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataOutputStreamBuilder.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataOutputStreamBuilder.java
index 1f668eb..86c284a 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataOutputStreamBuilder.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FSDataOutputStreamBuilder.java
@@ -54,16 +54,29 @@ import static org.apache.hadoop.fs.CommonConfigurationKeysPublic.IO_FILE_BUFFER_
* options accordingly, for example:
*
* <code>
- * FSDataOutputStreamBuilder builder = fs.createFile(path);
- * builder.permission(perm)
+ *
+ * // Don't
+ * if (fs instanceof FooFileSystem) {
+ * FooFileSystem fs = (FooFileSystem) fs;
+ * OutputStream out = dfs.createFile(path)
+ * .optionA()
+ * .optionB("value")
+ * .cache()
+ * .build()
+ * } else if (fs instanceof BarFileSystem) {
+ * ...
+ * }
+ *
+ * // Do
+ * OutputStream out = fs.createFile(path)
+ * .permission(perm)
* .bufferSize(bufSize)
- * .opt("dfs.outputstream.builder.lazy-persist", true)
- * .opt("dfs.outputstream.builder.ec.policy-name", "rs-3-2-64k")
- * .opt("fs.local.o-direct", true)
- * .must("fs.s3a.fast-upload", true)
- * .must("fs.azure.buffer-size", 256 * 1024 * 1024);
- * FSDataOutputStream out = builder.build();
- * ...
+ * .opt("foofs:option.a", true)
+ * .opt("foofs:option.b", "value")
+ * .opt("barfs:cache", true)
+ * .must("foofs:cache", true)
+ * .must("barfs:cache-size", 256 * 1024 * 1024)
+ * .build();
* </code>
*
* If the option is not related to the file system, the option will be ignored.
@@ -263,6 +276,8 @@ public abstract class FSDataOutputStreamBuilder
/**
* Set optional boolean parameter for the Builder.
+ *
+ * @see #opt(String, String)
*/
public B opt(@Nonnull final String key, boolean value) {
mandatoryKeys.remove(key);
@@ -272,6 +287,8 @@ public abstract class FSDataOutputStreamBuilder
/**
* Set optional int parameter for the Builder.
+ *
+ * @see #opt(String, String)
*/
public B opt(@Nonnull final String key, int value) {
mandatoryKeys.remove(key);
@@ -281,6 +298,8 @@ public abstract class FSDataOutputStreamBuilder
/**
* Set optional float parameter for the Builder.
+ *
+ * @see #opt(String, String)
*/
public B opt(@Nonnull final String key, float value) {
mandatoryKeys.remove(key);
@@ -290,6 +309,8 @@ public abstract class FSDataOutputStreamBuilder
/**
* Set optional double parameter for the Builder.
+ *
+ * @see #opt(String, String)
*/
public B opt(@Nonnull final String key, double value) {
mandatoryKeys.remove(key);
@@ -299,6 +320,8 @@ public abstract class FSDataOutputStreamBuilder
/**
* Set an array of string values as optional parameter for the Builder.
+ *
+ * @see #opt(String, String)
*/
public B opt(@Nonnull final String key, @Nonnull final String... values) {
mandatoryKeys.remove(key);
@@ -310,8 +333,7 @@ public abstract class FSDataOutputStreamBuilder
* Set mandatory option to the Builder.
*
* If the option is not supported or unavailable on the {@link FileSystem},
- * the client should expect {@link #build()} throws
- * {@link IllegalArgumentException}.
+ * the client should expect {@link #build()} throws IllegalArgumentException.
*/
public B must(@Nonnull final String key, @Nonnull final String value) {
mandatoryKeys.add(key);
@@ -319,35 +341,55 @@ public abstract class FSDataOutputStreamBuilder
return getThisBuilder();
}
- /** Set mandatory boolean option. */
+ /**
+ * Set mandatory boolean option.
+ *
+ * @see #must(String, String)
+ */
public B must(@Nonnull final String key, boolean value) {
mandatoryKeys.add(key);
options.setBoolean(key, value);
return getThisBuilder();
}
- /** Set mandatory int option. */
+ /**
+ * Set mandatory int option.
+ *
+ * @see #must(String, String)
+ */
public B must(@Nonnull final String key, int value) {
mandatoryKeys.add(key);
options.setInt(key, value);
return getThisBuilder();
}
- /** Set mandatory float option. */
+ /**
+ * Set mandatory float option.
+ *
+ * @see #must(String, String)
+ */
public B must(@Nonnull final String key, float value) {
mandatoryKeys.add(key);
options.setFloat(key, value);
return getThisBuilder();
}
- /** Set mandatory double option. */
+ /**
+ * Set mandatory double option.
+ *
+ * @see #must(String, String)
+ */
public B must(@Nonnull final String key, double value) {
mandatoryKeys.add(key);
options.setDouble(key, value);
return getThisBuilder();
}
- /** Set a string array as mandatory option. */
+ /**
+ * Set a string array as mandatory option.
+ *
+ * @see #must(String, String)
+ */
public B must(@Nonnull final String key, @Nonnull final String... values) {
mandatoryKeys.add(key);
options.setStrings(key, values);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/99e558b1/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md
index d7e57ce..1e522c7 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/filesystem.md
@@ -553,7 +553,7 @@ on a path that exists and is a file. Instead the operation returns false.
FS' = FS
result = False
-### `FSDataOutputStream create(Path, ...)`
+### <a name='FileSystem.create'></a> `FSDataOutputStream create(Path, ...)`
FSDataOutputStream create(Path p,
@@ -616,7 +616,24 @@ this precondition fails.
* Not covered: symlinks. The resolved path of the symlink is used as the final path argument to the `create()` operation
-### `FSDataOutputStream append(Path p, int bufferSize, Progressable progress)`
+### `FSDataOutputStreamBuilder createFile(Path p)`
+
+Make a `FSDataOutputStreamBuilder` to specify the parameters to create a file.
+
+#### Implementation Notes
+
+`createFile(p)` returns a `FSDataOutputStreamBuilder` only and does not make
+change on filesystem immediately. When `build()` is invoked on the `FSDataOutputStreamBuilder`,
+the builder parameters are verified and [`create(Path p)`](#FileSystem.create)
+is invoked on the underlying filesystem. `build()` has the same preconditions
+and postconditions as [`create(Path p)`](#FileSystem.create).
+
+* Similar to [`create(Path p)`](#FileSystem.create), files are overwritten
+by default, unless specify `builder.overwrite(false)`.
+* Unlike [`create(Path p)`](#FileSystem.create), missing parent directories are
+not created by default, unless specify `builder.recursive()`.
+
+### <a name='FileSystem.append'></a> `FSDataOutputStream append(Path p, int bufferSize, Progressable progress)`
Implementations without a compliant call SHOULD throw `UnsupportedOperationException`.
@@ -634,6 +651,18 @@ Implementations without a compliant call SHOULD throw `UnsupportedOperationExcep
Return: `FSDataOutputStream`, which can update the entry `FS.Files[p]`
by appending data to the existing list.
+### `FSDataOutputStreamBuilder appendFile(Path p)`
+
+Make a `FSDataOutputStreamBuilder` to specify the parameters to append to an
+existing file.
+
+#### Implementation Notes
+
+`appendFile(p)` returns a `FSDataOutputStreamBuilder` only and does not make
+change on filesystem immediately. When `build()` is invoked on the `FSDataOutputStreamBuilder`,
+the builder parameters are verified and [`append()`](#FileSystem.append) is
+invoked on the underlying filesystem. `build()` has the same preconditions and
+postconditions as [`append()`](#FileSystem.append).
### `FSDataInputStream open(Path f, int bufferSize)`
http://git-wip-us.apache.org/repos/asf/hadoop/blob/99e558b1/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/fsdataoutputstreambuilder.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/fsdataoutputstreambuilder.md b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/fsdataoutputstreambuilder.md
new file mode 100644
index 0000000..4ea1fd1
--- /dev/null
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/fsdataoutputstreambuilder.md
@@ -0,0 +1,182 @@
+<!---
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+
+<!-- ============================================================= -->
+<!-- CLASS: FSDataOutputStreamBuilder -->
+<!-- ============================================================= -->
+
+# class `org.apache.hadoop.fs.FSDataOutputStreamBuilder`
+
+<!-- MACRO{toc|fromDepth=1|toDepth=2} -->
+
+Builder pattern for `FSDataOutputStream` and its subclasses. It is used to
+create a new file or open an existing file on `FileSystem` for write.
+
+## Invariants
+
+The `FSDataOutputStreamBuilder` interface does not validate parameters
+and modify the state of `FileSystem` until [`build()`](#Builder.build) is
+invoked.
+
+## Implementation-agnostic parameters.
+
+### <a name="Builder.create"></a> `FSDataOutputStreamBuilder create()`
+
+Specify `FSDataOutputStreamBuilder` to create a file on `FileSystem`, equivalent
+to `CreateFlag#CREATE`.
+
+### <a name="Builder.append"></a> `FSDataOutputStreamBuilder append()`
+
+Specify `FSDataOutputStreamBuilder` to append to an existing file on
+`FileSystem`, equivalent to `CreateFlag#APPEND`.
+
+### <a name="Builder.overwrite"></a> `FSDataOutputStreamBuilder overwrite(boolean overwrite)`
+
+Specify `FSDataOutputStreamBuilder` to overwrite an existing file or not. If
+giving `overwrite==true`, it truncates an existing file, equivalent to
+`CreateFlag#OVERWITE`.
+
+### <a name="Builder.permission"></a> `FSDataOutputStreamBuilder permission(FsPermission permission)`
+
+Set permission for the file.
+
+### <a name="Builder.bufferSize"></a> `FSDataOutputStreamBuilder bufferSize(int bufSize)`
+
+Set the size of the buffer to be used.
+
+### <a name="Builder.replication"></a> `FSDataOutputStreamBuilder replication(short replica)`
+
+Set the replication factor.
+
+### <a name="Builder.blockSize"></a> `FSDataOutputStreamBuilder blockSize(long size)`
+
+Set block size in bytes.
+
+### <a name="Builder.recursive"></a> `FSDataOutputStreamBuilder recursive()`
+
+Create parent directories if they do not exist.
+
+### <a name="Builder.progress"></a> `FSDataOutputStreamBuilder progress(Progresable prog)`
+
+Set the facility of reporting progress.
+
+### <a name="Builder.checksumOpt"></a> `FSDataOutputStreamBuilder checksumOpt(ChecksumOpt chksumOpt)`
+
+Set checksum opt.
+
+### Set optional or mandatory parameters
+
+ FSDataOutputStreamBuilder opt(String key, ...)
+ FSDataOutputStreamBuilder must(String key, ...)
+
+Set optional or mandatory parameters to the builder. Using `opt()` or `must()`,
+client can specify FS-specific parameters without inspecting the concrete type
+of `FileSystem`.
+
+ // Don't
+ if (fs instanceof FooFileSystem) {
+ FooFileSystem fs = (FooFileSystem) fs;
+ out = dfs.createFile(path)
+ .optionA()
+ .optionB("value")
+ .cache()
+ .build()
+ } else if (fs instanceof BarFileSystem) {
+ ...
+ }
+
+ // Do
+ out = fs.createFile(path)
+ .permission(perm)
+ .bufferSize(bufSize)
+ .opt("foofs:option.a", true)
+ .opt("foofs:option.b", "value")
+ .opt("barfs:cache", true)
+ .must("foofs:cache", true)
+ .must("barfs:cache-size", 256 * 1024 * 1024)
+ .build();
+
+#### Implementation Notes
+
+The concrete `FileSystem` and/or `FSDataOutputStreamBuilder` implementation
+MUST verify that implementation-agnostic parameters (i.e., "syncable") or
+implementation-specific parameters (i.e., "foofs:cache")
+are supported. `FileSystem` will satisfy optional parameters (via `opt(key, ...)`)
+on best effort. If the mandatory parameters (via `must(key, ...)`) can not be satisfied
+in the `FileSystem`, `IllegalArgumentException` should be thrown in `build()`.
+
+The behavior of resolving the conflicts between the parameters set by
+builder methods (i.e., `bufferSize()`) and `opt()`/`must()` is undefined.
+
+## HDFS-specific parameters.
+
+`HdfsDataOutputStreamBuilder extends FSDataOutputStreamBuilder` provides additional
+HDFS-specific parameters, for further customize file creation / append behavior.
+
+### `FSDataOutpuStreamBuilder favoredNodes(InetSocketAddress[] nodes)`
+
+Set favored DataNodes for new blocks.
+
+### `FSDataOutputStreamBuilder syncBlock()`
+
+Force closed blocks to the disk device. See `CreateFlag#SYNC_BLOCK`
+
+### `FSDataOutputStreamBuilder lazyPersist()`
+
+Create the block on transient storage if possible.
+
+### `FSDataOutputStreamBuilder newBlock()`
+
+Append data to a new block instead of the end of the last partial block.
+
+### `FSDataOutputStreamBuilder noLocalWrite()`
+
+Advise that a block replica NOT be written to the local DataNode.
+
+### `FSDataOutputStreamBuilder ecPolicyName()`
+
+Enforce the file to be a striped file with erasure coding policy 'policyName',
+no matter what its parent directory's replication or erasure coding policy is.
+
+### `FSDataOutputStreamBuilder replicate()`
+
+Enforce the file to be a replicated file, no matter what its parent directory's
+replication or erasure coding policy is.
+
+## Builder interface
+
+### <a name="Builder.build"></a> `FSDataOutputStream build()`
+
+Create a new file or append an existing file on the underlying `FileSystem`,
+and return `FSDataOutputStream` for write.
+
+#### Preconditions
+
+The following combinations of parameters are not supported:
+
+ if APPEND|OVERWRITE: raise HadoopIllegalArgumentException
+ if CREATE|APPEND|OVERWRITE: raise HadoopIllegalArgumentExdeption
+
+`FileSystem` may reject the request for other reasons and throw `IOException`,
+see `FileSystem#create(path, ...)` and `FileSystem#append()`.
+
+#### Postconditions
+
+ FS' where :
+ FS'.Files'[p] == []
+ ancestors(p) is-subset-of FS'.Directories'
+
+ result = FSDataOutputStream
+
+The result is `FSDataOutputStream` to be used to write data to filesystem.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/99e558b1/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/index.md
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/index.md b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/index.md
index 66a7eb3..532b6c7 100644
--- a/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/index.md
+++ b/hadoop-common-project/hadoop-common/src/site/markdown/filesystem/index.md
@@ -33,5 +33,6 @@ HDFS as these are commonly expected by Hadoop client applications.
1. [Model](model.html)
1. [FileSystem class](filesystem.html)
1. [FSDataInputStream class](fsdatainputstream.html)
+1. [FSDataOutputStreamBuilder class](fsdataoutputstreambuilder.html)
2. [Testing with the Filesystem specification](testing.html)
2. [Extending the specification and its tests](extending.html)
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[15/50] [abbrv] hadoop git commit: YARN-6988. container-executor
fails for docker when command length > 4096 B. Contributed by Eric Badger
Posted by as...@apache.org.
YARN-6988. container-executor fails for docker when command length > 4096 B. Contributed by Eric Badger
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ab1a8ae8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ab1a8ae8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ab1a8ae8
Branch: refs/heads/YARN-5972
Commit: ab1a8ae85f8c61304a0f437cdc61cc5aeda36a4b
Parents: dd7916d
Author: Jason Lowe <jl...@apache.org>
Authored: Thu Aug 17 15:50:14 2017 -0500
Committer: Jason Lowe <jl...@apache.org>
Committed: Thu Aug 17 15:50:14 2017 -0500
----------------------------------------------------------------------
.../impl/container-executor.c | 38 +++++++++++++-------
.../main/native/container-executor/impl/util.h | 7 ++++
2 files changed, 33 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ab1a8ae8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
index 9f754c4..7361808 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
@@ -1417,9 +1417,10 @@ int run_docker(const char *command_file) {
char* docker_command = parse_docker_command_file(command_file);
char* docker_binary = get_section_value(DOCKER_BINARY_KEY, &executor_cfg);
docker_binary = check_docker_binary(docker_binary);
+ size_t command_size = MIN(sysconf(_SC_ARG_MAX), 128*1024);
- char* docker_command_with_binary = calloc(sizeof(char), EXECUTOR_PATH_MAX);
- snprintf(docker_command_with_binary, EXECUTOR_PATH_MAX, "%s %s", docker_binary, docker_command);
+ char* docker_command_with_binary = calloc(sizeof(char), command_size);
+ snprintf(docker_command_with_binary, command_size, "%s %s", docker_binary, docker_command);
char **args = split_delimiter(docker_command_with_binary, " ");
int exit_code = -1;
@@ -1567,16 +1568,24 @@ int launch_docker_container_as_user(const char * user, const char *app_id,
char *script_file_dest = NULL;
char *cred_file_dest = NULL;
char *exit_code_file = NULL;
- char docker_command_with_binary[EXECUTOR_PATH_MAX];
- char docker_wait_command[EXECUTOR_PATH_MAX];
- char docker_logs_command[EXECUTOR_PATH_MAX];
- char docker_inspect_command[EXECUTOR_PATH_MAX];
- char docker_rm_command[EXECUTOR_PATH_MAX];
+ char *docker_command_with_binary = NULL;
+ char *docker_wait_command = NULL;
+ char *docker_logs_command = NULL;
+ char *docker_inspect_command = NULL;
+ char *docker_rm_command = NULL;
int container_file_source =-1;
int cred_file_source = -1;
int BUFFER_SIZE = 4096;
char buffer[BUFFER_SIZE];
+ size_t command_size = MIN(sysconf(_SC_ARG_MAX), 128*1024);
+
+ docker_command_with_binary = calloc(sizeof(char), command_size);
+ docker_wait_command = calloc(sizeof(char), command_size);
+ docker_logs_command = calloc(sizeof(char), command_size);
+ docker_inspect_command = calloc(sizeof(char), command_size);
+ docker_rm_command = calloc(sizeof(char), command_size);
+
gid_t user_gid = getegid();
uid_t prev_uid = geteuid();
@@ -1621,7 +1630,7 @@ int launch_docker_container_as_user(const char * user, const char *app_id,
goto cleanup;
}
- snprintf(docker_command_with_binary, EXECUTOR_PATH_MAX, "%s %s", docker_binary, docker_command);
+ snprintf(docker_command_with_binary, command_size, "%s %s", docker_binary, docker_command);
fprintf(LOGFILE, "Launching docker container...\n");
FILE* start_docker = popen(docker_command_with_binary, "r");
@@ -1634,7 +1643,7 @@ int launch_docker_container_as_user(const char * user, const char *app_id,
goto cleanup;
}
- snprintf(docker_inspect_command, EXECUTOR_PATH_MAX,
+ snprintf(docker_inspect_command, command_size,
"%s inspect --format {{.State.Pid}} %s",
docker_binary, container_id);
@@ -1679,7 +1688,7 @@ int launch_docker_container_as_user(const char * user, const char *app_id,
goto cleanup;
}
- snprintf(docker_wait_command, EXECUTOR_PATH_MAX,
+ snprintf(docker_wait_command, command_size,
"%s wait %s", docker_binary, container_id);
fprintf(LOGFILE, "Waiting for docker container to finish...\n");
@@ -1693,7 +1702,7 @@ int launch_docker_container_as_user(const char * user, const char *app_id,
if(exit_code != 0) {
fprintf(ERRORFILE, "Docker container exit code was not zero: %d\n",
exit_code);
- snprintf(docker_logs_command, EXECUTOR_PATH_MAX, "%s logs --tail=250 %s",
+ snprintf(docker_logs_command, command_size, "%s logs --tail=250 %s",
docker_binary, container_id);
FILE* logs = popen(docker_logs_command, "r");
if(logs != NULL) {
@@ -1723,7 +1732,7 @@ int launch_docker_container_as_user(const char * user, const char *app_id,
}
fprintf(LOGFILE, "Removing docker container post-exit...\n");
- snprintf(docker_rm_command, EXECUTOR_PATH_MAX,
+ snprintf(docker_rm_command, command_size,
"%s rm %s", docker_binary, container_id);
FILE* rm_docker = popen(docker_rm_command, "w");
if (pclose (rm_docker) != 0)
@@ -1763,6 +1772,11 @@ cleanup:
free(exit_code_file);
free(script_file_dest);
free(cred_file_dest);
+ free(docker_command_with_binary);
+ free(docker_wait_command);
+ free(docker_logs_command);
+ free(docker_inspect_command);
+ free(docker_rm_command);
return exit_code;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ab1a8ae8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.h
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.h b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.h
index a8a12a9..fa21def 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.h
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/util.h
@@ -66,6 +66,13 @@ enum errorcodes {
ERROR_COMPILING_REGEX = 42
};
+/* Macros for min/max. */
+#ifndef MIN
+#define MIN(a,b) (((a)<(b))?(a):(b))
+#endif /* MIN */
+#ifndef MAX
+#define MAX(a,b) (((a)>(b))?(a):(b))
+#endif /* MAX */
// the log file for messages
extern FILE *LOGFILE;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[37/50] [abbrv] hadoop git commit: YARN-2416.
InvalidStateTransitonException in ResourceManager if AMLauncher does not
receive response for startContainers() call in time. Contributed by Jonathan
Eagles
Posted by as...@apache.org.
YARN-2416. InvalidStateTransitonException in ResourceManager if AMLauncher does not receive response for startContainers() call in time. Contributed by Jonathan Eagles
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/3efcd51c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/3efcd51c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/3efcd51c
Branch: refs/heads/YARN-5972
Commit: 3efcd51c3b3eb667d83e08b500bb7a7ea559fabe
Parents: 4ec5acc
Author: Jason Lowe <jl...@apache.org>
Authored: Tue Aug 22 12:56:09 2017 -0500
Committer: Jason Lowe <jl...@apache.org>
Committed: Tue Aug 22 12:56:09 2017 -0500
----------------------------------------------------------------------
.../rmapp/attempt/RMAppAttemptImpl.java | 25 ++++++++++++---
.../attempt/TestRMAppAttemptTransitions.java | 32 +++++++++++++-------
2 files changed, 41 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3efcd51c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
index 7d453bd..d748860 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptImpl.java
@@ -184,7 +184,10 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
new ExpiredTransition();
private static final AttemptFailedTransition FAILED_TRANSITION =
new AttemptFailedTransition();
-
+ private static final AMRegisteredTransition REGISTERED_TRANSITION =
+ new AMRegisteredTransition();
+ private static final AMLaunchedTransition LAUNCHED_TRANSITION =
+ new AMLaunchedTransition();
private RMAppAttemptEvent eventCausingFinalSaving;
private RMAppAttemptState targetedFinalState;
private RMAppAttemptState recoveredFinalState;
@@ -314,7 +317,7 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
// Transitions from ALLOCATED State
.addTransition(RMAppAttemptState.ALLOCATED, RMAppAttemptState.LAUNCHED,
- RMAppAttemptEventType.LAUNCHED, new AMLaunchedTransition())
+ RMAppAttemptEventType.LAUNCHED, LAUNCHED_TRANSITION)
.addTransition(RMAppAttemptState.ALLOCATED, RMAppAttemptState.FINAL_SAVING,
RMAppAttemptEventType.LAUNCH_FAILED,
new FinalSavingTransition(new LaunchFailedTransition(),
@@ -328,6 +331,8 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
RMAppAttemptEventType.FAIL,
new FinalSavingTransition(FAILED_TRANSITION,
RMAppAttemptState.FAILED))
+ .addTransition(RMAppAttemptState.ALLOCATED, RMAppAttemptState.RUNNING,
+ RMAppAttemptEventType.REGISTERED, REGISTERED_TRANSITION)
.addTransition(RMAppAttemptState.ALLOCATED, RMAppAttemptState.FINAL_SAVING,
RMAppAttemptEventType.CONTAINER_FINISHED,
new FinalSavingTransition(
@@ -335,7 +340,7 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
// Transitions from LAUNCHED State
.addTransition(RMAppAttemptState.LAUNCHED, RMAppAttemptState.RUNNING,
- RMAppAttemptEventType.REGISTERED, new AMRegisteredTransition())
+ RMAppAttemptEventType.REGISTERED, REGISTERED_TRANSITION)
.addTransition(RMAppAttemptState.LAUNCHED,
EnumSet.of(RMAppAttemptState.LAUNCHED, RMAppAttemptState.FINAL_SAVING),
RMAppAttemptEventType.CONTAINER_FINISHED,
@@ -357,6 +362,8 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
RMAppAttemptState.FAILED))
// Transitions from RUNNING State
+ .addTransition(RMAppAttemptState.RUNNING, RMAppAttemptState.RUNNING,
+ RMAppAttemptEventType.LAUNCHED)
.addTransition(RMAppAttemptState.RUNNING, RMAppAttemptState.FINAL_SAVING,
RMAppAttemptEventType.UNREGISTERED, new AMUnregisteredTransition())
.addTransition(RMAppAttemptState.RUNNING, RMAppAttemptState.RUNNING,
@@ -421,6 +428,7 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
RMAppAttemptState.FAILED,
RMAppAttemptState.FAILED,
EnumSet.of(
+ RMAppAttemptEventType.LAUNCHED,
RMAppAttemptEventType.EXPIRE,
RMAppAttemptEventType.KILL,
RMAppAttemptEventType.FAIL,
@@ -438,6 +446,7 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
new FinalTransition(RMAppAttemptState.FINISHED))
.addTransition(RMAppAttemptState.FINISHING, RMAppAttemptState.FINISHING,
EnumSet.of(
+ RMAppAttemptEventType.LAUNCHED,
RMAppAttemptEventType.UNREGISTERED,
RMAppAttemptEventType.STATUS_UPDATE,
RMAppAttemptEventType.CONTAINER_ALLOCATED,
@@ -451,6 +460,7 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
RMAppAttemptState.FINISHED,
RMAppAttemptState.FINISHED,
EnumSet.of(
+ RMAppAttemptEventType.LAUNCHED,
RMAppAttemptEventType.EXPIRE,
RMAppAttemptEventType.UNREGISTERED,
RMAppAttemptEventType.CONTAINER_ALLOCATED,
@@ -1291,7 +1301,7 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
* 2) OR AMLivelinessMonitor expires this attempt (when am doesn't
* heart beat back).
*/
- (new AMLaunchedTransition()).transition(appAttempt, event);
+ LAUNCHED_TRANSITION.transition(appAttempt, event);
return RMAppAttemptState.LAUNCHED;
}
}
@@ -1516,7 +1526,8 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
@Override
public void transition(RMAppAttemptImpl appAttempt,
RMAppAttemptEvent event) {
- if (event.getType() == RMAppAttemptEventType.LAUNCHED) {
+ if (event.getType() == RMAppAttemptEventType.LAUNCHED
+ || event.getType() == RMAppAttemptEventType.REGISTERED) {
appAttempt.launchAMEndTime = System.currentTimeMillis();
long delay = appAttempt.launchAMEndTime -
appAttempt.launchAMStartTime;
@@ -1651,6 +1662,10 @@ public class RMAppAttemptImpl implements RMAppAttempt, Recoverable {
@Override
public void transition(RMAppAttemptImpl appAttempt,
RMAppAttemptEvent event) {
+ if (!RMAppAttemptState.LAUNCHED.equals(appAttempt.getState())) {
+ // registered received before launch
+ LAUNCHED_TRANSITION.transition(appAttempt, event);
+ }
long delay = System.currentTimeMillis() - appAttempt.launchAMEndTime;
ClusterMetrics.getMetrics().addAMRegisterDelay(delay);
RMAppAttemptRegistrationEvent registrationEvent
http://git-wip-us.apache.org/repos/asf/hadoop/blob/3efcd51c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
index 7702ab1..f6406ff 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
@@ -526,12 +526,9 @@ public class TestRMAppAttemptTransitions {
verifyApplicationAttemptFinished(RMAppAttemptState.FAILED);
}
- /**
- * {@link RMAppAttemptState#LAUNCHED}
- */
- private void testAppAttemptLaunchedState(Container container) {
- assertEquals(RMAppAttemptState.LAUNCHED,
- applicationAttempt.getAppAttemptState());
+ private void testAppAttemptLaunchedState(Container container,
+ RMAppAttemptState state) {
+ assertEquals(state, applicationAttempt.getAppAttemptState());
assertEquals(container, applicationAttempt.getMasterContainer());
if (UserGroupInformation.isSecurityEnabled()) {
// ClientTokenMasterKey has been registered in SecretManager, it's able to
@@ -686,13 +683,18 @@ public class TestRMAppAttemptTransitions {
}
private void launchApplicationAttempt(Container container) {
+ launchApplicationAttempt(container, RMAppAttemptState.LAUNCHED);
+ }
+
+ private void launchApplicationAttempt(Container container,
+ RMAppAttemptState state) {
applicationAttempt.handle(
- new RMAppAttemptEvent(applicationAttempt.getAppAttemptId(),
+ new RMAppAttemptEvent(applicationAttempt.getAppAttemptId(),
RMAppAttemptEventType.LAUNCHED));
- testAppAttemptLaunchedState(container);
+ testAppAttemptLaunchedState(container, state);
}
-
+
private void runApplicationAttempt(Container container,
String host,
int rpcPort,
@@ -723,7 +725,7 @@ public class TestRMAppAttemptTransitions {
when(submissionContext.getUnmanagedAM()).thenReturn(true);
// submit AM and check it goes to LAUNCHED state
scheduleApplicationAttempt();
- testAppAttemptLaunchedState(null);
+ testAppAttemptLaunchedState(null, RMAppAttemptState.LAUNCHED);
verify(amLivelinessMonitor, times(1)).register(
applicationAttempt.getAppAttemptId());
@@ -930,7 +932,15 @@ public class TestRMAppAttemptTransitions {
applicationAttempt.createApplicationAttemptState());
testAppAttemptFailedState(amContainer, diagnostics);
}
-
+
+ @Test(timeout = 10000)
+ public void testAllocatedToRunning() {
+ Container amContainer = allocateApplicationAttempt();
+ // Register attempt event arrives before launched attempt event
+ runApplicationAttempt(amContainer, "host", 8042, "oldtrackingurl", false);
+ launchApplicationAttempt(amContainer, RMAppAttemptState.RUNNING);
+ }
+
@Test(timeout = 10000)
public void testCreateAppAttemptReport() {
RMAppAttemptState[] attemptStates = RMAppAttemptState.values();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[35/50] [abbrv] hadoop git commit: HADOOP-14787. AliyunOSS: Implement
the `createNonRecursive` operator. Contributed by Genmao Yu
Posted by as...@apache.org.
HADOOP-14787. AliyunOSS: Implement the `createNonRecursive` operator.
Contributed by Genmao Yu
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/27ab5f73
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/27ab5f73
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/27ab5f73
Branch: refs/heads/YARN-5972
Commit: 27ab5f7385c70f16fd593edc336c573c69f19331
Parents: d5ff57a
Author: Steve Loughran <st...@apache.org>
Authored: Tue Aug 22 11:55:48 2017 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Tue Aug 22 11:55:48 2017 +0100
----------------------------------------------------------------------
.../fs/aliyun/oss/AliyunOSSFileSystem.java | 27 ++++++++++++++++++++
1 file changed, 27 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/27ab5f73/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java
index 0491087..3561b02 100644
--- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java
+++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystem.java
@@ -22,11 +22,13 @@ import java.io.FileNotFoundException;
import java.io.IOException;
import java.net.URI;
import java.util.ArrayList;
+import java.util.EnumSet;
import java.util.List;
import org.apache.commons.collections.CollectionUtils;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CreateFlag;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileAlreadyExistsException;
@@ -103,6 +105,31 @@ public class AliyunOSSFileSystem extends FileSystem {
store, key, progress, statistics), (Statistics)(null));
}
+ /**
+ * {@inheritDoc}
+ * @throws FileNotFoundException if the parent directory is not present -or
+ * is not a directory.
+ */
+ @Override
+ public FSDataOutputStream createNonRecursive(Path path,
+ FsPermission permission,
+ EnumSet<CreateFlag> flags,
+ int bufferSize,
+ short replication,
+ long blockSize,
+ Progressable progress) throws IOException {
+ Path parent = path.getParent();
+ if (parent != null) {
+ // expect this to raise an exception if there is no parent
+ if (!getFileStatus(parent).isDirectory()) {
+ throw new FileAlreadyExistsException("Not a directory: " + parent);
+ }
+ }
+ return create(path, permission,
+ flags.contains(CreateFlag.OVERWRITE), bufferSize,
+ replication, blockSize, progress);
+ }
+
@Override
public boolean delete(Path path, boolean recursive) throws IOException {
try {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[46/50] [abbrv] hadoop git commit: HDFS-10899. Add functionality to
re-encrypt EDEKs.
Posted by as...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1000a2af/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirEncryptionZoneOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirEncryptionZoneOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirEncryptionZoneOp.java
index 22039d1..2552cf5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirEncryptionZoneOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirEncryptionZoneOp.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.hdfs.server.namenode;
import static org.apache.hadoop.hdfs.server.common.HdfsServerConstants.CRYPTO_XATTR_FILE_ENCRYPTION_INFO;
+import java.io.FileNotFoundException;
import java.io.IOException;
import java.security.GeneralSecurityException;
import java.security.PrivilegedExceptionAction;
@@ -31,6 +32,7 @@ import java.util.Map;
import org.apache.hadoop.crypto.CipherSuite;
import org.apache.hadoop.crypto.CryptoProtocolVersion;
import org.apache.hadoop.crypto.key.KeyProvider;
+import org.apache.hadoop.crypto.key.KeyProvider.KeyVersion;
import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension;
import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension.EncryptedKeyVersion;
import org.apache.hadoop.fs.FileEncryptionInfo;
@@ -42,15 +44,22 @@ import org.apache.hadoop.fs.BatchedRemoteIterator.BatchedListEntries;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.hdfs.XAttrHelper;
import org.apache.hadoop.hdfs.protocol.EncryptionZone;
+import org.apache.hadoop.hdfs.protocol.ZoneReencryptionStatus;
import org.apache.hadoop.hdfs.protocol.SnapshotAccessControlException;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos;
+import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.ReencryptionInfoProto;
+import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.ZoneEncryptionInfoProto;
import org.apache.hadoop.hdfs.protocolPB.PBHelperClient;
import org.apache.hadoop.hdfs.server.namenode.FSDirectory.DirOp;
+import org.apache.hadoop.hdfs.server.namenode.ReencryptionUpdater.FileEdekInfo;
import org.apache.hadoop.security.SecurityUtil;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
import com.google.protobuf.InvalidProtocolBufferException;
+import org.apache.hadoop.util.Time;
+
+import static org.apache.hadoop.hdfs.server.common.HdfsServerConstants.CRYPTO_XATTR_ENCRYPTION_ZONE;
import static org.apache.hadoop.util.Time.monotonicNow;
/**
@@ -216,18 +225,206 @@ final class FSDirEncryptionZoneOp {
}
}
+ static void reencryptEncryptionZone(final FSDirectory fsd,
+ final String zone, final String keyVersionName,
+ final boolean logRetryCache) throws IOException {
+ final List<XAttr> xAttrs = Lists.newArrayListWithCapacity(1);
+ final FSPermissionChecker pc = fsd.getPermissionChecker();
+ fsd.writeLock();
+ try {
+ final INodesInPath iip = fsd.resolvePath(pc, zone, DirOp.WRITE);
+ final XAttr xattr = fsd.ezManager
+ .reencryptEncryptionZone(iip, keyVersionName);
+ xAttrs.add(xattr);
+ } finally {
+ fsd.writeUnlock();
+ }
+ fsd.getEditLog().logSetXAttrs(zone, xAttrs, logRetryCache);
+ }
+
+ static void cancelReencryptEncryptionZone(final FSDirectory fsd,
+ final String zone, final boolean logRetryCache) throws IOException {
+ final List<XAttr> xattrs;
+ final FSPermissionChecker pc = fsd.getPermissionChecker();
+ fsd.writeLock();
+ try {
+ final INodesInPath iip = fsd.resolvePath(pc, zone, DirOp.WRITE);
+ xattrs = fsd.ezManager.cancelReencryptEncryptionZone(iip);
+ } finally {
+ fsd.writeUnlock();
+ }
+ if (xattrs != null && !xattrs.isEmpty()) {
+ fsd.getEditLog().logSetXAttrs(zone, xattrs, logRetryCache);
+ }
+ }
+
+ static BatchedListEntries<ZoneReencryptionStatus> listReencryptionStatus(
+ final FSDirectory fsd, final long prevId)
+ throws IOException {
+ fsd.readLock();
+ try {
+ return fsd.ezManager.listReencryptionStatus(prevId);
+ } finally {
+ fsd.readUnlock();
+ }
+ }
+
+ /**
+ * Update re-encryption progress (submitted). Caller should
+ * logSync after calling this, outside of the FSN lock.
+ * <p>
+ * The reencryption status is updated during SetXAttrs.
+ */
+ static XAttr updateReencryptionSubmitted(final FSDirectory fsd,
+ final INodesInPath iip, final String ezKeyVersionName)
+ throws IOException {
+ assert fsd.hasWriteLock();
+ Preconditions.checkNotNull(ezKeyVersionName, "ezKeyVersionName is null.");
+ final ZoneEncryptionInfoProto zoneProto = getZoneEncryptionInfoProto(iip);
+ Preconditions.checkNotNull(zoneProto, "ZoneEncryptionInfoProto is null.");
+
+ final ReencryptionInfoProto newProto = PBHelperClient
+ .convert(ezKeyVersionName, Time.now(), false, 0, 0, null, null);
+ final ZoneEncryptionInfoProto newZoneProto = PBHelperClient
+ .convert(PBHelperClient.convert(zoneProto.getSuite()),
+ PBHelperClient.convert(zoneProto.getCryptoProtocolVersion()),
+ zoneProto.getKeyName(), newProto);
+
+ final XAttr xattr = XAttrHelper
+ .buildXAttr(CRYPTO_XATTR_ENCRYPTION_ZONE, newZoneProto.toByteArray());
+ final List<XAttr> xattrs = Lists.newArrayListWithCapacity(1);
+ xattrs.add(xattr);
+ FSDirXAttrOp.unprotectedSetXAttrs(fsd, iip, xattrs,
+ EnumSet.of(XAttrSetFlag.REPLACE));
+ return xattr;
+ }
+
+ /**
+ * Update re-encryption progress (start, checkpoint). Caller should
+ * logSync after calling this, outside of the FSN lock.
+ * <p>
+ * The reencryption status is updated during SetXAttrs.
+ * Original reencryption status is passed in to get existing information
+ * such as ezkeyVersionName and submissionTime.
+ */
+ static XAttr updateReencryptionProgress(final FSDirectory fsd,
+ final INode zoneNode, final ZoneReencryptionStatus origStatus,
+ final String lastFile, final long numReencrypted, final long numFailures)
+ throws IOException {
+ assert fsd.hasWriteLock();
+ Preconditions.checkNotNull(zoneNode, "Zone node is null");
+ INodesInPath iip = INodesInPath.fromINode(zoneNode);
+ final ZoneEncryptionInfoProto zoneProto = getZoneEncryptionInfoProto(iip);
+ Preconditions.checkNotNull(zoneProto, "ZoneEncryptionInfoProto is null.");
+ Preconditions.checkNotNull(origStatus, "Null status for " + iip.getPath());
+
+ final ReencryptionInfoProto newProto = PBHelperClient
+ .convert(origStatus.getEzKeyVersionName(),
+ origStatus.getSubmissionTime(), false,
+ origStatus.getFilesReencrypted() + numReencrypted,
+ origStatus.getNumReencryptionFailures() + numFailures, null,
+ lastFile);
+
+ final ZoneEncryptionInfoProto newZoneProto = PBHelperClient
+ .convert(PBHelperClient.convert(zoneProto.getSuite()),
+ PBHelperClient.convert(zoneProto.getCryptoProtocolVersion()),
+ zoneProto.getKeyName(), newProto);
+
+ final XAttr xattr = XAttrHelper
+ .buildXAttr(CRYPTO_XATTR_ENCRYPTION_ZONE, newZoneProto.toByteArray());
+ final List<XAttr> xattrs = Lists.newArrayListWithCapacity(1);
+ xattrs.add(xattr);
+ FSDirXAttrOp.unprotectedSetXAttrs(fsd, iip, xattrs,
+ EnumSet.of(XAttrSetFlag.REPLACE));
+ return xattr;
+ }
+
+ /**
+ * Log re-encrypt complete (cancel, or 100% re-encrypt) to edits.
+ * Caller should logSync after calling this, outside of the FSN lock.
+ * <p>
+ * Original reencryption status is passed in to get existing information,
+ * this should include whether it is finished due to cancellation.
+ * The reencryption status is updated during SetXAttrs for completion time.
+ */
+ static List<XAttr> updateReencryptionFinish(final FSDirectory fsd,
+ final INodesInPath zoneIIP, final ZoneReencryptionStatus origStatus)
+ throws IOException {
+ assert origStatus != null;
+ assert fsd.hasWriteLock();
+ fsd.ezManager.getReencryptionStatus()
+ .markZoneCompleted(zoneIIP.getLastINode().getId());
+ final XAttr xattr =
+ generateNewXAttrForReencryptionFinish(zoneIIP, origStatus);
+ final List<XAttr> xattrs = Lists.newArrayListWithCapacity(1);
+ xattrs.add(xattr);
+ FSDirXAttrOp.unprotectedSetXAttrs(fsd, zoneIIP, xattrs,
+ EnumSet.of(XAttrSetFlag.REPLACE));
+ return xattrs;
+ }
+
+ static XAttr generateNewXAttrForReencryptionFinish(final INodesInPath iip,
+ final ZoneReencryptionStatus status) throws IOException {
+ final ZoneEncryptionInfoProto zoneProto = getZoneEncryptionInfoProto(iip);
+ final ReencryptionInfoProto newRiProto = PBHelperClient
+ .convert(status.getEzKeyVersionName(), status.getSubmissionTime(),
+ status.isCanceled(), status.getFilesReencrypted(),
+ status.getNumReencryptionFailures(), Time.now(), null);
+
+ final ZoneEncryptionInfoProto newZoneProto = PBHelperClient
+ .convert(PBHelperClient.convert(zoneProto.getSuite()),
+ PBHelperClient.convert(zoneProto.getCryptoProtocolVersion()),
+ zoneProto.getKeyName(), newRiProto);
+
+ final XAttr xattr = XAttrHelper
+ .buildXAttr(CRYPTO_XATTR_ENCRYPTION_ZONE, newZoneProto.toByteArray());
+ return xattr;
+ }
+
+ private static ZoneEncryptionInfoProto getZoneEncryptionInfoProto(
+ final INodesInPath iip) throws IOException {
+ final XAttr fileXAttr = FSDirXAttrOp
+ .unprotectedGetXAttrByPrefixedName(iip, CRYPTO_XATTR_ENCRYPTION_ZONE);
+ if (fileXAttr == null) {
+ throw new IOException(
+ "Could not find reencryption XAttr for file " + iip.getPath());
+ }
+ try {
+ return ZoneEncryptionInfoProto.parseFrom(fileXAttr.getValue());
+ } catch (InvalidProtocolBufferException e) {
+ throw new IOException(
+ "Could not parse file encryption info for " + "inode " + iip
+ .getPath(), e);
+ }
+ }
+
+ /**
+ * Save the batch's edeks to file xattrs.
+ */
+ static void saveFileXAttrsForBatch(FSDirectory fsd,
+ List<FileEdekInfo> batch) {
+ assert fsd.getFSNamesystem().hasWriteLock();
+ if (batch != null && !batch.isEmpty()) {
+ for (FileEdekInfo entry : batch) {
+ final INode inode = fsd.getInode(entry.getInodeId());
+ Preconditions.checkNotNull(inode);
+ fsd.getEditLog().logSetXAttrs(inode.getFullPathName(),
+ inode.getXAttrFeature().getXAttrs(), false);
+ }
+ }
+ }
+
/**
* Set the FileEncryptionInfo for an INode.
*
* @param fsd fsdirectory
- * @param src the path of a directory which will be the root of the
- * encryption zone.
* @param info file encryption information
+ * @param flag action when setting xattr. Either CREATE or REPLACE.
* @throws IOException
*/
static void setFileEncryptionInfo(final FSDirectory fsd,
- final INodesInPath iip, final FileEncryptionInfo info)
- throws IOException {
+ final INodesInPath iip, final FileEncryptionInfo info,
+ final XAttrSetFlag flag) throws IOException {
// Make the PB for the xattr
final HdfsProtos.PerFileEncryptionInfoProto proto =
PBHelperClient.convertPerFileEncInfo(info);
@@ -238,8 +435,7 @@ final class FSDirEncryptionZoneOp {
xAttrs.add(fileEncryptionAttr);
fsd.writeLock();
try {
- FSDirXAttrOp.unprotectedSetXAttrs(fsd, iip, xAttrs,
- EnumSet.of(XAttrSetFlag.CREATE));
+ FSDirXAttrOp.unprotectedSetXAttrs(fsd, iip, xAttrs, EnumSet.of(flag));
} finally {
fsd.writeUnlock();
}
@@ -500,4 +696,34 @@ final class FSDirEncryptionZoneOp {
this.edek = edek;
}
}
+
+ /**
+ * Get the last key version name for the given EZ. This will contact
+ * the KMS to getKeyVersions.
+ * @param zone the encryption zone
+ * @param pc the permission checker
+ * @return the last element from the list of keyVersionNames returned by KMS.
+ * @throws IOException
+ */
+ static KeyVersion getLatestKeyVersion(final FSDirectory dir,
+ final String zone, final FSPermissionChecker pc) throws IOException {
+ final EncryptionZone ez;
+ assert dir.getProvider() != null;
+ dir.readLock();
+ try {
+ final INodesInPath iip = dir.resolvePath(pc, zone, DirOp.READ);
+ if (iip.getLastINode() == null) {
+ throw new FileNotFoundException(zone + " does not exist.");
+ }
+ dir.ezManager.checkEncryptionZoneRoot(iip.getLastINode(), iip.getPath());
+ ez = FSDirEncryptionZoneOp.getEZForPath(dir, iip);
+ } finally {
+ dir.readUnlock();
+ }
+ // Contact KMS out of locks.
+ KeyVersion currKv = dir.getProvider().getCurrentKey(ez.getKeyName());
+ Preconditions.checkNotNull(currKv,
+ "No current key versions for key name " + ez.getKeyName());
+ return currKv;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1000a2af/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
index 7ab05d7..012e916 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.hdfs.server.namenode;
import com.google.common.base.Preconditions;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.HadoopIllegalArgumentException;
+import org.apache.hadoop.fs.XAttrSetFlag;
import org.apache.hadoop.hdfs.AddBlockFlag;
import org.apache.hadoop.fs.CreateFlag;
import org.apache.hadoop.fs.FileAlreadyExistsException;
@@ -397,7 +398,8 @@ class FSDirWriteFileOp {
newNode.getFileUnderConstructionFeature().getClientName(),
newNode.getId());
if (feInfo != null) {
- FSDirEncryptionZoneOp.setFileEncryptionInfo(fsd, iip, feInfo);
+ FSDirEncryptionZoneOp.setFileEncryptionInfo(fsd, iip, feInfo,
+ XAttrSetFlag.CREATE);
}
setNewINodeStoragePolicy(fsd.getBlockManager(), iip, isLazyPersist);
fsd.getEditLog().logOpenFile(src, newNode, overwrite, logRetryEntry);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1000a2af/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java
index ddc088c..acdade7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirXAttrOp.java
@@ -29,6 +29,7 @@ import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSUtil;
import org.apache.hadoop.hdfs.XAttrHelper;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos;
+import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.ReencryptionInfoProto;
import org.apache.hadoop.hdfs.protocolPB.PBHelperClient;
import org.apache.hadoop.hdfs.server.namenode.FSDirectory.DirOp;
import org.apache.hadoop.security.AccessControlException;
@@ -275,6 +276,12 @@ class FSDirXAttrOp {
PBHelperClient.convert(ezProto.getSuite()),
PBHelperClient.convert(ezProto.getCryptoProtocolVersion()),
ezProto.getKeyName());
+
+ if (ezProto.hasReencryptionProto()) {
+ ReencryptionInfoProto reProto = ezProto.getReencryptionProto();
+ fsd.ezManager.getReencryptionStatus()
+ .updateZoneStatus(inode.getId(), iip.getPath(), reProto);
+ }
}
if (!isFile && SECURITY_XATTR_UNREADABLE_BY_SUPERUSER.equals(xaName)) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1000a2af/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
index 87b1156..e6aa533 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirectory.java
@@ -50,6 +50,7 @@ import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
import org.apache.hadoop.hdfs.protocol.SnapshotAccessControlException;
import org.apache.hadoop.hdfs.protocol.UnresolvedPathException;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos;
+import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.ReencryptionInfoProto;
import org.apache.hadoop.hdfs.protocolPB.PBHelperClient;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfo;
import org.apache.hadoop.hdfs.server.blockmanagement.BlockInfoStriped;
@@ -1358,12 +1359,18 @@ public class FSDirectory implements Closeable {
}
try {
final HdfsProtos.ZoneEncryptionInfoProto ezProto =
- HdfsProtos.ZoneEncryptionInfoProto.parseFrom(
- xattr.getValue());
+ HdfsProtos.ZoneEncryptionInfoProto.parseFrom(xattr.getValue());
ezManager.unprotectedAddEncryptionZone(inode.getId(),
PBHelperClient.convert(ezProto.getSuite()),
PBHelperClient.convert(ezProto.getCryptoProtocolVersion()),
ezProto.getKeyName());
+ if (ezProto.hasReencryptionProto()) {
+ final ReencryptionInfoProto reProto = ezProto.getReencryptionProto();
+ // inodes parents may not be loaded if this is done during fsimage
+ // loading so cannot set full path now. Pass in null to indicate that.
+ ezManager.getReencryptionStatus()
+ .updateZoneStatus(inode.getId(), null, reProto);
+ }
} catch (InvalidProtocolBufferException e) {
NameNode.LOG.warn("Error parsing protocol buffer of " +
"EZ XAttr " + xattr.getName() + " dir:" + inode.getFullPathName());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1000a2af/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 2313335..12d96d8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -89,9 +89,11 @@ import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_REPLICATION_DEFAULT;
import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_REPLICATION_KEY;
import static org.apache.hadoop.hdfs.server.namenode.FSDirStatAndListingOp.*;
+import org.apache.hadoop.crypto.key.KeyProvider.KeyVersion;
import org.apache.hadoop.hdfs.protocol.BlocksStats;
import org.apache.hadoop.hdfs.protocol.ECBlockGroupsStats;
import org.apache.hadoop.hdfs.protocol.OpenFileEntry;
+import org.apache.hadoop.hdfs.protocol.ZoneReencryptionStatus;
import org.apache.hadoop.hdfs.server.namenode.metrics.ReplicatedBlocksMBean;
import org.apache.hadoop.hdfs.server.protocol.SlowDiskReports;
import static org.apache.hadoop.util.Time.now;
@@ -199,6 +201,7 @@ import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
import org.apache.hadoop.hdfs.protocol.EncryptionZone;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants.ReencryptAction;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.hdfs.protocol.IllegalECPolicyException;
@@ -1230,6 +1233,8 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
dir.updateCountForQuota();
// Enable quota checks.
dir.enableQuotaChecks();
+ dir.ezManager.startReencryptThreads();
+
if (haEnabled) {
// Renew all of the leases before becoming active.
// This is because, while we were in standby mode,
@@ -1321,6 +1326,9 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
// so that the tailer starts from the right spot.
getFSImage().updateLastAppliedTxIdFromWritten();
}
+ if (dir != null) {
+ dir.ezManager.stopReencryptThread();
+ }
if (cacheManager != null) {
cacheManager.stopMonitorThread();
cacheManager.clearDirectiveStats();
@@ -7031,6 +7039,84 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
}
}
+ void reencryptEncryptionZone(final String zone, final ReencryptAction action,
+ final boolean logRetryCache) throws IOException {
+ boolean success = false;
+ try {
+ Preconditions.checkNotNull(zone, "zone is null.");
+ checkSuperuserPrivilege();
+ checkOperation(OperationCategory.WRITE);
+ checkNameNodeSafeMode("NameNode in safemode, cannot " + action
+ + " re-encryption on zone " + zone);
+ reencryptEncryptionZoneInt(zone, action, logRetryCache);
+ success = true;
+ } finally {
+ logAuditEvent(success, action + "reencryption", zone, null, null);
+ }
+ }
+
+ BatchedListEntries<ZoneReencryptionStatus> listReencryptionStatus(
+ final long prevId) throws IOException {
+ final String operationName = "listReencryptionStatus";
+ boolean success = false;
+ checkSuperuserPrivilege();
+ checkOperation(OperationCategory.READ);
+ readLock();
+ try {
+ checkSuperuserPrivilege();
+ checkOperation(OperationCategory.READ);
+ final BatchedListEntries<ZoneReencryptionStatus> ret =
+ FSDirEncryptionZoneOp.listReencryptionStatus(dir, prevId);
+ success = true;
+ return ret;
+ } finally {
+ readUnlock(operationName);
+ logAuditEvent(success, operationName, null);
+ }
+ }
+
+ private void reencryptEncryptionZoneInt(final String zone,
+ final ReencryptAction action, final boolean logRetryCache)
+ throws IOException {
+ if (getProvider() == null) {
+ throw new IOException("No key provider configured, re-encryption "
+ + "operation is rejected");
+ }
+ FSPermissionChecker pc = getPermissionChecker();
+ // get keyVersionName out of the lock. This keyVersionName will be used
+ // as the target keyVersion for the entire re-encryption.
+ // This means all edek's keyVersion will be compared with this one, and
+ // kms is only contacted if the edek's keyVersion is different.
+ final KeyVersion kv =
+ FSDirEncryptionZoneOp.getLatestKeyVersion(dir, zone, pc);
+ provider.invalidateCache(kv.getName());
+ writeLock();
+ try {
+ checkSuperuserPrivilege();
+ checkOperation(OperationCategory.WRITE);
+ checkNameNodeSafeMode(
+ "NameNode in safemode, cannot " + action + " re-encryption on zone "
+ + zone);
+ switch (action) {
+ case START:
+ FSDirEncryptionZoneOp
+ .reencryptEncryptionZone(dir, zone, kv.getVersionName(),
+ logRetryCache);
+ break;
+ case CANCEL:
+ FSDirEncryptionZoneOp
+ .cancelReencryptEncryptionZone(dir, zone, logRetryCache);
+ break;
+ default:
+ throw new IOException(
+ "Re-encryption action " + action + " is not supported");
+ }
+ } finally {
+ writeUnlock();
+ }
+ getEditLog().logSync();
+ }
+
/**
* Set an erasure coding policy on the given path.
* @param srcArg The path of the target directory.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1000a2af/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
index 7871202..3fbb7bd 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
@@ -105,6 +105,7 @@ import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.FSLimitException;
import org.apache.hadoop.hdfs.protocol.LastBlockWithStatus;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants.ReencryptAction;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.RollingUpgradeAction;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
@@ -116,6 +117,7 @@ import org.apache.hadoop.hdfs.protocol.QuotaByStorageTypeExceededException;
import org.apache.hadoop.hdfs.protocol.QuotaExceededException;
import org.apache.hadoop.hdfs.protocol.RecoveryInProgressException;
import org.apache.hadoop.hdfs.protocol.BlocksStats;
+import org.apache.hadoop.hdfs.protocol.ZoneReencryptionStatus;
import org.apache.hadoop.hdfs.protocol.RollingUpgradeInfo;
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus;
@@ -2052,6 +2054,31 @@ public class NameNodeRpcServer implements NamenodeProtocols {
}
@Override // ClientProtocol
+ public void reencryptEncryptionZone(final String zone,
+ final ReencryptAction action) throws IOException {
+ checkNNStartup();
+ namesystem.checkOperation(OperationCategory.WRITE);
+ final CacheEntry cacheEntry = RetryCache.waitForCompletion(retryCache);
+ if (cacheEntry != null && cacheEntry.isSuccess()) {
+ return;
+ }
+ boolean success = false;
+ try {
+ namesystem.reencryptEncryptionZone(zone, action, cacheEntry != null);
+ success = true;
+ } finally {
+ RetryCache.setState(cacheEntry, success);
+ }
+ }
+
+ @Override // ClientProtocol
+ public BatchedEntries<ZoneReencryptionStatus> listReencryptionStatus(
+ final long prevId) throws IOException {
+ checkNNStartup();
+ return namesystem.listReencryptionStatus(prevId);
+ }
+
+ @Override // ClientProtocol
public void setErasureCodingPolicy(String src, String ecPolicyName)
throws IOException {
checkNNStartup();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1000a2af/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ReencryptionHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ReencryptionHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ReencryptionHandler.java
new file mode 100644
index 0000000..729b894
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ReencryptionHandler.java
@@ -0,0 +1,940 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.namenode;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Stopwatch;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension.EncryptedKeyVersion;
+import org.apache.hadoop.fs.FileEncryptionInfo;
+import org.apache.hadoop.fs.XAttr;
+import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
+import org.apache.hadoop.hdfs.protocol.ReencryptionStatus;
+import org.apache.hadoop.hdfs.protocol.ZoneReencryptionStatus;
+import org.apache.hadoop.hdfs.protocol.ZoneReencryptionStatus.State;
+import org.apache.hadoop.hdfs.server.namenode.snapshot.Snapshot;
+import org.apache.hadoop.hdfs.server.namenode.ReencryptionUpdater.FileEdekInfo;
+import org.apache.hadoop.hdfs.server.namenode.ReencryptionUpdater.ReencryptionTask;
+import org.apache.hadoop.hdfs.server.namenode.ReencryptionUpdater.ZoneSubmissionTracker;
+import org.apache.hadoop.hdfs.util.ReadOnlyList;
+import org.apache.hadoop.ipc.RetriableException;
+import org.apache.hadoop.util.Daemon;
+import org.apache.hadoop.util.StopWatch;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.security.GeneralSecurityException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ExecutorCompletionService;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicInteger;
+
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_REENCRYPT_BATCH_SIZE_DEFAULT;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_REENCRYPT_BATCH_SIZE_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_REENCRYPT_SLEEP_INTERVAL_DEFAULT;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_REENCRYPT_SLEEP_INTERVAL_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_REENCRYPT_THROTTLE_LIMIT_HANDLER_RATIO_DEFAULT;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_REENCRYPT_THROTTLE_LIMIT_HANDLER_RATIO_KEY;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_REENCRYPT_EDEK_THREADS_DEFAULT;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_REENCRYPT_EDEK_THREADS_KEY;
+
+/**
+ * Class for handling re-encrypt EDEK operations.
+ * <p>
+ * For each EZ, ReencryptionHandler walks the tree in a depth-first order,
+ * and submits batches of (files + existing edeks) as re-encryption tasks
+ * to a thread pool. Each thread in the pool then contacts the KMS to
+ * re-encrypt the edeks. ReencryptionUpdater tracks the tasks and updates
+ * file xattrs with the new edeks.
+ * <p>
+ * File renames are disabled in the EZ that's being re-encrypted. Newly created
+ * files will have new edeks, because the edek cache is drained upon the
+ * submission of a re-encryption command.
+ * <p>
+ * It is assumed only 1 ReencryptionHandler will be running, because:
+ * 1. The bottleneck of the entire re-encryption appears to be on the KMS.
+ * 2. Even with multiple handlers, since updater requires writelock and is
+ * single-threaded, the performance gain is limited.
+ * <p>
+ * This class uses the FSDirectory lock for synchronization.
+ */
+@InterfaceAudience.Private
+public class ReencryptionHandler implements Runnable {
+
+ public static final Logger LOG =
+ LoggerFactory.getLogger(ReencryptionHandler.class);
+
+ // 2000 is based on buffer size = 512 * 1024, and SetXAttr op size is
+ // 100 - 200 bytes (depending on the xattr value).
+ // The buffer size is hard-coded, see outputBufferCapacity from QJM.
+ private static final int MAX_BATCH_SIZE_WITHOUT_FLOODING = 2000;
+
+ private final EncryptionZoneManager ezManager;
+ private final FSDirectory dir;
+ private final long interval;
+ private final int reencryptBatchSize;
+ private double throttleLimitHandlerRatio;
+ private final int reencryptThreadPoolSize;
+ // stopwatches for throttling
+ private final StopWatch throttleTimerAll = new StopWatch();
+ private final StopWatch throttleTimerLocked = new StopWatch();
+
+ private ExecutorCompletionService<ReencryptionTask> batchService;
+ private BlockingQueue<Runnable> taskQueue;
+ // protected by ReencryptionHandler object lock
+ private final Map<Long, ZoneSubmissionTracker> submissions =
+ new ConcurrentHashMap<>();
+
+ // The current batch that the handler is working on. Handler is designed to
+ // be single-threaded, see class javadoc for more details.
+ private ReencryptionBatch currentBatch;
+
+ private final ReencryptionUpdater reencryptionUpdater;
+ private ExecutorService updaterExecutor;
+
+ // Vars for unit tests.
+ private volatile boolean shouldPauseForTesting = false;
+ private volatile int pauseAfterNthSubmission = 0;
+
+ /**
+ * Stop the re-encryption updater thread, as well as all EDEK re-encryption
+ * tasks submitted.
+ */
+ void stopThreads() {
+ assert dir.hasWriteLock();
+ for (ZoneSubmissionTracker zst : submissions.values()) {
+ zst.cancelAllTasks();
+ }
+ if (updaterExecutor != null) {
+ updaterExecutor.shutdownNow();
+ }
+ }
+
+ /**
+ * Start the re-encryption updater thread.
+ */
+ void startUpdaterThread() {
+ updaterExecutor = Executors.newSingleThreadExecutor(
+ new ThreadFactoryBuilder().setDaemon(true)
+ .setNameFormat("reencryptionUpdaterThread #%d").build());
+ updaterExecutor.execute(reencryptionUpdater);
+ }
+
+ @VisibleForTesting
+ synchronized void pauseForTesting() {
+ shouldPauseForTesting = true;
+ LOG.info("Pausing re-encrypt handler for testing.");
+ notify();
+ }
+
+ @VisibleForTesting
+ synchronized void resumeForTesting() {
+ shouldPauseForTesting = false;
+ LOG.info("Resuming re-encrypt handler for testing.");
+ notify();
+ }
+
+ @VisibleForTesting
+ void pauseForTestingAfterNthSubmission(final int count) {
+ assert pauseAfterNthSubmission == 0;
+ pauseAfterNthSubmission = count;
+ }
+
+ @VisibleForTesting
+ void pauseUpdaterForTesting() {
+ reencryptionUpdater.pauseForTesting();
+ }
+
+ @VisibleForTesting
+ void resumeUpdaterForTesting() {
+ reencryptionUpdater.resumeForTesting();
+ }
+
+ @VisibleForTesting
+ void pauseForTestingAfterNthCheckpoint(final long zoneId, final int count) {
+ reencryptionUpdater.pauseForTestingAfterNthCheckpoint(zoneId, count);
+ }
+
+ private synchronized void checkPauseForTesting() throws InterruptedException {
+ assert !dir.hasReadLock();
+ assert !dir.getFSNamesystem().hasReadLock();
+ while (shouldPauseForTesting) {
+ LOG.info("Sleeping in the re-encrypt handler for unit test.");
+ wait();
+ LOG.info("Continuing re-encrypt handler after pausing.");
+ }
+ }
+
+ ReencryptionHandler(final EncryptionZoneManager ezMgr,
+ final Configuration conf) {
+ this.ezManager = ezMgr;
+ Preconditions.checkNotNull(ezManager.getProvider(),
+ "No provider set, cannot re-encrypt");
+ this.dir = ezMgr.getFSDirectory();
+ this.interval =
+ conf.getTimeDuration(DFS_NAMENODE_REENCRYPT_SLEEP_INTERVAL_KEY,
+ DFS_NAMENODE_REENCRYPT_SLEEP_INTERVAL_DEFAULT,
+ TimeUnit.MILLISECONDS);
+ Preconditions.checkArgument(interval > 0,
+ DFS_NAMENODE_REENCRYPT_SLEEP_INTERVAL_KEY + " is not positive.");
+ this.reencryptBatchSize = conf.getInt(DFS_NAMENODE_REENCRYPT_BATCH_SIZE_KEY,
+ DFS_NAMENODE_REENCRYPT_BATCH_SIZE_DEFAULT);
+ Preconditions.checkArgument(reencryptBatchSize > 0,
+ DFS_NAMENODE_REENCRYPT_BATCH_SIZE_KEY + " is not positive.");
+ if (reencryptBatchSize > MAX_BATCH_SIZE_WITHOUT_FLOODING) {
+ LOG.warn("Re-encryption batch size is {}. It could cause edit log buffer "
+ + "to be full and trigger a logSync within the writelock, greatly "
+ + "impacting namenode throughput.", reencryptBatchSize);
+ }
+ this.throttleLimitHandlerRatio =
+ conf.getDouble(DFS_NAMENODE_REENCRYPT_THROTTLE_LIMIT_HANDLER_RATIO_KEY,
+ DFS_NAMENODE_REENCRYPT_THROTTLE_LIMIT_HANDLER_RATIO_DEFAULT);
+ LOG.info("Configured throttleLimitHandlerRatio={} for re-encryption",
+ throttleLimitHandlerRatio);
+ Preconditions.checkArgument(throttleLimitHandlerRatio > 0.0f,
+ DFS_NAMENODE_REENCRYPT_THROTTLE_LIMIT_HANDLER_RATIO_KEY
+ + " is not positive.");
+ this.reencryptThreadPoolSize =
+ conf.getInt(DFS_NAMENODE_REENCRYPT_EDEK_THREADS_KEY,
+ DFS_NAMENODE_REENCRYPT_EDEK_THREADS_DEFAULT);
+
+ taskQueue = new LinkedBlockingQueue<>();
+ ThreadPoolExecutor threadPool =
+ new ThreadPoolExecutor(reencryptThreadPoolSize, reencryptThreadPoolSize,
+ 60, TimeUnit.SECONDS, taskQueue, new Daemon.DaemonFactory() {
+ private final AtomicInteger ind = new AtomicInteger(0);
+
+ @Override
+ public Thread newThread(Runnable r) {
+ Thread t = super.newThread(r);
+ t.setName("reencryption edek Thread-" + ind.getAndIncrement());
+ return t;
+ }
+ }, new ThreadPoolExecutor.CallerRunsPolicy() {
+
+ @Override
+ public void rejectedExecution(Runnable runnable,
+ ThreadPoolExecutor e) {
+ LOG.info("Execution rejected, executing in current thread");
+ super.rejectedExecution(runnable, e);
+ }
+ });
+
+ threadPool.allowCoreThreadTimeOut(true);
+ this.batchService = new ExecutorCompletionService(threadPool);
+ reencryptionUpdater =
+ new ReencryptionUpdater(dir, batchService, this, conf);
+ currentBatch = new ReencryptionBatch(reencryptBatchSize);
+ }
+
+ ReencryptionStatus getReencryptionStatus() {
+ return ezManager.getReencryptionStatus();
+ }
+
+ void cancelZone(final long zoneId, final String zoneName) throws IOException {
+ assert dir.hasWriteLock();
+ final ZoneReencryptionStatus zs =
+ getReencryptionStatus().getZoneStatus(zoneId);
+ if (zs == null || zs.getState() == State.Completed) {
+ throw new IOException("Zone " + zoneName + " is not under re-encryption");
+ }
+ zs.cancel();
+ ZoneSubmissionTracker zst = submissions.get(zoneId);
+ if (zst != null) {
+ zst.cancelAllTasks();
+ }
+ }
+
+ void removeZone(final long zoneId) {
+ assert dir.hasWriteLock();
+ LOG.info("Removing zone {} from re-encryption.", zoneId);
+ ZoneSubmissionTracker zst = submissions.get(zoneId);
+ if (zst != null) {
+ zst.cancelAllTasks();
+ }
+ submissions.remove(zoneId);
+ getReencryptionStatus().removeZone(zoneId);
+ }
+
+ ZoneSubmissionTracker getTracker(final long zoneId) {
+ dir.hasReadLock();
+ return unprotectedGetTracker(zoneId);
+ }
+
+ /**
+ * get the tracker without holding the FSDirectory lock. This is only used for
+ * testing, when updater checks about pausing.
+ */
+ ZoneSubmissionTracker unprotectedGetTracker(final long zoneId) {
+ return submissions.get(zoneId);
+ }
+
+ /**
+ * Add a dummy tracker (with 1 task that has 0 files to re-encrypt)
+ * for the zone. This is necessary to complete the re-encryption in case
+ * no file in the entire zone needs re-encryption at all. We cannot simply
+ * update zone status and set zone xattrs, because in the handler we only hold
+ * readlock, and setting xattrs requires upgrading to a writelock.
+ *
+ * @param zoneId
+ */
+ void addDummyTracker(final long zoneId) {
+ assert dir.hasReadLock();
+ assert !submissions.containsKey(zoneId);
+ final ZoneSubmissionTracker zst = new ZoneSubmissionTracker();
+ zst.setSubmissionDone();
+
+ Future future = batchService.submit(
+ new EDEKReencryptCallable(zoneId, new ReencryptionBatch(), this));
+ zst.addTask(future);
+ submissions.put(zoneId, zst);
+ }
+
+ /**
+ * Main loop. It takes at most 1 zone per scan, and executes until the zone
+ * is completed.
+ * {@see #reencryptEncryptionZoneInt(Long)}.
+ */
+ @Override
+ public void run() {
+ LOG.info("Starting up re-encrypt thread with interval={} millisecond.",
+ interval);
+ while (true) {
+ try {
+ synchronized (this) {
+ wait(interval);
+ }
+ checkPauseForTesting();
+ } catch (InterruptedException ie) {
+ LOG.info("Re-encrypt handler interrupted. Exiting");
+ Thread.currentThread().interrupt();
+ return;
+ }
+
+ final Long zoneId;
+ dir.readLock();
+ try {
+ zoneId = getReencryptionStatus().getNextUnprocessedZone();
+ if (zoneId == null) {
+ // empty queue.
+ continue;
+ }
+ LOG.info("Executing re-encrypt commands on zone {}. Current zones:{}",
+ zoneId, getReencryptionStatus());
+ } finally {
+ dir.readUnlock();
+ }
+
+ try {
+ reencryptEncryptionZone(zoneId);
+ } catch (RetriableException | SafeModeException re) {
+ LOG.info("Re-encryption caught exception, will retry", re);
+ getReencryptionStatus().markZoneForRetry(zoneId);
+ } catch (IOException ioe) {
+ LOG.warn("IOException caught when re-encrypting zone {}", zoneId, ioe);
+ } catch (InterruptedException ie) {
+ LOG.info("Re-encrypt handler interrupted. Exiting.");
+ Thread.currentThread().interrupt();
+ return;
+ } catch (Throwable t) {
+ LOG.error("Re-encrypt handler thread exiting. Exception caught when"
+ + " re-encrypting zone {}.", zoneId, t);
+ return;
+ }
+ }
+ }
+
+ /**
+ * Re-encrypts a zone by recursively iterating all paths inside the zone,
+ * in lexicographic order.
+ * Files are re-encrypted, and subdirs are processed during iteration.
+ *
+ * @param zoneId the Zone's id.
+ * @throws IOException
+ * @throws InterruptedException
+ */
+ void reencryptEncryptionZone(final long zoneId)
+ throws IOException, InterruptedException {
+ throttleTimerAll.reset().start();
+ throttleTimerLocked.reset();
+ final INode zoneNode;
+ final ZoneReencryptionStatus zs;
+
+ readLock();
+ try {
+ getReencryptionStatus().markZoneStarted(zoneId);
+ zoneNode = dir.getInode(zoneId);
+ // start re-encrypting the zone from the beginning
+ if (zoneNode == null) {
+ LOG.info("Directory with id {} removed during re-encrypt, skipping",
+ zoneId);
+ return;
+ }
+ if (!zoneNode.isDirectory()) {
+ LOG.info("Cannot re-encrypt directory with id {} because it's not a"
+ + " directory.", zoneId);
+ return;
+ }
+
+ zs = getReencryptionStatus().getZoneStatus(zoneId);
+ assert zs != null;
+ // Only costly log FullPathName here once, and use id elsewhere.
+ LOG.info("Re-encrypting zone {}(id={})", zoneNode.getFullPathName(),
+ zoneId);
+ if (zs.getLastCheckpointFile() == null) {
+ // new re-encryption
+ reencryptDir(zoneNode.asDirectory(), zoneId, HdfsFileStatus.EMPTY_NAME,
+ zs.getEzKeyVersionName());
+ } else {
+ // resuming from a past re-encryption
+ restoreFromLastProcessedFile(zoneId, zs);
+ }
+ // save the last batch and mark complete
+ submitCurrentBatch(zoneId);
+ LOG.info("Submission completed of zone {} for re-encryption.", zoneId);
+ reencryptionUpdater.markZoneSubmissionDone(zoneId);
+ } finally {
+ readUnlock();
+ }
+ }
+
+ List<XAttr> completeReencryption(final INode zoneNode) throws IOException {
+ assert dir.hasWriteLock();
+ assert dir.getFSNamesystem().hasWriteLock();
+ final Long zoneId = zoneNode.getId();
+ ZoneReencryptionStatus zs = getReencryptionStatus().getZoneStatus(zoneId);
+ assert zs != null;
+ LOG.info("Re-encryption completed on zone {}. Re-encrypted {} files,"
+ + " failures encountered: {}.", zoneNode.getFullPathName(),
+ zs.getFilesReencrypted(), zs.getNumReencryptionFailures());
+ // This also removes the zone from reencryptionStatus
+ submissions.remove(zoneId);
+ return FSDirEncryptionZoneOp
+ .updateReencryptionFinish(dir, INodesInPath.fromINode(zoneNode), zs);
+ }
+
+ /**
+ * Restore the re-encryption from the progress inside ReencryptionStatus.
+ * This means start from exactly the lastProcessedFile (LPF), skipping all
+ * earlier paths in lexicographic order. Lexicographically-later directories
+ * on the LPF parent paths are added to subdirs.
+ */
+ private void restoreFromLastProcessedFile(final long zoneId,
+ final ZoneReencryptionStatus zs)
+ throws IOException, InterruptedException {
+ final INodeDirectory parent;
+ final byte[] startAfter;
+ final INodesInPath lpfIIP =
+ dir.getINodesInPath(zs.getLastCheckpointFile(), FSDirectory.DirOp.READ);
+ parent = lpfIIP.getLastINode().getParent();
+ startAfter = lpfIIP.getLastINode().getLocalNameBytes();
+ reencryptDir(parent, zoneId, startAfter, zs.getEzKeyVersionName());
+ }
+
+ /**
+ * Iterate through all files directly inside parent, and recurse down
+ * directories. The listing is done in batch, and can optionally start after
+ * a position.
+ * <p>
+ * Each batch is then send to the threadpool, where KMS will be contacted and
+ * edek re-encrypted. {@link ReencryptionUpdater} handles the tasks completed
+ * from the threadpool.
+ * <p>
+ * The iteration of the inode tree is done in a depth-first fashion. But
+ * instead of holding all INodeDirectory's in memory on the fly, only the
+ * path components to the current inode is held. This is to reduce memory
+ * consumption.
+ *
+ * @param parent The inode id of parent directory
+ * @param zoneId Id of the EZ inode
+ * @param startAfter Full path of a file the re-encrypt should start after.
+ * @throws IOException
+ * @throws InterruptedException
+ */
+ private void reencryptDir(final INodeDirectory parent, final long zoneId,
+ byte[] startAfter, final String ezKeyVerName)
+ throws IOException, InterruptedException {
+ List<byte[]> startAfters = new ArrayList<>();
+ if (parent == null) {
+ return;
+ }
+ INode curr = parent;
+ // construct startAfters all the way up to the zone inode.
+ startAfters.add(startAfter);
+ while (curr.getId() != zoneId) {
+ startAfters.add(0, curr.getLocalNameBytes());
+ curr = curr.getParent();
+ }
+ curr = reencryptDirInt(zoneId, parent, startAfters, ezKeyVerName);
+ while (!startAfters.isEmpty()) {
+ if (curr == null) {
+ // lock was reacquired, re-resolve path.
+ curr = resolvePaths(zoneId, startAfters);
+ }
+ curr = reencryptDirInt(zoneId, curr, startAfters, ezKeyVerName);
+ }
+ }
+
+ /**
+ * Resolve the cursor of re-encryption to an inode.
+ * <p>
+ * The parent of the lowest level startAfter is returned. If somewhere in the
+ * middle of startAfters changed, the parent of the lowest unchanged level is
+ * returned.
+ *
+ * @param zoneId Id of the EZ inode.
+ * @param startAfters the cursor, represented by a list of path bytes.
+ * @return the parent inode corresponding to the startAfters, or null if
+ * the EZ node (furthest parent) is deleted.
+ */
+ private INode resolvePaths(final long zoneId, List<byte[]> startAfters)
+ throws IOException {
+ // If the readlock was reacquired, we need to resolve the paths again
+ // in case things have changed. If our cursor file/dir is changed,
+ // continue from the next one.
+ INode zoneNode = dir.getInode(zoneId);
+ if (zoneNode == null) {
+ throw new FileNotFoundException("Zone " + zoneId + " is deleted.");
+ }
+ INodeDirectory parent = zoneNode.asDirectory();
+ for (int i = 0; i < startAfters.size(); ++i) {
+ if (i == startAfters.size() - 1) {
+ // last startAfter does not need to be resolved, since search for
+ // nextChild will cover that automatically.
+ break;
+ }
+ INode curr =
+ parent.getChild(startAfters.get(i), Snapshot.CURRENT_STATE_ID);
+ if (curr == null) {
+ // inode at this level has changed. Update startAfters to point to
+ // the next dir at the parent level (and dropping any startAfters
+ // at lower levels).
+ for (; i < startAfters.size(); ++i) {
+ startAfters.remove(startAfters.size() - 1);
+ }
+ break;
+ }
+ parent = curr.asDirectory();
+ }
+ return parent;
+ }
+
+ /**
+ * Submit the current batch to the thread pool.
+ *
+ * @param zoneId Id of the EZ INode
+ * @throws IOException
+ * @throws InterruptedException
+ */
+ private void submitCurrentBatch(final long zoneId)
+ throws IOException, InterruptedException {
+ assert dir.hasReadLock();
+ if (currentBatch.isEmpty()) {
+ return;
+ }
+ ZoneSubmissionTracker zst = submissions.get(zoneId);
+ if (zst == null) {
+ zst = new ZoneSubmissionTracker();
+ submissions.put(zoneId, zst);
+ }
+ Future future = batchService
+ .submit(new EDEKReencryptCallable(zoneId, currentBatch, this));
+ zst.addTask(future);
+ LOG.info("Submitted batch (start:{}, size:{}) of zone {} to re-encrypt.",
+ currentBatch.getFirstFilePath(), currentBatch.size(), zoneId);
+ currentBatch = new ReencryptionBatch(reencryptBatchSize);
+ // flip the pause flag if this is nth submission.
+ // The actual pause need to happen outside of the lock.
+ if (pauseAfterNthSubmission > 0) {
+ if (--pauseAfterNthSubmission == 0) {
+ shouldPauseForTesting = true;
+ }
+ }
+ }
+
+ final class ReencryptionBatch {
+ // First file's path, for logging purpose.
+ private String firstFilePath;
+ private final List<FileEdekInfo> batch;
+
+ ReencryptionBatch() {
+ this(reencryptBatchSize);
+ }
+
+ ReencryptionBatch(int initialCapacity) {
+ batch = new ArrayList<>(initialCapacity);
+ }
+
+ void add(final INodeFile inode) throws IOException {
+ assert dir.hasReadLock();
+ Preconditions.checkNotNull(inode, "INodeFile is null");
+ if (batch.isEmpty()) {
+ firstFilePath = inode.getFullPathName();
+ }
+ batch.add(new FileEdekInfo(dir, inode));
+ }
+
+ String getFirstFilePath() {
+ return firstFilePath;
+ }
+
+ boolean isEmpty() {
+ return batch.isEmpty();
+ }
+
+ int size() {
+ return batch.size();
+ }
+
+ void clear() {
+ batch.clear();
+ }
+
+ List<FileEdekInfo> getBatch() {
+ return batch;
+ }
+ }
+
+ /**
+ * Simply contacts the KMS for re-encryption. No NN locks held.
+ */
+ private static class EDEKReencryptCallable
+ implements Callable<ReencryptionTask> {
+ private final long zoneNodeId;
+ private final ReencryptionBatch batch;
+ private final ReencryptionHandler handler;
+
+ EDEKReencryptCallable(final long zoneId,
+ final ReencryptionBatch currentBatch, final ReencryptionHandler rh) {
+ zoneNodeId = zoneId;
+ batch = currentBatch;
+ handler = rh;
+ }
+
+ @Override
+ public ReencryptionTask call() {
+ LOG.info("Processing batched re-encryption for zone {}, batch size {},"
+ + " start:{}", zoneNodeId, batch.size(), batch.getFirstFilePath());
+ if (batch.isEmpty()) {
+ return new ReencryptionTask(zoneNodeId, 0, batch);
+ }
+ final Stopwatch kmsSW = new Stopwatch().start();
+
+ int numFailures = 0;
+ String result = "Completed";
+ if (!reencryptEdeks()) {
+ numFailures += batch.size();
+ result = "Failed to";
+ }
+ LOG.info("{} re-encrypting one batch of {} edeks from KMS,"
+ + " time consumed: {}, start: {}.", result,
+ batch.size(), kmsSW.stop(), batch.getFirstFilePath());
+ return new ReencryptionTask(zoneNodeId, numFailures, batch);
+ }
+
+ private boolean reencryptEdeks() {
+ // communicate with the kms out of lock
+ final List<EncryptedKeyVersion> edeks = new ArrayList<>(batch.size());
+ for (FileEdekInfo entry : batch.getBatch()) {
+ edeks.add(entry.getExistingEdek());
+ }
+ // provider already has LoadBalancingKMSClientProvider's reties. It that
+ // fails, just fail this callable.
+ try {
+ handler.ezManager.getProvider().reencryptEncryptedKeys(edeks);
+ EncryptionFaultInjector.getInstance().reencryptEncryptedKeys();
+ } catch (GeneralSecurityException | IOException ex) {
+ LOG.warn("Failed to re-encrypt one batch of {} edeks, start:{}",
+ batch.size(), batch.getFirstFilePath(), ex);
+ return false;
+ }
+ int i = 0;
+ for (FileEdekInfo entry : batch.getBatch()) {
+ assert i < edeks.size();
+ entry.setEdek(edeks.get(i++));
+ }
+ return true;
+ }
+ }
+
+ /**
+ * Iterates the parent directory, and add direct children files to
+ * current batch. If batch size meets configured threshold, a Callable
+ * is created and sent to the thread pool, which will communicate to the KMS
+ * to get new edeks.
+ * <p>
+ * Locks could be released and reacquired when a Callable is created.
+ *
+ * @param zoneId Id of the EZ INode
+ * @return The inode which was just processed, if lock is held in the entire
+ * process. Null if lock is released.
+ * @throws IOException
+ * @throws InterruptedException
+ */
+ private INode reencryptDirInt(final long zoneId, INode curr,
+ List<byte[]> startAfters, final String ezKeyVerName)
+ throws IOException, InterruptedException {
+ assert dir.hasReadLock();
+ assert dir.getFSNamesystem().hasReadLock();
+ Preconditions.checkNotNull(curr, "Current inode can't be null");
+ checkZoneReady(zoneId);
+ final INodeDirectory parent =
+ curr.isDirectory() ? curr.asDirectory() : curr.getParent();
+ ReadOnlyList<INode> children =
+ parent.getChildrenList(Snapshot.CURRENT_STATE_ID);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Re-encrypting directory {}", parent.getFullPathName());
+ }
+
+ final byte[] startAfter = startAfters.get(startAfters.size() - 1);
+ boolean lockReleased = false;
+ for (int i = INodeDirectory.nextChild(children, startAfter);
+ i < children.size(); ++i) {
+ final INode inode = children.get(i);
+ if (!reencryptINode(inode, ezKeyVerName)) {
+ // inode wasn't added for re-encryption. Recurse down if it's a dir,
+ // skip otherwise.
+ if (!inode.isDirectory()) {
+ continue;
+ }
+ if (ezManager.isEncryptionZoneRoot(inode, inode.getFullPathName())) {
+ // nested EZ, ignore.
+ LOG.info("{}({}) is a nested EZ, skipping for re-encryption",
+ inode.getFullPathName(), inode.getId());
+ continue;
+ }
+ // add 1 level to the depth-first search.
+ curr = inode;
+ if (!startAfters.isEmpty()) {
+ startAfters.remove(startAfters.size() - 1);
+ startAfters.add(curr.getLocalNameBytes());
+ }
+ startAfters.add(HdfsFileStatus.EMPTY_NAME);
+ return lockReleased ? null : curr;
+ }
+ if (currentBatch.size() >= reencryptBatchSize) {
+ final byte[] currentStartAfter = inode.getLocalNameBytes();
+ final String parentPath = parent.getFullPathName();
+ submitCurrentBatch(zoneId);
+ lockReleased = true;
+ readUnlock();
+ try {
+ throttle();
+ checkPauseForTesting();
+ } finally {
+ readLock();
+ }
+ checkZoneReady(zoneId);
+
+ // Things could have changed when the lock was released.
+ // Re-resolve the parent inode.
+ FSPermissionChecker pc = dir.getPermissionChecker();
+ INode newParent =
+ dir.resolvePath(pc, parentPath, FSDirectory.DirOp.READ)
+ .getLastINode();
+ if (newParent == null || !newParent.equals(parent)) {
+ // parent dir is deleted or recreated. We're done.
+ return null;
+ }
+ children = parent.getChildrenList(Snapshot.CURRENT_STATE_ID);
+ // -1 to counter the ++ on the for loop
+ i = INodeDirectory.nextChild(children, currentStartAfter) - 1;
+ }
+ }
+ // Successfully finished this dir, adjust pointers to 1 level up, and
+ // startAfter this dir.
+ startAfters.remove(startAfters.size() - 1);
+ if (!startAfters.isEmpty()) {
+ startAfters.remove(startAfters.size() - 1);
+ startAfters.add(curr.getLocalNameBytes());
+ }
+ curr = curr.getParent();
+ return lockReleased ? null : curr;
+ }
+
+ private void readLock() {
+ dir.getFSNamesystem().readLock();
+ dir.readLock();
+ throttleTimerLocked.start();
+ }
+
+ private void readUnlock() {
+ dir.readUnlock();
+ dir.getFSNamesystem().readUnlock("reencryptHandler");
+ throttleTimerLocked.stop();
+ }
+
+ /**
+ * Throttles the ReencryptionHandler in 3 aspects:
+ * 1. Prevents generating more Callables than the CPU could possibly handle.
+ * 2. Prevents generating more Callables than the ReencryptionUpdater can
+ * handle, under its own throttling
+ * 3. Prevents contending FSN/FSD read locks. This is done based on the
+ * DFS_NAMENODE_REENCRYPT_THROTTLE_LIMIT_RATIO_KEY configuration.
+ * <p>
+ * Item 1 and 2 are to control NN heap usage.
+ *
+ * @throws InterruptedException
+ */
+ @VisibleForTesting
+ void throttle() throws InterruptedException {
+ // 1.
+ final int numCores = Runtime.getRuntime().availableProcessors();
+ if (taskQueue.size() >= numCores) {
+ LOG.debug("Re-encryption handler throttling because queue size {} is"
+ + "larger than number of cores {}", taskQueue.size(), numCores);
+ while (taskQueue.size() >= numCores) {
+ Thread.sleep(100);
+ }
+ }
+
+ // 2. if tasks are piling up on the updater, don't create new callables
+ // until the queue size goes down.
+ final int maxTasksPiled = Runtime.getRuntime().availableProcessors() * 2;
+ int totalTasks = 0;
+ for (ZoneSubmissionTracker zst : submissions.values()) {
+ totalTasks += zst.getTasks().size();
+ }
+ if (totalTasks >= maxTasksPiled) {
+ LOG.debug("Re-encryption handler throttling because total tasks pending"
+ + " re-encryption updater is {}", totalTasks);
+ while (totalTasks >= maxTasksPiled) {
+ Thread.sleep(500);
+ totalTasks = 0;
+ for (ZoneSubmissionTracker zst : submissions.values()) {
+ totalTasks += zst.getTasks().size();
+ }
+ }
+ }
+
+ // 3.
+ if (throttleLimitHandlerRatio >= 1.0) {
+ return;
+ }
+ final long expect = (long) (throttleTimerAll.now(TimeUnit.MILLISECONDS)
+ * throttleLimitHandlerRatio);
+ final long actual = throttleTimerLocked.now(TimeUnit.MILLISECONDS);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Re-encryption handler throttling expect: {}, actual: {},"
+ + " throttleTimerAll:{}", expect, actual,
+ throttleTimerAll.now(TimeUnit.MILLISECONDS));
+ }
+ if (expect - actual < 0) {
+ // in case throttleLimitHandlerRatio is very small, expect will be 0.
+ // so sleepMs should not be calculated from expect, to really meet the
+ // ratio. e.g. if ratio is 0.001, expect = 0 and actual = 1, sleepMs
+ // should be 1000 - throttleTimerAll.now()
+ final long sleepMs =
+ (long) (actual / throttleLimitHandlerRatio) - throttleTimerAll
+ .now(TimeUnit.MILLISECONDS);
+ LOG.debug("Throttling re-encryption, sleeping for {} ms", sleepMs);
+ Thread.sleep(sleepMs);
+ }
+ throttleTimerAll.reset().start();
+ throttleTimerLocked.reset();
+ }
+
+ /**
+ * Process an Inode for re-encryption. Add to current batch if it's a file,
+ * no-op otherwise.
+ *
+ * @param inode the inode
+ * @return true if inode is added to currentBatch and should be re-encrypted.
+ * false otherwise: could be inode is not a file, or inode's edek's
+ * key version is not changed.
+ * @throws IOException
+ * @throws InterruptedException
+ */
+ private boolean reencryptINode(final INode inode, final String ezKeyVerName)
+ throws IOException, InterruptedException {
+ dir.hasReadLock();
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("Processing {} for re-encryption", inode.getFullPathName());
+ }
+ if (!inode.isFile()) {
+ return false;
+ }
+ FileEncryptionInfo feInfo = FSDirEncryptionZoneOp
+ .getFileEncryptionInfo(dir, INodesInPath.fromINode(inode));
+ if (feInfo == null) {
+ LOG.warn("File {} skipped re-encryption because it is not encrypted! "
+ + "This is very likely a bug.", inode.getId());
+ return false;
+ }
+ if (ezKeyVerName.equals(feInfo.getEzKeyVersionName())) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("File {} skipped re-encryption because edek's key version"
+ + " name is not changed.", inode.getFullPathName());
+ }
+ return false;
+ }
+ currentBatch.add(inode.asFile());
+ return true;
+ }
+
+ /**
+ * Check whether zone is ready for re-encryption. Throws IOE if it's not.
+ * 1. If EZ is deleted.
+ * 2. if the re-encryption is canceled.
+ * 3. If NN is not active or is in safe mode.
+ *
+ * @throws IOException if zone does not exist / is cancelled, or if NN is not
+ * ready for write.
+ */
+ void checkZoneReady(final long zoneId)
+ throws RetriableException, SafeModeException, IOException {
+ final ZoneReencryptionStatus zs =
+ getReencryptionStatus().getZoneStatus(zoneId);
+ if (zs == null) {
+ throw new IOException("Zone " + zoneId + " status cannot be found.");
+ }
+ if (zs.isCanceled()) {
+ throw new IOException("Re-encryption is canceled for zone " + zoneId);
+ }
+ dir.getFSNamesystem()
+ .checkNameNodeSafeMode("NN is in safe mode, cannot re-encrypt.");
+ // re-encryption should be cancelled when NN goes to standby. Just
+ // double checking for sanity.
+ dir.getFSNamesystem().checkOperation(NameNode.OperationCategory.WRITE);
+ }
+
+ /**
+ * Called when a new zone is submitted for re-encryption. This will interrupt
+ * the background thread if it's waiting for the next
+ * DFS_NAMENODE_REENCRYPT_SLEEP_INTERVAL_KEY.
+ */
+ synchronized void notifyNewSubmission() {
+ LOG.debug("Notifying handler for new re-encryption command.");
+ this.notify();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1000a2af/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ReencryptionUpdater.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ReencryptionUpdater.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ReencryptionUpdater.java
new file mode 100644
index 0000000..690a0e9
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ReencryptionUpdater.java
@@ -0,0 +1,523 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.namenode;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension.EncryptedKeyVersion;
+import org.apache.hadoop.fs.FileEncryptionInfo;
+import org.apache.hadoop.fs.XAttr;
+import org.apache.hadoop.fs.XAttrSetFlag;
+import org.apache.hadoop.hdfs.protocol.ZoneReencryptionStatus;
+import org.apache.hadoop.hdfs.server.namenode.ReencryptionHandler.ReencryptionBatch;
+import org.apache.hadoop.ipc.RetriableException;
+import org.apache.hadoop.util.StopWatch;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.Iterator;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.ListIterator;
+import java.util.concurrent.CompletionService;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_REENCRYPT_THROTTLE_LIMIT_UPDATER_RATIO_DEFAULT;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_REENCRYPT_THROTTLE_LIMIT_UPDATER_RATIO_KEY;
+
+/**
+ * Class for finalizing re-encrypt EDEK operations, by updating file xattrs with
+ * edeks returned from reencryption.
+ * <p>
+ * The tasks are submitted by ReencryptionHandler.
+ * <p>
+ * It is assumed only 1 Updater will be running, since updating file xattrs
+ * requires namespace write lock, and performance gain from multi-threading
+ * is limited.
+ */
+@InterfaceAudience.Private
+public final class ReencryptionUpdater implements Runnable {
+
+ public static final Logger LOG =
+ LoggerFactory.getLogger(ReencryptionUpdater.class);
+
+ private volatile boolean shouldPauseForTesting = false;
+ private volatile int pauseAfterNthCheckpoint = 0;
+ private volatile long pauseZoneId = 0;
+
+ private double throttleLimitRatio;
+ private final StopWatch throttleTimerAll = new StopWatch();
+ private final StopWatch throttleTimerLocked = new StopWatch();
+
+ private volatile long faultRetryInterval = 60000;
+
+ /**
+ * Class to track re-encryption submissions of a single zone. It contains
+ * all the submitted futures, and statistics about how far the futures are
+ * processed.
+ */
+ static final class ZoneSubmissionTracker {
+ private boolean submissionDone;
+ private LinkedList<Future> tasks;
+ private int numCheckpointed;
+ private int numFutureDone;
+
+ ZoneSubmissionTracker() {
+ submissionDone = false;
+ tasks = new LinkedList<>();
+ numCheckpointed = 0;
+ numFutureDone = 0;
+ }
+
+ LinkedList<Future> getTasks() {
+ return tasks;
+ }
+
+ void cancelAllTasks() {
+ if (!tasks.isEmpty()) {
+ LOG.info("Cancelling {} re-encryption tasks", tasks.size());
+ for (Future f : tasks) {
+ f.cancel(true);
+ }
+ }
+ }
+
+ void addTask(final Future task) {
+ tasks.add(task);
+ }
+
+ private boolean isCompleted() {
+ return submissionDone && tasks.isEmpty();
+ }
+
+ void setSubmissionDone() {
+ submissionDone = true;
+ }
+ }
+
+ /**
+ * Class representing the task for one batch of a re-encryption command. It
+ * also contains statistics about how far this single batch has been executed.
+ */
+ static final class ReencryptionTask {
+ private final long zoneId;
+ private boolean processed = false;
+ private int numFilesUpdated = 0;
+ private int numFailures = 0;
+ private String lastFile = null;
+ private final ReencryptionBatch batch;
+
+ ReencryptionTask(final long id, final int failures,
+ final ReencryptionBatch theBatch) {
+ zoneId = id;
+ numFailures = failures;
+ batch = theBatch;
+ }
+ }
+
+ /**
+ * Class that encapsulates re-encryption details of a file. It contains the
+ * file inode, stores the initial edek of the file, and the new edek
+ * after re-encryption.
+ * <p>
+ * Assumptions are the object initialization happens when dir lock is held,
+ * and inode is valid and is encrypted during initialization.
+ * <p>
+ * Namespace changes may happen during re-encryption, and if inode is changed
+ * the re-encryption is skipped.
+ */
+ static final class FileEdekInfo {
+ private final long inodeId;
+ private final EncryptedKeyVersion existingEdek;
+ private EncryptedKeyVersion edek = null;
+
+ FileEdekInfo(FSDirectory dir, INodeFile inode) throws IOException {
+ assert dir.hasReadLock();
+ Preconditions.checkNotNull(inode, "INodeFile is null");
+ inodeId = inode.getId();
+ final FileEncryptionInfo fei = FSDirEncryptionZoneOp
+ .getFileEncryptionInfo(dir, INodesInPath.fromINode(inode));
+ Preconditions.checkNotNull(fei,
+ "FileEncryptionInfo is null for " + inodeId);
+ existingEdek = EncryptedKeyVersion
+ .createForDecryption(fei.getKeyName(), fei.getEzKeyVersionName(),
+ fei.getIV(), fei.getEncryptedDataEncryptionKey());
+ }
+
+ long getInodeId() {
+ return inodeId;
+ }
+
+ EncryptedKeyVersion getExistingEdek() {
+ return existingEdek;
+ }
+
+ void setEdek(final EncryptedKeyVersion ekv) {
+ assert ekv != null;
+ edek = ekv;
+ }
+ }
+
+ @VisibleForTesting
+ synchronized void pauseForTesting() {
+ shouldPauseForTesting = true;
+ LOG.info("Pausing re-encrypt updater for testing.");
+ notify();
+ }
+
+ @VisibleForTesting
+ synchronized void resumeForTesting() {
+ shouldPauseForTesting = false;
+ LOG.info("Resuming re-encrypt updater for testing.");
+ notify();
+ }
+
+ @VisibleForTesting
+ void pauseForTestingAfterNthCheckpoint(final long zoneId, final int count) {
+ assert pauseAfterNthCheckpoint == 0;
+ pauseAfterNthCheckpoint = count;
+ pauseZoneId = zoneId;
+ }
+
+ private final FSDirectory dir;
+ private final CompletionService<ReencryptionTask> batchService;
+ private final ReencryptionHandler handler;
+
+ ReencryptionUpdater(final FSDirectory fsd,
+ final CompletionService<ReencryptionTask> service,
+ final ReencryptionHandler rh, final Configuration conf) {
+ dir = fsd;
+ batchService = service;
+ handler = rh;
+ this.throttleLimitRatio =
+ conf.getDouble(DFS_NAMENODE_REENCRYPT_THROTTLE_LIMIT_UPDATER_RATIO_KEY,
+ DFS_NAMENODE_REENCRYPT_THROTTLE_LIMIT_UPDATER_RATIO_DEFAULT);
+ Preconditions.checkArgument(throttleLimitRatio > 0.0f,
+ DFS_NAMENODE_REENCRYPT_THROTTLE_LIMIT_UPDATER_RATIO_KEY
+ + " is not positive.");
+ }
+
+ /**
+ * Called by the submission thread to indicate all tasks have been submitted.
+ * If this is called but no tasks has been submitted, the re-encryption is
+ * considered complete.
+ *
+ * @param zoneId Id of the zone inode.
+ * @throws IOException
+ * @throws InterruptedException
+ */
+ void markZoneSubmissionDone(final long zoneId)
+ throws IOException, InterruptedException {
+ final ZoneSubmissionTracker tracker = handler.getTracker(zoneId);
+ if (tracker != null) {
+ tracker.submissionDone = true;
+ } else {
+ // Caller thinks submission is done, but no tasks submitted - meaning
+ // no files in the EZ need to be re-encrypted. Complete directly.
+ handler.addDummyTracker(zoneId);
+ }
+ }
+
+ @Override
+ public void run() {
+ throttleTimerAll.start();
+ while (true) {
+ try {
+ // Assuming single-threaded updater.
+ takeAndProcessTasks();
+ } catch (InterruptedException ie) {
+ LOG.warn("Re-encryption updater thread interrupted. Exiting.");
+ Thread.currentThread().interrupt();
+ return;
+ } catch (IOException ioe) {
+ LOG.warn("Re-encryption updater thread exception.", ioe);
+ } catch (Throwable t) {
+ LOG.error("Re-encryption updater thread exiting.", t);
+ return;
+ }
+ }
+ }
+
+ /**
+ * Process a completed ReencryptionTask. Each inode id is resolved to an INode
+ * object, skip if the inode is deleted.
+ * <p>
+ * Only file xattr is updated by this method. Re-encryption progress is not
+ * updated.
+ *
+ * @param zoneNodePath full path of the EZ inode.
+ * @param task the completed task.
+ * @throws IOException
+ * @throws InterruptedException
+ */
+ private void processTaskEntries(final String zoneNodePath,
+ final ReencryptionTask task) throws IOException, InterruptedException {
+ assert dir.hasWriteLock();
+ if (!task.batch.isEmpty() && task.numFailures == 0) {
+ LOG.debug(
+ "Updating file xattrs for re-encrypting zone {}," + " starting at {}",
+ zoneNodePath, task.batch.getFirstFilePath());
+ for (Iterator<FileEdekInfo> it = task.batch.getBatch().iterator();
+ it.hasNext();) {
+ FileEdekInfo entry = it.next();
+ // resolve the inode again, and skip if it's doesn't exist
+ LOG.trace("Updating {} for re-encryption.", entry.getInodeId());
+ final INode inode = dir.getInode(entry.getInodeId());
+ if (inode == null) {
+ LOG.debug("INode {} doesn't exist, skipping re-encrypt.",
+ entry.getInodeId());
+ // also remove from batch so later it's not saved.
+ it.remove();
+ continue;
+ }
+
+ // Cautiously check file encryption info, and only update if we're sure
+ // it's still using the same edek.
+ Preconditions.checkNotNull(entry.edek);
+ final FileEncryptionInfo fei = FSDirEncryptionZoneOp
+ .getFileEncryptionInfo(dir, INodesInPath.fromINode(inode));
+ if (!fei.getKeyName().equals(entry.edek.getEncryptionKeyName())) {
+ LOG.debug("Inode {} EZ key changed, skipping re-encryption.",
+ entry.getInodeId());
+ it.remove();
+ continue;
+ }
+ if (fei.getEzKeyVersionName()
+ .equals(entry.edek.getEncryptionKeyVersionName())) {
+ LOG.debug(
+ "Inode {} EZ key version unchanged, skipping re-encryption.",
+ entry.getInodeId());
+ it.remove();
+ continue;
+ }
+ if (!Arrays.equals(fei.getEncryptedDataEncryptionKey(),
+ entry.existingEdek.getEncryptedKeyVersion().getMaterial())) {
+ LOG.debug("Inode {} existing edek changed, skipping re-encryption",
+ entry.getInodeId());
+ it.remove();
+ continue;
+ }
+ FileEncryptionInfo newFei = new FileEncryptionInfo(fei.getCipherSuite(),
+ fei.getCryptoProtocolVersion(),
+ entry.edek.getEncryptedKeyVersion().getMaterial(),
+ entry.edek.getEncryptedKeyIv(), fei.getKeyName(),
+ entry.edek.getEncryptionKeyVersionName());
+ final INodesInPath iip = INodesInPath.fromINode(inode);
+ FSDirEncryptionZoneOp
+ .setFileEncryptionInfo(dir, iip, newFei, XAttrSetFlag.REPLACE);
+ task.lastFile = iip.getPath();
+ ++task.numFilesUpdated;
+ }
+
+ LOG.info("Updated xattrs on {}({}) files in zone {} for re-encryption,"
+ + " starting:{}.", task.numFilesUpdated, task.batch.size(),
+ zoneNodePath, task.batch.getFirstFilePath());
+ }
+ task.processed = true;
+ }
+
+ /**
+ * Iterate tasks for the given zone, and update progress accordingly. The
+ * checkpoint indicates all files before it are done re-encryption, so it will
+ * be updated to the position where all tasks before are completed.
+ *
+ * @param zoneNode the EZ inode.
+ * @param tracker the zone submission tracker.
+ * @return the list containing the last checkpointed xattr. Empty if
+ * no checkpoint happened.
+ * @throws ExecutionException
+ * @throws IOException
+ * @throws InterruptedException
+ */
+ private List<XAttr> processCheckpoints(final INode zoneNode,
+ final ZoneSubmissionTracker tracker)
+ throws ExecutionException, IOException, InterruptedException {
+ assert dir.hasWriteLock();
+ final long zoneId = zoneNode.getId();
+ final String zonePath = zoneNode.getFullPathName();
+ final ZoneReencryptionStatus status =
+ handler.getReencryptionStatus().getZoneStatus(zoneId);
+ assert status != null;
+ // always start from the beginning, because the checkpoint means all files
+ // before it are re-encrypted.
+ final LinkedList<Future> tasks = tracker.getTasks();
+ final List<XAttr> xAttrs = Lists.newArrayListWithCapacity(1);
+ ListIterator<Future> iter = tasks.listIterator();
+ while (iter.hasNext()) {
+ Future<ReencryptionTask> curr = iter.next();
+ if (!curr.isDone() || !curr.get().processed) {
+ // still has earlier tasks not completed, skip here.
+ break;
+ }
+ ReencryptionTask task = curr.get();
+ LOG.debug("Updating re-encryption checkpoint with completed task."
+ + " last: {} size:{}.", task.lastFile, task.batch.size());
+ assert zoneId == task.zoneId;
+ try {
+ final XAttr xattr = FSDirEncryptionZoneOp
+ .updateReencryptionProgress(dir, zoneNode, status, task.lastFile,
+ task.numFilesUpdated, task.numFailures);
+ xAttrs.clear();
+ xAttrs.add(xattr);
+ } catch (IOException ie) {
+ LOG.warn("Failed to update re-encrypted progress to xattr for zone {}",
+ zonePath, ie);
+ ++task.numFailures;
+ }
+ ++tracker.numCheckpointed;
+ iter.remove();
+ }
+ if (tracker.isCompleted()) {
+ LOG.debug("Removed re-encryption tracker for zone {} because it completed"
+ + " with {} tasks.", zonePath, tracker.numCheckpointed);
+ return handler.completeReencryption(zoneNode);
+ }
+ return xAttrs;
+ }
+
+ private void takeAndProcessTasks() throws Exception {
+ final Future<ReencryptionTask> completed = batchService.take();
+ throttle();
+ checkPauseForTesting();
+ ReencryptionTask task = completed.get();
+ if (completed.isCancelled()) {
+ LOG.debug("Skipped canceled re-encryption task for zone {}, last: {}",
+ task.zoneId, task.lastFile);
+ }
+
+ boolean shouldRetry;
+ do {
+ dir.getFSNamesystem().writeLock();
+ try {
+ throttleTimerLocked.start();
+ processTask(task);
+ shouldRetry = false;
+ } catch (RetriableException | SafeModeException re) {
+ // Keep retrying until succeed.
+ LOG.info("Exception when processing re-encryption task for zone {}, "
+ + "retrying...", task.zoneId, re);
+ shouldRetry = true;
+ Thread.sleep(faultRetryInterval);
+ } catch (IOException ioe) {
+ LOG.warn("Failure processing re-encryption task for zone {}",
+ task.zoneId, ioe);
+ ++task.numFailures;
+ task.processed = true;
+ shouldRetry = false;
+ } finally {
+ dir.getFSNamesystem().writeUnlock("reencryptUpdater");
+ throttleTimerLocked.stop();
+ }
+ // logSync regardless, to prevent edit log buffer overflow triggering
+ // logSync inside FSN writelock.
+ dir.getEditLog().logSync();
+ } while (shouldRetry);
+ }
+
+ private void processTask(ReencryptionTask task)
+ throws InterruptedException, ExecutionException, IOException {
+ final List<XAttr> xAttrs;
+ final String zonePath;
+ dir.writeLock();
+ try {
+ handler.checkZoneReady(task.zoneId);
+ final INode zoneNode = dir.getInode(task.zoneId);
+ if (zoneNode == null) {
+ // ez removed.
+ return;
+ }
+ zonePath = zoneNode.getFullPathName();
+ LOG.info("Processing returned re-encryption task for zone {}({}), "
+ + "batch size {}, start:{}", zonePath, task.zoneId,
+ task.batch.size(), task.batch.getFirstFilePath());
+ final ZoneSubmissionTracker tracker =
+ handler.getTracker(zoneNode.getId());
+ Preconditions.checkNotNull(tracker, "zone tracker not found " + zonePath);
+ tracker.numFutureDone++;
+ EncryptionFaultInjector.getInstance().reencryptUpdaterProcessOneTask();
+ processTaskEntries(zonePath, task);
+ EncryptionFaultInjector.getInstance().reencryptUpdaterProcessCheckpoint();
+ xAttrs = processCheckpoints(zoneNode, tracker);
+ } finally {
+ dir.writeUnlock();
+ }
+ FSDirEncryptionZoneOp.saveFileXAttrsForBatch(dir, task.batch.getBatch());
+ if (!xAttrs.isEmpty()) {
+ dir.getEditLog().logSetXAttrs(zonePath, xAttrs, false);
+ }
+ }
+
+ private synchronized void checkPauseForTesting() throws InterruptedException {
+ assert !dir.hasWriteLock();
+ assert !dir.getFSNamesystem().hasWriteLock();
+ if (pauseAfterNthCheckpoint != 0) {
+ ZoneSubmissionTracker tracker =
+ handler.unprotectedGetTracker(pauseZoneId);
+ if (tracker != null) {
+ if (tracker.numFutureDone == pauseAfterNthCheckpoint) {
+ shouldPauseForTesting = true;
+ pauseAfterNthCheckpoint = 0;
+ }
+ }
+ }
+ while (shouldPauseForTesting) {
+ LOG.info("Sleeping in the re-encryption updater for unit test.");
+ wait();
+ LOG.info("Continuing re-encryption updater after pausing.");
+ }
+ }
+
+ /**
+ * Throttles the ReencryptionUpdater to prevent from contending FSN/FSD write
+ * locks. This is done by the configuration.
+ */
+ private void throttle() throws InterruptedException {
+ if (throttleLimitRatio >= 1.0) {
+ return;
+ }
+
+ final long expect = (long) (throttleTimerAll.now(TimeUnit.MILLISECONDS)
+ * throttleLimitRatio);
+ final long actual = throttleTimerLocked.now(TimeUnit.MILLISECONDS);
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Re-encryption updater throttling expect: {}, actual: {},"
+ + " throttleTimerAll:{}", expect, actual,
+ throttleTimerAll.now(TimeUnit.MILLISECONDS));
+ }
+ if (expect - actual < 0) {
+ // in case throttleLimitHandlerRatio is very small, expect will be 0.
+ // so sleepMs should not be calculated from expect, to really meet the
+ // ratio. e.g. if ratio is 0.001, expect = 0 and actual = 1, sleepMs
+ // should be 1000 - throttleTimerAll.now()
+ final long sleepMs =
+ (long) (actual / throttleLimitRatio) - throttleTimerAll
+ .now(TimeUnit.MILLISECONDS);
+ LOG.debug("Throttling re-encryption, sleeping for {} ms", sleepMs);
+ Thread.sleep(sleepMs);
+ }
+ throttleTimerAll.reset().start();
+ throttleTimerLocked.reset();
+ }
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[50/50] [abbrv] hadoop git commit: YARN-5216. Expose configurable
preemption policy for OPPORTUNISTIC containers running on the NM. (Hitesh
Sharma via asuresh)
Posted by as...@apache.org.
YARN-5216. Expose configurable preemption policy for OPPORTUNISTIC containers running on the NM. (Hitesh Sharma via asuresh)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/10eeb8b5
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/10eeb8b5
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/10eeb8b5
Branch: refs/heads/YARN-5972
Commit: 10eeb8b50de601dea4fcff0a9aa4b1278f3d1aa2
Parents: 96423b5
Author: Arun Suresh <as...@apache.org>
Authored: Sat Dec 24 17:16:52 2016 -0800
Committer: Arun Suresh <as...@apache.org>
Committed: Thu Aug 24 12:34:59 2017 -0700
----------------------------------------------------------------------
.../hadoop/yarn/conf/YarnConfiguration.java | 9 ++
.../src/main/resources/yarn-default.xml | 9 ++
.../containermanager/container/Container.java | 2 +
.../container/ContainerImpl.java | 32 ++++--
.../scheduler/ContainerScheduler.java | 85 ++++++++++++---
.../TestContainerSchedulerQueuing.java | 103 +++++++++++++++++++
.../nodemanager/webapp/MockContainer.java | 5 +
7 files changed, 219 insertions(+), 26 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/10eeb8b5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 86f45b8..8f321c8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -1030,6 +1030,15 @@ public class YarnConfiguration extends Configuration {
NM_PREFIX + "container-retry-minimum-interval-ms";
public static final int DEFAULT_NM_CONTAINER_RETRY_MINIMUM_INTERVAL_MS = 1000;
+ /**
+ * Use container pause as the preemption policy over kill in the container
+ * queue at a NodeManager.
+ **/
+ public static final String NM_CONTAINER_QUEUING_USE_PAUSE_FOR_PREEMPTION =
+ NM_PREFIX + "opportunistic-containers-use-pause-for-preemption";
+ public static final boolean
+ DEFAULT_NM_CONTAINER_QUEUING_USE_PAUSE_FOR_PREEMPTION = false;
+
/** Interval at which the delayed token removal thread runs */
public static final String RM_DELAYED_DELEGATION_TOKEN_REMOVAL_INTERVAL_MS =
RM_PREFIX + "delayed.delegation-token.removal-interval-ms";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/10eeb8b5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index f93de44..cc9692a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -2943,6 +2943,15 @@
<property>
<description>
+ Use container pause as the preemption policy over kill in the container
+ queue at a NodeManager.
+ </description>
+ <name>yarn.nodemanager.opportunistic-containers-use-pause-for-preemption</name>
+ <value>false</value>
+ </property>
+
+ <property>
+ <description>
Error filename pattern, to identify the file in the container's
Log directory which contain the container's error log. As error file
redirection is done by client/AM and yarn will not be aware of the error
http://git-wip-us.apache.org/repos/asf/hadoop/blob/10eeb8b5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java
index f6e567c..38ba84c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java
@@ -94,4 +94,6 @@ public interface Container extends EventHandler<ContainerEvent> {
void sendKillEvent(int exitStatus, String description);
boolean isRecovering();
+
+ void sendPauseEvent(String description);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/10eeb8b5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
index 031ce54..9c0c3ac 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
@@ -796,15 +796,22 @@ public class ContainerImpl implements Container {
@SuppressWarnings("unchecked") // dispatcher not typed
@Override
public void sendLaunchEvent() {
- ContainersLauncherEventType launcherEvent =
- ContainersLauncherEventType.LAUNCH_CONTAINER;
- if (recoveredStatus == RecoveredContainerStatus.LAUNCHED) {
- // try to recover a container that was previously launched
- launcherEvent = ContainersLauncherEventType.RECOVER_CONTAINER;
+ if (ContainerState.PAUSED == getContainerState()) {
+ dispatcher.getEventHandler().handle(
+ new ContainerResumeEvent(containerId,
+ "Container Resumed as some resources freed up"));
+ } else {
+ ContainersLauncherEventType launcherEvent =
+ ContainersLauncherEventType.LAUNCH_CONTAINER;
+ if (recoveredStatus == RecoveredContainerStatus.LAUNCHED) {
+ // try to recover a container that was previously launched
+ launcherEvent = ContainersLauncherEventType.RECOVER_CONTAINER;
+ }
+ containerLaunchStartTime = clock.getTime();
+ dispatcher.getEventHandler().handle(
+ new ContainersLauncherEvent(this, launcherEvent));
}
- containerLaunchStartTime = clock.getTime();
- dispatcher.getEventHandler().handle(
- new ContainersLauncherEvent(this, launcherEvent));
+
}
@SuppressWarnings("unchecked") // dispatcher not typed
@@ -824,6 +831,13 @@ public class ContainerImpl implements Container {
}
@SuppressWarnings("unchecked") // dispatcher not typed
+ @Override
+ public void sendPauseEvent(String description) {
+ dispatcher.getEventHandler().handle(
+ new ContainerPauseEvent(containerId, description));
+ }
+
+ @SuppressWarnings("unchecked") // dispatcher not typed
private void sendRelaunchEvent() {
ContainersLauncherEventType launcherEvent =
ContainersLauncherEventType.RELAUNCH_CONTAINER;
@@ -1779,7 +1793,7 @@ public class ContainerImpl implements Container {
/**
* Transitions upon receiving PAUSE_CONTAINER.
- * - RUNNING -> PAUSED
+ * - RUNNING -> PAUSING
*/
@SuppressWarnings("unchecked") // dispatcher not typed
static class PauseContainerTransition implements
http://git-wip-us.apache.org/repos/asf/hadoop/blob/10eeb8b5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java
index 644bdae..e39d2c1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java
@@ -19,6 +19,7 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager.scheduler;
import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
import org.apache.hadoop.yarn.api.records.ContainerId;
@@ -34,6 +35,7 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Cont
import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor
.ChangeMonitoringContainerResourceEvent;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitor;
@@ -74,7 +76,7 @@ public class ContainerScheduler extends AbstractService implements
queuedOpportunisticContainers = new LinkedHashMap<>();
// Used to keep track of containers that have been marked to be killed
- // to make room for a guaranteed container.
+ // or paused to make room for a guaranteed container.
private final Map<ContainerId, Container> oppContainersToKill =
new HashMap<>();
@@ -98,6 +100,8 @@ public class ContainerScheduler extends AbstractService implements
private final AsyncDispatcher dispatcher;
private final NodeManagerMetrics metrics;
+ private Boolean usePauseEventForPreemption = false;
+
/**
* Instantiate a Container Scheduler.
* @param context NodeManager Context.
@@ -112,6 +116,17 @@ public class ContainerScheduler extends AbstractService implements
DEFAULT_NM_OPPORTUNISTIC_CONTAINERS_MAX_QUEUE_LENGTH));
}
+
+ @Override
+ public void serviceInit(Configuration conf) throws Exception {
+ super.serviceInit(conf);
+ this.usePauseEventForPreemption =
+ conf.getBoolean(
+ YarnConfiguration.NM_CONTAINER_QUEUING_USE_PAUSE_FOR_PREEMPTION,
+ YarnConfiguration.
+ DEFAULT_NM_CONTAINER_QUEUING_USE_PAUSE_FOR_PREEMPTION);
+ }
+
@VisibleForTesting
public ContainerScheduler(Context context, AsyncDispatcher dispatcher,
NodeManagerMetrics metrics, int qLength) {
@@ -136,8 +151,9 @@ public class ContainerScheduler extends AbstractService implements
case SCHEDULE_CONTAINER:
scheduleContainer(event.getContainer());
break;
+ case CONTAINER_PAUSED:
case CONTAINER_COMPLETED:
- onContainerCompleted(event.getContainer());
+ onResourcesReclaimed(event.getContainer());
break;
case UPDATE_CONTAINER:
if (event instanceof UpdateContainerSchedulerEvent) {
@@ -203,9 +219,9 @@ public class ContainerScheduler extends AbstractService implements
queuedGuaranteedContainers.put(containerId,
updateEvent.getContainer());
}
- //Kill opportunistic containers if any to make room for
+ //Kill/pause opportunistic containers if any to make room for
// promotion request
- killOpportunisticContainers(updateEvent.getContainer());
+ reclaimOpportunisticContainerResources(updateEvent.getContainer());
} else {
// Demotion of queued container.. Should not happen too often
// since you should not find too many queued guaranteed
@@ -238,6 +254,12 @@ public class ContainerScheduler extends AbstractService implements
return this.queuedOpportunisticContainers.size();
}
+ @VisibleForTesting
+ public void setUsePauseEventForPreemption(
+ boolean usePauseEventForPreemption) {
+ this.usePauseEventForPreemption = usePauseEventForPreemption;
+ }
+
public OpportunisticContainersStatus getOpportunisticContainersStatus() {
this.opportunisticContainersStatus.setQueuedOpportContainers(
getNumQueuedOpportunisticContainers());
@@ -252,7 +274,7 @@ public class ContainerScheduler extends AbstractService implements
return this.opportunisticContainersStatus;
}
- private void onContainerCompleted(Container container) {
+ private void onResourcesReclaimed(Container container) {
oppContainersToKill.remove(container.getContainerId());
// This could be killed externally for eg. by the ContainerManager,
@@ -282,6 +304,24 @@ public class ContainerScheduler extends AbstractService implements
// Start pending guaranteed containers, if resources available.
boolean resourcesAvailable =
startContainersFromQueue(queuedGuaranteedContainers.values());
+ // Resume opportunistic containers, if resource available.
+ if (resourcesAvailable) {
+ List<Container> pausedContainers = new ArrayList<Container>();
+ Map<ContainerId, Container> containers =
+ context.getContainers();
+ for (Map.Entry<ContainerId, Container>entry : containers.entrySet()) {
+ ContainerId contId = entry.getKey();
+ // Find containers that were not already started and are in paused state
+ if(false == runningContainers.containsKey(contId)) {
+ if(containers.get(contId).getContainerState()
+ == ContainerState.PAUSED) {
+ pausedContainers.add(containers.get(contId));
+ }
+ }
+ }
+ resourcesAvailable =
+ startContainersFromQueue(pausedContainers);
+ }
// Start opportunistic containers, if resources available.
if (resourcesAvailable) {
startContainersFromQueue(queuedOpportunisticContainers.values());
@@ -378,7 +418,7 @@ public class ContainerScheduler extends AbstractService implements
// if the guaranteed container is queued, we need to preempt opportunistic
// containers for make room for it
if (queuedGuaranteedContainers.containsKey(container.getContainerId())) {
- killOpportunisticContainers(container);
+ reclaimOpportunisticContainerResources(container);
}
} else {
// Given an opportunistic container, we first try to start as many queuing
@@ -396,19 +436,30 @@ public class ContainerScheduler extends AbstractService implements
}
}
- private void killOpportunisticContainers(Container container) {
- List<Container> extraOpportContainersToKill =
- pickOpportunisticContainersToKill(container.getContainerId());
+ @SuppressWarnings("unchecked")
+ private void reclaimOpportunisticContainerResources(Container container) {
+ List<Container> extraOppContainersToReclaim =
+ pickOpportunisticContainersToReclaimResources(
+ container.getContainerId());
// Kill the opportunistic containers that were chosen.
- for (Container contToKill : extraOpportContainersToKill) {
- contToKill.sendKillEvent(
- ContainerExitStatus.KILLED_BY_CONTAINER_SCHEDULER,
- "Container Killed to make room for Guaranteed Container.");
- oppContainersToKill.put(contToKill.getContainerId(), contToKill);
+ for (Container contToReclaim : extraOppContainersToReclaim) {
+ String preemptionAction = usePauseEventForPreemption == true ? "paused" :
+ "resumed";
LOG.info(
- "Opportunistic container {} will be killed in order to start the "
+ "Container {} will be {} to start the "
+ "execution of guaranteed container {}.",
- contToKill.getContainerId(), container.getContainerId());
+ contToReclaim.getContainerId(), preemptionAction,
+ container.getContainerId());
+
+ if (usePauseEventForPreemption) {
+ contToReclaim.sendPauseEvent(
+ "Container Paused to make room for Guaranteed Container");
+ } else {
+ contToReclaim.sendKillEvent(
+ ContainerExitStatus.KILLED_BY_CONTAINER_SCHEDULER,
+ "Container Killed to make room for Guaranteed Container.");
+ }
+ oppContainersToKill.put(contToReclaim.getContainerId(), contToReclaim);
}
}
@@ -423,7 +474,7 @@ public class ContainerScheduler extends AbstractService implements
container.sendLaunchEvent();
}
- private List<Container> pickOpportunisticContainersToKill(
+ private List<Container> pickOpportunisticContainersToReclaimResources(
ContainerId containerToStartId) {
// The opportunistic containers that need to be killed for the
// given container to start.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/10eeb8b5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerQueuing.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerQueuing.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerQueuing.java
index 9676568..f3fc724 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerQueuing.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerQueuing.java
@@ -23,6 +23,8 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
import org.apache.hadoop.fs.UnsupportedFileSystemException;
import org.apache.hadoop.security.UserGroupInformation;
@@ -49,6 +51,7 @@ import org.apache.hadoop.yarn.server.nodemanager.DefaultContainerExecutor;
import org.apache.hadoop.yarn.server.nodemanager.DeletionService;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.BaseContainerManagerTest;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManagerImpl;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitor;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor.ContainersMonitorImpl;
@@ -124,18 +127,38 @@ public class TestContainerSchedulerQueuing extends BaseContainerManagerTest {
@Override
protected ContainerExecutor createContainerExecutor() {
DefaultContainerExecutor exec = new DefaultContainerExecutor() {
+ ConcurrentMap<String, Boolean> oversleepMap =
+ new ConcurrentHashMap<String, Boolean>();
@Override
public int launchContainer(ContainerStartContext ctx)
throws IOException, ConfigurationException {
+ oversleepMap.put(ctx.getContainer().getContainerId().toString(), false);
if (delayContainers) {
try {
Thread.sleep(10000);
+ if(oversleepMap.get(ctx.getContainer().getContainerId().toString())
+ == true) {
+ Thread.sleep(10000);
+ }
} catch (InterruptedException e) {
// Nothing..
}
}
return super.launchContainer(ctx);
}
+
+ @Override
+ public void pauseContainer(Container container) {
+ // To mimic pausing we force the container to be in the PAUSED state
+ // a little longer by oversleeping.
+ oversleepMap.put(container.getContainerId().toString(), true);
+ LOG.info("Container was paused");
+ }
+
+ @Override
+ public void resumeContainer(Container container) {
+ LOG.info("Container was resumed");
+ }
};
exec.setConf(conf);
return spy(exec);
@@ -506,6 +529,86 @@ public class TestContainerSchedulerQueuing extends BaseContainerManagerTest {
}
/**
+ * Submit two OPPORTUNISTIC and one GUARANTEED containers. The resources
+ * requests by each container as such that only one can run in parallel.
+ * Thus, the OPPORTUNISTIC container that started running, will be
+ * paused for the GUARANTEED container to start.
+ * Once the GUARANTEED container finishes its execution, the remaining
+ * OPPORTUNISTIC container will be executed.
+ * @throws Exception
+ */
+ @Test
+ public void testPauseOpportunisticForGuaranteedContainer() throws Exception {
+ containerManager.start();
+ containerManager.getContainerScheduler().
+ setUsePauseEventForPreemption(true);
+ ContainerLaunchContext containerLaunchContext =
+ recordFactory.newRecordInstance(ContainerLaunchContext.class);
+
+ List<StartContainerRequest> list = new ArrayList<>();
+ list.add(StartContainerRequest.newInstance(
+ containerLaunchContext,
+ createContainerToken(createContainerId(0), DUMMY_RM_IDENTIFIER,
+ context.getNodeId(),
+ user, BuilderUtils.newResource(2048, 1),
+ context.getContainerTokenSecretManager(), null,
+ ExecutionType.OPPORTUNISTIC)));
+
+ StartContainersRequest allRequests =
+ StartContainersRequest.newInstance(list);
+ containerManager.startContainers(allRequests);
+
+ BaseContainerManagerTest.waitForNMContainerState(containerManager,
+ createContainerId(0), ContainerState.RUNNING, 40);
+
+ list = new ArrayList<>();
+ list.add(StartContainerRequest.newInstance(
+ containerLaunchContext,
+ createContainerToken(createContainerId(1), DUMMY_RM_IDENTIFIER,
+ context.getNodeId(),
+ user, BuilderUtils.newResource(2048, 1),
+ context.getContainerTokenSecretManager(), null,
+ ExecutionType.GUARANTEED)));
+ allRequests =
+ StartContainersRequest.newInstance(list);
+
+ containerManager.startContainers(allRequests);
+
+ BaseContainerManagerTest.waitForNMContainerState(containerManager,
+ createContainerId(1), ContainerState.RUNNING, 40);
+
+ // Get container statuses. Container 0 should be paused, container 1
+ // should be running.
+ List<ContainerId> statList = new ArrayList<ContainerId>();
+ for (int i = 0; i < 2; i++) {
+ statList.add(createContainerId(i));
+ }
+ GetContainerStatusesRequest statRequest =
+ GetContainerStatusesRequest.newInstance(statList);
+ List<ContainerStatus> containerStatuses = containerManager
+ .getContainerStatuses(statRequest).getContainerStatuses();
+ for (ContainerStatus status : containerStatuses) {
+ if (status.getContainerId().equals(createContainerId(0))) {
+ Assert.assertTrue(status.getDiagnostics().contains(
+ "Container Paused to make room for Guaranteed Container"));
+ } else if (status.getContainerId().equals(createContainerId(1))) {
+ Assert.assertEquals(
+ org.apache.hadoop.yarn.api.records.ContainerState.RUNNING,
+ status.getState());
+ }
+ System.out.println("\nStatus : [" + status + "]\n");
+ }
+
+ // Make sure that the GUARANTEED container completes
+ BaseContainerManagerTest.waitForNMContainerState(containerManager,
+ createContainerId(1), ContainerState.DONE, 40);
+ // Make sure that the PAUSED opportunistic container resumes and
+ // starts running
+ BaseContainerManagerTest.waitForNMContainerState(containerManager,
+ createContainerId(0), ContainerState.DONE, 40);
+ }
+
+ /**
* 1. Submit a long running GUARANTEED container to hog all NM resources.
* 2. Submit 6 OPPORTUNISTIC containers, all of which will be queued.
* 3. Update the Queue Limit to 2.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/10eeb8b5/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java
index 4561e85c..7e03c95 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/MockContainer.java
@@ -235,4 +235,9 @@ public class MockContainer implements Container {
public boolean isRecovering() {
return false;
}
+
+ @Override
+ public void sendPauseEvent(String description) {
+
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[08/50] [abbrv] hadoop git commit: MAPREDUCE-6940. Copy-paste error
in the TaskAttemptUnsuccessfulCompletionEvent constructor. Contributed by
Oleg Danilov
Posted by as...@apache.org.
MAPREDUCE-6940. Copy-paste error in the TaskAttemptUnsuccessfulCompletionEvent constructor. Contributed by Oleg Danilov
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/0acc5e00
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/0acc5e00
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/0acc5e00
Branch: refs/heads/YARN-5972
Commit: 0acc5e00362602f027524637a86ca1bf80982986
Parents: de462da
Author: Jason Lowe <jl...@apache.org>
Authored: Wed Aug 16 16:34:06 2017 -0500
Committer: Jason Lowe <jl...@apache.org>
Committed: Wed Aug 16 16:34:06 2017 -0500
----------------------------------------------------------------------
.../TaskAttemptUnsuccessfulCompletionEvent.java | 28 ++++++++++----------
1 file changed, 14 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/0acc5e00/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptUnsuccessfulCompletionEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptUnsuccessfulCompletionEvent.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptUnsuccessfulCompletionEvent.java
index 1732d91..1752967 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptUnsuccessfulCompletionEvent.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/jobhistory/TaskAttemptUnsuccessfulCompletionEvent.java
@@ -60,7 +60,7 @@ public class TaskAttemptUnsuccessfulCompletionEvent implements HistoryEvent {
int[] physMemKbytes;
private static final Counters EMPTY_COUNTERS = new Counters();
- /**
+ /**
* Create an event to record the unsuccessful completion of attempts
* @param id Attempt ID
* @param taskType Type of the task
@@ -74,7 +74,7 @@ public class TaskAttemptUnsuccessfulCompletionEvent implements HistoryEvent {
* @param allSplits the "splits", or a pixelated graph of various
* measurable worker node state variables against progress.
* Currently there are four; wallclock time, CPU time,
- * virtual memory and physical memory.
+ * virtual memory and physical memory.
*/
public TaskAttemptUnsuccessfulCompletionEvent
(TaskAttemptID id, TaskType taskType,
@@ -101,7 +101,7 @@ public class TaskAttemptUnsuccessfulCompletionEvent implements HistoryEvent {
ProgressSplitsBlock.arrayGetPhysMemKbytes(allSplits);
}
- /**
+ /**
* @deprecated please use the constructor with an additional
* argument, an array of splits arrays instead. See
* {@link org.apache.hadoop.mapred.ProgressSplitsBlock}
@@ -117,19 +117,19 @@ public class TaskAttemptUnsuccessfulCompletionEvent implements HistoryEvent {
*/
public TaskAttemptUnsuccessfulCompletionEvent
(TaskAttemptID id, TaskType taskType,
- String status, long finishTime,
+ String status, long finishTime,
String hostname, String error) {
this(id, taskType, status, finishTime, hostname, -1, "",
error, EMPTY_COUNTERS, null);
}
-
+
public TaskAttemptUnsuccessfulCompletionEvent
(TaskAttemptID id, TaskType taskType,
String status, long finishTime,
String hostname, int port, String rackName,
String error, int[][] allSplits) {
this(id, taskType, status, finishTime, hostname, port,
- rackName, error, EMPTY_COUNTERS, null);
+ rackName, error, EMPTY_COUNTERS, allSplits);
}
TaskAttemptUnsuccessfulCompletionEvent() {}
@@ -162,9 +162,9 @@ public class TaskAttemptUnsuccessfulCompletionEvent implements HistoryEvent {
}
return datum;
}
-
-
-
+
+
+
public void setDatum(Object odatum) {
this.datum =
(TaskAttemptUnsuccessfulCompletion)odatum;
@@ -208,12 +208,12 @@ public class TaskAttemptUnsuccessfulCompletionEvent implements HistoryEvent {
public String getHostname() { return hostname; }
/** Get the rpc port for the host where the attempt executed */
public int getPort() { return port; }
-
+
/** Get the rack name of the node where the attempt ran */
public String getRackName() {
return rackName == null ? null : rackName.toString();
}
-
+
/** Get the error string */
public String getError() { return error.toString(); }
/** Get the task status */
@@ -224,12 +224,12 @@ public class TaskAttemptUnsuccessfulCompletionEvent implements HistoryEvent {
Counters getCounters() { return counters; }
/** Get the event type */
public EventType getEventType() {
- // Note that the task type can be setup/map/reduce/cleanup but the
+ // Note that the task type can be setup/map/reduce/cleanup but the
// attempt-type can only be map/reduce.
// find out if the task failed or got killed
boolean failed = TaskStatus.State.FAILED.toString().equals(getTaskStatus());
- return getTaskId().getTaskType() == TaskType.MAP
- ? (failed
+ return getTaskId().getTaskType() == TaskType.MAP
+ ? (failed
? EventType.MAP_ATTEMPT_FAILED
: EventType.MAP_ATTEMPT_KILLED)
: (failed
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[40/50] [abbrv] hadoop git commit: YARN-7053. Move curator
transaction support to ZKCuratorManager. (Jonathan Hung via Subru).
Posted by as...@apache.org.
YARN-7053. Move curator transaction support to ZKCuratorManager. (Jonathan Hung via Subru).
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4249172e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4249172e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4249172e
Branch: refs/heads/YARN-5972
Commit: 4249172e1419acdb2b69ae3db43dc59da2aa2e03
Parents: c379310
Author: Subru Krishnan <su...@apache.org>
Authored: Tue Aug 22 19:20:57 2017 -0700
Committer: Subru Krishnan <su...@apache.org>
Committed: Tue Aug 22 19:20:57 2017 -0700
----------------------------------------------------------------------
.../hadoop/util/curator/ZKCuratorManager.java | 88 +++++++++++-
.../util/curator/TestZKCuratorManager.java | 39 ++++++
.../recovery/ZKRMStateStore.java | 139 ++++++-------------
3 files changed, 164 insertions(+), 102 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4249172e/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/curator/ZKCuratorManager.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/curator/ZKCuratorManager.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/curator/ZKCuratorManager.java
index 9a031af..e1efcb5 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/curator/ZKCuratorManager.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/curator/ZKCuratorManager.java
@@ -26,6 +26,8 @@ import java.util.List;
import org.apache.curator.framework.AuthInfo;
import org.apache.curator.framework.CuratorFramework;
import org.apache.curator.framework.CuratorFrameworkFactory;
+import org.apache.curator.framework.api.transaction.CuratorTransaction;
+import org.apache.curator.framework.api.transaction.CuratorTransactionFinal;
import org.apache.curator.retry.RetryNTimes;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
@@ -54,7 +56,6 @@ public final class ZKCuratorManager {
/** Curator for ZooKeeper. */
private CuratorFramework curator;
-
public ZKCuratorManager(Configuration config) throws IOException {
this.conf = config;
}
@@ -119,7 +120,6 @@ public final class ZKCuratorManager {
/**
* Start the connection to the ZooKeeper ensemble.
- * @param conf Configuration for the connection.
* @throws IOException If the connection cannot be started.
*/
public void start() throws IOException {
@@ -128,7 +128,6 @@ public final class ZKCuratorManager {
/**
* Start the connection to the ZooKeeper ensemble.
- * @param conf Configuration for the connection.
* @param authInfos List of authentication keys.
* @throws IOException If the connection cannot be started.
*/
@@ -337,4 +336,87 @@ public final class ZKCuratorManager {
public static String getNodePath(String root, String nodeName) {
return root + "/" + nodeName;
}
+
+ public void safeCreate(String path, byte[] data, List<ACL> acl,
+ CreateMode mode, List<ACL> fencingACL, String fencingNodePath)
+ throws Exception {
+ if (!exists(path)) {
+ SafeTransaction transaction = createTransaction(fencingACL,
+ fencingNodePath);
+ transaction.create(path, data, acl, mode);
+ transaction.commit();
+ }
+ }
+
+ /**
+ * Deletes the path. Checks for existence of path as well.
+ * @param path Path to be deleted.
+ * @throws Exception if any problem occurs while performing deletion.
+ */
+ public void safeDelete(final String path, List<ACL> fencingACL,
+ String fencingNodePath) throws Exception {
+ if (exists(path)) {
+ SafeTransaction transaction = createTransaction(fencingACL,
+ fencingNodePath);
+ transaction.delete(path);
+ transaction.commit();
+ }
+ }
+
+ public void safeSetData(String path, byte[] data, int version,
+ List<ACL> fencingACL, String fencingNodePath)
+ throws Exception {
+ SafeTransaction transaction = createTransaction(fencingACL,
+ fencingNodePath);
+ transaction.setData(path, data, version);
+ transaction.commit();
+ }
+
+ public SafeTransaction createTransaction(List<ACL> fencingACL,
+ String fencingNodePath) throws Exception {
+ return new SafeTransaction(fencingACL, fencingNodePath);
+ }
+
+ /**
+ * Use curator transactions to ensure zk-operations are performed in an all
+ * or nothing fashion. This is equivalent to using ZooKeeper#multi.
+ *
+ * TODO (YARN-3774): Curator 3.0 introduces CuratorOp similar to Op. We ll
+ * have to rewrite this inner class when we adopt that.
+ */
+ public class SafeTransaction {
+ private CuratorTransactionFinal transactionFinal;
+ private String fencingNodePath;
+
+ SafeTransaction(List<ACL> fencingACL, String fencingNodePath)
+ throws Exception {
+ this.fencingNodePath = fencingNodePath;
+ CuratorTransaction transaction = curator.inTransaction();
+ transactionFinal = transaction.create()
+ .withMode(CreateMode.PERSISTENT).withACL(fencingACL)
+ .forPath(fencingNodePath, new byte[0]).and();
+ }
+
+ public void commit() throws Exception {
+ transactionFinal = transactionFinal.delete()
+ .forPath(fencingNodePath).and();
+ transactionFinal.commit();
+ }
+
+ public void create(String path, byte[] data, List<ACL> acl, CreateMode mode)
+ throws Exception {
+ transactionFinal = transactionFinal.create()
+ .withMode(mode).withACL(acl).forPath(path, data).and();
+ }
+
+ public void delete(String path) throws Exception {
+ transactionFinal = transactionFinal.delete().forPath(path).and();
+ }
+
+ public void setData(String path, byte[] data, int version)
+ throws Exception {
+ transactionFinal = transactionFinal.setData()
+ .withVersion(version).forPath(path, data).and();
+ }
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4249172e/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/curator/TestZKCuratorManager.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/curator/TestZKCuratorManager.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/curator/TestZKCuratorManager.java
index 3e78a44..486e89a 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/curator/TestZKCuratorManager.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/curator/TestZKCuratorManager.java
@@ -21,11 +21,15 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertTrue;
+import java.util.Arrays;
import java.util.List;
import org.apache.curator.test.TestingServer;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.util.ZKUtil;
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.data.ACL;
import org.junit.After;
import org.junit.Before;
import org.junit.Test;
@@ -92,4 +96,39 @@ public class TestZKCuratorManager {
children = curator.getChildren("/");
assertEquals(2, children.size());
}
+
+ @Test
+ public void testTransaction() throws Exception {
+ List<ACL> zkAcl = ZKUtil.parseACLs(CommonConfigurationKeys.ZK_ACL_DEFAULT);
+ String fencingNodePath = "/fencing";
+ String node1 = "/node1";
+ String node2 = "/node2";
+ byte[] testData = "testData".getBytes("UTF-8");
+ assertFalse(curator.exists(fencingNodePath));
+ assertFalse(curator.exists(node1));
+ assertFalse(curator.exists(node2));
+ ZKCuratorManager.SafeTransaction txn = curator.createTransaction(
+ zkAcl, fencingNodePath);
+ txn.create(node1, testData, zkAcl, CreateMode.PERSISTENT);
+ txn.create(node2, testData, zkAcl, CreateMode.PERSISTENT);
+ assertFalse(curator.exists(fencingNodePath));
+ assertFalse(curator.exists(node1));
+ assertFalse(curator.exists(node2));
+ txn.commit();
+ assertFalse(curator.exists(fencingNodePath));
+ assertTrue(curator.exists(node1));
+ assertTrue(curator.exists(node2));
+ assertTrue(Arrays.equals(testData, curator.getData(node1)));
+ assertTrue(Arrays.equals(testData, curator.getData(node2)));
+
+ byte[] setData = "setData".getBytes("UTF-8");
+ txn = curator.createTransaction(zkAcl, fencingNodePath);
+ txn.setData(node1, setData, -1);
+ txn.delete(node2);
+ assertTrue(curator.exists(node2));
+ assertTrue(Arrays.equals(testData, curator.getData(node1)));
+ txn.commit();
+ assertFalse(curator.exists(node2));
+ assertTrue(Arrays.equals(setData, curator.getData(node1)));
+ }
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4249172e/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
index a445e75..ac67dcd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
@@ -22,8 +22,6 @@ import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.curator.framework.CuratorFramework;
-import org.apache.curator.framework.api.transaction.CuratorTransaction;
-import org.apache.curator.framework.api.transaction.CuratorTransactionFinal;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.conf.Configuration;
@@ -31,6 +29,7 @@ import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.security.token.delegation.DelegationKey;
import org.apache.hadoop.util.ZKUtil;
import org.apache.hadoop.util.curator.ZKCuratorManager;
+import org.apache.hadoop.util.curator.ZKCuratorManager.SafeTransaction;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ReservationId;
@@ -416,9 +415,10 @@ public class ZKRMStateStore extends RMStateStore {
((VersionPBImpl) CURRENT_VERSION_INFO).getProto().toByteArray();
if (exists(versionNodePath)) {
- safeSetData(versionNodePath, data, -1);
+ zkManager.safeSetData(versionNodePath, data, -1, zkAcl, fencingNodePath);
} else {
- safeCreate(versionNodePath, data, zkAcl, CreateMode.PERSISTENT);
+ zkManager.safeCreate(versionNodePath, data, zkAcl, CreateMode.PERSISTENT,
+ zkAcl, fencingNodePath);
}
}
@@ -447,12 +447,14 @@ public class ZKRMStateStore extends RMStateStore {
// increment epoch and store it
byte[] storeData = Epoch.newInstance(currentEpoch + 1).getProto()
.toByteArray();
- safeSetData(epochNodePath, storeData, -1);
+ zkManager.safeSetData(epochNodePath, storeData, -1, zkAcl,
+ fencingNodePath);
} else {
// initialize epoch node with 1 for the next time.
byte[] storeData = Epoch.newInstance(currentEpoch + 1).getProto()
.toByteArray();
- safeCreate(epochNodePath, storeData, zkAcl, CreateMode.PERSISTENT);
+ zkManager.safeCreate(epochNodePath, storeData, zkAcl,
+ CreateMode.PERSISTENT, zkAcl, fencingNodePath);
}
return currentEpoch;
@@ -721,7 +723,7 @@ public class ZKRMStateStore extends RMStateStore {
// No apps stored under parent path.
if (children != null && children.isEmpty()) {
try {
- safeDelete(parentAppNode);
+ zkManager.safeDelete(parentAppNode, zkAcl, fencingNodePath);
if (LOG.isDebugEnabled()) {
LOG.debug("No leaf app node exists. Removing parent node " +
parentAppNode);
@@ -749,7 +751,8 @@ public class ZKRMStateStore extends RMStateStore {
byte[] appStateData = appStateDataPB.getProto().toByteArray();
if (appStateData.length <= zknodeLimit) {
- safeCreate(nodeCreatePath, appStateData, zkAcl, CreateMode.PERSISTENT);
+ zkManager.safeCreate(nodeCreatePath, appStateData, zkAcl,
+ CreateMode.PERSISTENT, zkAcl, fencingNodePath);
} else {
if (LOG.isDebugEnabled()) {
LOG.debug("Application state data size for " + appId + " is "
@@ -780,7 +783,8 @@ public class ZKRMStateStore extends RMStateStore {
String rootNode =
getSplitAppNodeParent(nodeUpdatePath, appIdNodeSplitIndex);
if (!exists(rootNode)) {
- safeCreate(rootNode, null, zkAcl, CreateMode.PERSISTENT);
+ zkManager.safeCreate(rootNode, null, zkAcl, CreateMode.PERSISTENT,
+ zkAcl, fencingNodePath);
}
}
}
@@ -794,9 +798,11 @@ public class ZKRMStateStore extends RMStateStore {
byte[] appStateData = appStateDataPB.getProto().toByteArray();
if (pathExists) {
- safeSetData(nodeUpdatePath, appStateData, -1);
+ zkManager.safeSetData(nodeUpdatePath, appStateData, -1, zkAcl,
+ fencingNodePath);
} else {
- safeCreate(nodeUpdatePath, appStateData, zkAcl, CreateMode.PERSISTENT);
+ zkManager.safeCreate(nodeUpdatePath, appStateData, zkAcl,
+ CreateMode.PERSISTENT, zkAcl, fencingNodePath);
if (LOG.isDebugEnabled()) {
LOG.debug("Path " + nodeUpdatePath + " for " + appId + " didn't " +
"exist. Creating a new znode to update the application state.");
@@ -839,9 +845,11 @@ public class ZKRMStateStore extends RMStateStore {
switch (operation) {
case UPDATE:
if (exists(path)) {
- safeSetData(path, attemptStateData, -1);
+ zkManager.safeSetData(path, attemptStateData, -1, zkAcl,
+ fencingNodePath);
} else {
- safeCreate(path, attemptStateData, zkAcl, CreateMode.PERSISTENT);
+ zkManager.safeCreate(path, attemptStateData, zkAcl,
+ CreateMode.PERSISTENT, zkAcl, fencingNodePath);
if (LOG.isDebugEnabled()) {
LOG.debug("Path " + path + " for " + appAttemptId + " didn't exist." +
" Created a new znode to update the application attempt state.");
@@ -849,10 +857,11 @@ public class ZKRMStateStore extends RMStateStore {
}
break;
case STORE:
- safeCreate(path, attemptStateData, zkAcl, CreateMode.PERSISTENT);
+ zkManager.safeCreate(path, attemptStateData, zkAcl, CreateMode.PERSISTENT,
+ zkAcl, fencingNodePath);
break;
case REMOVE:
- safeDelete(path);
+ zkManager.safeDelete(path, zkAcl, fencingNodePath);
break;
default:
break;
@@ -930,10 +939,10 @@ public class ZKRMStateStore extends RMStateStore {
for (ApplicationAttemptId attemptId : attempts) {
String attemptRemovePath =
getNodePath(appIdRemovePath, attemptId.toString());
- safeDelete(attemptRemovePath);
+ zkManager.safeDelete(attemptRemovePath, zkAcl, fencingNodePath);
}
}
- safeDelete(appIdRemovePath);
+ zkManager.safeDelete(appIdRemovePath, zkAcl, fencingNodePath);
} else {
CuratorFramework curatorFramework = zkManager.getCurator();
curatorFramework.delete().deletingChildrenIfNeeded().
@@ -947,7 +956,7 @@ public class ZKRMStateStore extends RMStateStore {
protected synchronized void storeRMDelegationTokenState(
RMDelegationTokenIdentifier rmDTIdentifier, Long renewDate)
throws Exception {
- SafeTransaction trx = new SafeTransaction();
+ SafeTransaction trx = zkManager.createTransaction(zkAcl, fencingNodePath);
addStoreOrUpdateOps(trx, rmDTIdentifier, renewDate, false);
trx.commit();
}
@@ -964,14 +973,14 @@ public class ZKRMStateStore extends RMStateStore {
+ rmDTIdentifier.getSequenceNumber());
}
- safeDelete(nodeRemovePath);
+ zkManager.safeDelete(nodeRemovePath, zkAcl, fencingNodePath);
}
@Override
protected synchronized void updateRMDelegationTokenState(
RMDelegationTokenIdentifier rmDTIdentifier, Long renewDate)
throws Exception {
- SafeTransaction trx = new SafeTransaction();
+ SafeTransaction trx = zkManager.createTransaction(zkAcl, fencingNodePath);
String nodeRemovePath =
getNodePath(delegationTokensRootPath, DELEGATION_TOKEN_PREFIX
+ rmDTIdentifier.getSequenceNumber());
@@ -1035,8 +1044,8 @@ public class ZKRMStateStore extends RMStateStore {
ByteArrayOutputStream os = new ByteArrayOutputStream();
try(DataOutputStream fsOut = new DataOutputStream(os)) {
delegationKey.write(fsOut);
- safeCreate(nodeCreatePath, os.toByteArray(), zkAcl,
- CreateMode.PERSISTENT);
+ zkManager.safeCreate(nodeCreatePath, os.toByteArray(), zkAcl,
+ CreateMode.PERSISTENT, zkAcl, fencingNodePath);
}
}
@@ -1051,7 +1060,7 @@ public class ZKRMStateStore extends RMStateStore {
LOG.debug("Removing RMDelegationKey_" + delegationKey.getKeyId());
}
- safeDelete(nodeRemovePath);
+ zkManager.safeDelete(nodeRemovePath, zkAcl, fencingNodePath);
}
@Override
@@ -1078,7 +1087,8 @@ public class ZKRMStateStore extends RMStateStore {
AMRMTokenSecretManagerState.newInstance(amrmTokenSecretManagerState);
byte[] stateData = data.getProto().toByteArray();
- safeSetData(amrmTokenSecretManagerRoot, stateData, -1);
+ zkManager.safeSetData(amrmTokenSecretManagerRoot, stateData, -1, zkAcl,
+ fencingNodePath);
}
@Override
@@ -1092,12 +1102,12 @@ public class ZKRMStateStore extends RMStateStore {
+ " for" + " plan " + planName);
}
- safeDelete(reservationPath);
+ zkManager.safeDelete(reservationPath, zkAcl, fencingNodePath);
List<String> reservationNodes = getChildren(planNodePath);
if (reservationNodes.isEmpty()) {
- safeDelete(planNodePath);
+ zkManager.safeDelete(planNodePath, zkAcl, fencingNodePath);
}
}
@@ -1105,7 +1115,7 @@ public class ZKRMStateStore extends RMStateStore {
protected synchronized void storeReservationState(
ReservationAllocationStateProto reservationAllocation, String planName,
String reservationIdName) throws Exception {
- SafeTransaction trx = new SafeTransaction();
+ SafeTransaction trx = zkManager.createTransaction(zkAcl, fencingNodePath);
addOrUpdateReservationState(reservationAllocation, planName,
reservationIdName, trx, false);
trx.commit();
@@ -1191,7 +1201,8 @@ public class ZKRMStateStore extends RMStateStore {
getNodePath(rootNode, nodeName.substring(0, splitIdx));
if (createParentIfNotExists && !exists(rootNodePath)) {
try {
- safeCreate(rootNodePath, null, zkAcl, CreateMode.PERSISTENT);
+ zkManager.safeCreate(rootNodePath, null, zkAcl, CreateMode.PERSISTENT,
+ zkAcl, fencingNodePath);
} catch (KeeperException.NodeExistsException e) {
if (LOG.isDebugEnabled()) {
LOG.debug("Unable to create app parent node " + rootNodePath +
@@ -1248,76 +1259,6 @@ public class ZKRMStateStore extends RMStateStore {
zkManager.delete(path);
}
- private void safeCreate(String path, byte[] data, List<ACL> acl,
- CreateMode mode) throws Exception {
- if (!exists(path)) {
- SafeTransaction transaction = new SafeTransaction();
- transaction.create(path, data, acl, mode);
- transaction.commit();
- }
- }
-
- /**
- * Deletes the path. Checks for existence of path as well.
- * @param path Path to be deleted.
- * @throws Exception if any problem occurs while performing deletion.
- */
- private void safeDelete(final String path) throws Exception {
- if (exists(path)) {
- SafeTransaction transaction = new SafeTransaction();
- transaction.delete(path);
- transaction.commit();
- }
- }
-
- private void safeSetData(String path, byte[] data, int version)
- throws Exception {
- SafeTransaction transaction = new SafeTransaction();
- transaction.setData(path, data, version);
- transaction.commit();
- }
-
- /**
- * Use curator transactions to ensure zk-operations are performed in an all
- * or nothing fashion. This is equivalent to using ZooKeeper#multi.
- *
- * TODO (YARN-3774): Curator 3.0 introduces CuratorOp similar to Op. We ll
- * have to rewrite this inner class when we adopt that.
- */
- private class SafeTransaction {
- private CuratorTransactionFinal transactionFinal;
-
- SafeTransaction() throws Exception {
- CuratorFramework curatorFramework = zkManager.getCurator();
- CuratorTransaction transaction = curatorFramework.inTransaction();
- transactionFinal = transaction.create()
- .withMode(CreateMode.PERSISTENT).withACL(zkAcl)
- .forPath(fencingNodePath, new byte[0]).and();
- }
-
- public void commit() throws Exception {
- transactionFinal = transactionFinal.delete()
- .forPath(fencingNodePath).and();
- transactionFinal.commit();
- }
-
- public void create(String path, byte[] data, List<ACL> acl, CreateMode mode)
- throws Exception {
- transactionFinal = transactionFinal.create()
- .withMode(mode).withACL(acl).forPath(path, data).and();
- }
-
- public void delete(String path) throws Exception {
- transactionFinal = transactionFinal.delete().forPath(path).and();
- }
-
- public void setData(String path, byte[] data, int version)
- throws Exception {
- transactionFinal = transactionFinal.setData()
- .withVersion(version).forPath(path, data).and();
- }
- }
-
/**
* Helper class that periodically attempts creating a znode to ensure that
* this RM continues to be the Active.
@@ -1332,7 +1273,7 @@ public class ZKRMStateStore extends RMStateStore {
try {
while (!isFencedState()) {
// Create and delete fencing node
- new SafeTransaction().commit();
+ zkManager.createTransaction(zkAcl, fencingNodePath).commit();
Thread.sleep(zkSessionTimeout);
}
} catch (InterruptedException ie) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[10/50] [abbrv] hadoop git commit: HDFS-12269. Better to return a Map
rather than HashMap in getErasureCodingCodecs. Contributed by Huafeng Wang.
Posted by as...@apache.org.
HDFS-12269. Better to return a Map rather than HashMap in getErasureCodingCodecs. Contributed by Huafeng Wang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/08aaa4b3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/08aaa4b3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/08aaa4b3
Branch: refs/heads/YARN-5972
Commit: 08aaa4b36fab44c3f47878b3c487db3b373ffccf
Parents: ab051bd
Author: Akira Ajisaka <aa...@apache.org>
Authored: Thu Aug 17 13:20:27 2017 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Thu Aug 17 13:20:27 2017 +0900
----------------------------------------------------------------------
.../java/org/apache/hadoop/io/erasurecode/CodecRegistry.java | 2 +-
.../src/main/java/org/apache/hadoop/hdfs/DFSClient.java | 2 +-
.../main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java | 3 +--
.../java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java | 4 ++--
.../hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java | 5 +++--
.../ClientNamenodeProtocolServerSideTranslatorPB.java | 3 +--
.../hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java | 4 ++--
.../org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java | 4 ++--
.../apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java | 4 ++--
.../src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java | 3 +--
.../java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java | 4 ++--
11 files changed, 18 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/08aaa4b3/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecRegistry.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecRegistry.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecRegistry.java
index fcf1349..daf91e2 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecRegistry.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/CodecRegistry.java
@@ -176,7 +176,7 @@ public final class CodecRegistry {
* @return a map of all codec names, and their corresponding code list
* separated by ','.
*/
- public HashMap<String, String> getCodec2CoderCompactMap() {
+ public Map<String, String> getCodec2CoderCompactMap() {
return coderNameCompactMap;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/08aaa4b3/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
index 88b273a..969522d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
@@ -2764,7 +2764,7 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
}
}
- public HashMap<String, String> getErasureCodingCodecs() throws IOException {
+ public Map<String, String> getErasureCodingCodecs() throws IOException {
checkOpen();
try (TraceScope ignored = tracer.newScope("getErasureCodingCodecs")) {
return namenode.getErasureCodingCodecs();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/08aaa4b3/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
index cd368d4..8f82d03 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
@@ -26,7 +26,6 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.EnumSet;
-import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -2585,7 +2584,7 @@ public class DistributedFileSystem extends FileSystem {
* @return all erasure coding codecs and coders supported by this file system.
* @throws IOException
*/
- public HashMap<String, String> getAllErasureCodingCodecs()
+ public Map<String, String> getAllErasureCodingCodecs()
throws IOException {
return dfs.getErasureCodingCodecs();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/08aaa4b3/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
index 45c6b32..eb9380d 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
@@ -19,8 +19,8 @@ package org.apache.hadoop.hdfs.protocol;
import java.io.IOException;
import java.util.EnumSet;
-import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
@@ -1601,7 +1601,7 @@ public interface ClientProtocol {
* @throws IOException
*/
@Idempotent
- HashMap<String, String> getErasureCodingCodecs() throws IOException;
+ Map<String, String> getErasureCodingCodecs() throws IOException;
/**
* Get the information about the EC policy for the path.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/08aaa4b3/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
index aed4117..ac06c1a 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
@@ -26,6 +26,7 @@ import java.util.List;
import com.google.common.collect.Lists;
+import java.util.Map;
import java.util.concurrent.TimeUnit;
import java.util.stream.Collectors;
@@ -1760,11 +1761,11 @@ public class ClientNamenodeProtocolTranslatorPB implements
}
@Override
- public HashMap<String, String> getErasureCodingCodecs() throws IOException {
+ public Map<String, String> getErasureCodingCodecs() throws IOException {
try {
GetErasureCodingCodecsResponseProto response = rpcProxy
.getErasureCodingCodecs(null, VOID_GET_EC_CODEC_REQUEST);
- HashMap<String, String> ecCodecs = new HashMap<String, String>();
+ Map<String, String> ecCodecs = new HashMap<>();
for (CodecProto codec : response.getCodecList()) {
ecCodecs.put(codec.getCodec(), codec.getCoders());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/08aaa4b3/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
index 38b81c6..a446276 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
@@ -21,7 +21,6 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.EnumSet;
-import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
@@ -1664,7 +1663,7 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
RpcController controller, GetErasureCodingCodecsRequestProto request)
throws ServiceException {
try {
- HashMap<String, String> codecs = server.getErasureCodingCodecs();
+ Map<String, String> codecs = server.getErasureCodingCodecs();
GetErasureCodingCodecsResponseProto.Builder resBuilder =
GetErasureCodingCodecsResponseProto.newBuilder();
for (Map.Entry<String, String> codec : codecs.entrySet()) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/08aaa4b3/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java
index 486503c..7895433 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java
@@ -25,8 +25,8 @@ import java.io.FileNotFoundException;
import java.io.IOException;
import java.util.Arrays;
import java.util.EnumSet;
-import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import java.util.stream.Collectors;
import com.google.common.base.Preconditions;
@@ -344,7 +344,7 @@ final class FSDirErasureCodingOp {
* @param fsn namespace
* @return {@link java.util.HashMap} array
*/
- static HashMap<String, String> getErasureCodingCodecs(final FSNamesystem fsn)
+ static Map<String, String> getErasureCodingCodecs(final FSNamesystem fsn)
throws IOException {
assert fsn.hasReadLock();
return CodecRegistry.getInstance().getCodec2CoderCompactMap();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/08aaa4b3/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 1cfaa54..2313335 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -7255,14 +7255,14 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
/**
* Get available erasure coding codecs and corresponding coders.
*/
- HashMap<String, String> getErasureCodingCodecs() throws IOException {
+ Map<String, String> getErasureCodingCodecs() throws IOException {
final String operationName = "getErasureCodingCodecs";
boolean success = false;
checkOperation(OperationCategory.READ);
readLock();
try {
checkOperation(OperationCategory.READ);
- final HashMap<String, String> ret =
+ final Map<String, String> ret =
FSDirErasureCodingOp.getErasureCodingCodecs(this);
success = true;
return ret;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/08aaa4b3/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
index d304d3d..7871202 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NameNodeRpcServer.java
@@ -37,9 +37,9 @@ import java.net.InetSocketAddress;
import java.util.Arrays;
import java.util.Collection;
import java.util.EnumSet;
-import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
+import java.util.Map;
import java.util.Set;
import java.util.concurrent.Callable;
@@ -2278,7 +2278,7 @@ public class NameNodeRpcServer implements NamenodeProtocols {
}
@Override // ClientProtocol
- public HashMap<String, String> getErasureCodingCodecs() throws IOException {
+ public Map<String, String> getErasureCodingCodecs() throws IOException {
checkNNStartup();
return namesystem.getErasureCodingCodecs();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/08aaa4b3/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java
index 46600a0..17a84f9 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java
@@ -33,7 +33,6 @@ import org.apache.hadoop.util.ToolRunner;
import java.io.IOException;
import java.util.Arrays;
import java.util.Collection;
-import java.util.HashMap;
import java.util.LinkedList;
import java.util.List;
import java.util.Map;
@@ -441,7 +440,7 @@ public class ECAdmin extends Configured implements Tool {
final DistributedFileSystem dfs = AdminHelper.getDFS(conf);
try {
- HashMap<String, String> codecs =
+ Map<String, String> codecs =
dfs.getAllErasureCodingCodecs();
if (codecs.isEmpty()) {
System.out.println("No erasure coding codecs are supported on the " +
http://git-wip-us.apache.org/repos/asf/hadoop/blob/08aaa4b3/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java
index 06edb1a..22e118f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java
@@ -50,8 +50,8 @@ import java.io.IOException;
import java.security.PrivilegedExceptionAction;
import java.util.Collection;
import java.util.EnumSet;
-import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import static org.apache.hadoop.test.GenericTestUtils.assertExceptionContains;
import static org.junit.Assert.*;
@@ -647,7 +647,7 @@ public class TestErasureCodingPolicies {
@Test
public void testGetAllErasureCodingCodecs() throws Exception {
- HashMap<String, String> allECCodecs = fs
+ Map<String, String> allECCodecs = fs
.getAllErasureCodingCodecs();
assertTrue("At least 3 system codecs should be enabled",
allECCodecs.size() >= 3);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[25/50] [abbrv] hadoop git commit: YARN-6979. [Addendum patch] Fixed
classname and added javadocs. (Kartheek Muthyala via asuresh)
Posted by as...@apache.org.
YARN-6979. [Addendum patch] Fixed classname and added javadocs. (Kartheek Muthyala via asuresh)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7a82d7bc
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7a82d7bc
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7a82d7bc
Branch: refs/heads/YARN-5972
Commit: 7a82d7bcea8124e1b65c275fac15bf2047d17471
Parents: 8410d86
Author: Arun Suresh <as...@apache.org>
Authored: Sun Aug 20 08:55:13 2017 -0700
Committer: Arun Suresh <as...@apache.org>
Committed: Sun Aug 20 10:24:05 2017 -0700
----------------------------------------------------------------------
.../CMgrDecreaseContainersResourceEvent.java | 37 ---------------
.../nodemanager/CMgrUpdateContainersEvent.java | 48 ++++++++++++++++++++
.../nodemanager/ContainerManagerEventType.java | 2 +-
.../nodemanager/NodeStatusUpdaterImpl.java | 8 ++--
.../containermanager/ContainerManagerImpl.java | 10 ++--
5 files changed, 57 insertions(+), 48 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7a82d7bc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/CMgrDecreaseContainersResourceEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/CMgrDecreaseContainersResourceEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/CMgrDecreaseContainersResourceEvent.java
deleted file mode 100644
index 9479d0b..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/CMgrDecreaseContainersResourceEvent.java
+++ /dev/null
@@ -1,37 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.yarn.server.nodemanager;
-
-import org.apache.hadoop.yarn.api.records.Container;
-import java.util.List;
-
-public class CMgrDecreaseContainersResourceEvent extends ContainerManagerEvent {
-
- private final List<Container> containersToDecrease;
-
- public CMgrDecreaseContainersResourceEvent(List<Container>
- containersToDecrease) {
- super(ContainerManagerEventType.DECREASE_CONTAINERS_RESOURCE);
- this.containersToDecrease = containersToDecrease;
- }
-
- public List<Container> getContainersToDecrease() {
- return this.containersToDecrease;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7a82d7bc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/CMgrUpdateContainersEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/CMgrUpdateContainersEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/CMgrUpdateContainersEvent.java
new file mode 100644
index 0000000..5e41701
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/CMgrUpdateContainersEvent.java
@@ -0,0 +1,48 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager;
+
+import org.apache.hadoop.yarn.api.records.Container;
+import java.util.List;
+
+/**
+ * Event used by the NodeStatusUpdater to notify the ContainerManager of
+ * container update commands it received from the RM.
+ */
+public class CMgrUpdateContainersEvent extends ContainerManagerEvent {
+
+ private final List<Container> containersToUpdate;
+
+ /**
+ * Create event.
+ * @param containersToUpdate Container to update.
+ */
+ public CMgrUpdateContainersEvent(List<Container> containersToUpdate) {
+ super(ContainerManagerEventType.UPDATE_CONTAINERS);
+ this.containersToUpdate = containersToUpdate;
+ }
+
+ /**
+ * Get containers to update.
+ * @return List of containers to update.
+ */
+ public List<Container> getContainersToUpdate() {
+ return this.containersToUpdate;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7a82d7bc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerManagerEventType.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerManagerEventType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerManagerEventType.java
index 8861bc7..8c5f7e2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerManagerEventType.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerManagerEventType.java
@@ -21,6 +21,6 @@ package org.apache.hadoop.yarn.server.nodemanager;
public enum ContainerManagerEventType {
FINISH_APPS,
FINISH_CONTAINERS,
- DECREASE_CONTAINERS_RESOURCE,
+ UPDATE_CONTAINERS,
SIGNAL_CONTAINERS
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7a82d7bc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
index 1d9256f..ade42e3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
@@ -1099,12 +1099,10 @@ public class NodeStatusUpdaterImpl extends AbstractService implements
parseCredentials(systemCredentials));
}
List<org.apache.hadoop.yarn.api.records.Container>
- containersToDecrease = response.getContainersToUpdate();
- if (!containersToDecrease.isEmpty()) {
+ containersToUpdate = response.getContainersToUpdate();
+ if (!containersToUpdate.isEmpty()) {
dispatcher.getEventHandler().handle(
- new CMgrDecreaseContainersResourceEvent(
- containersToDecrease)
- );
+ new CMgrUpdateContainersEvent(containersToUpdate));
}
// SignalContainer request originally comes from end users via
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7a82d7bc/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
index 12931bc..22484b7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
@@ -96,7 +96,7 @@ import org.apache.hadoop.yarn.server.api.records.ContainerQueuingLimit;
import org.apache.hadoop.yarn.server.api.records.OpportunisticContainersStatus;
import org.apache.hadoop.yarn.server.nodemanager.CMgrCompletedAppsEvent;
import org.apache.hadoop.yarn.server.nodemanager.CMgrCompletedContainersEvent;
-import org.apache.hadoop.yarn.server.nodemanager.CMgrDecreaseContainersResourceEvent;
+import org.apache.hadoop.yarn.server.nodemanager.CMgrUpdateContainersEvent;
import org.apache.hadoop.yarn.server.nodemanager.CMgrSignalContainersEvent;
import org.apache.hadoop.yarn.server.nodemanager.ContainerExecutor;
import org.apache.hadoop.yarn.server.nodemanager.ContainerManagerEvent;
@@ -1599,11 +1599,11 @@ public class ContainerManagerImpl extends CompositeService implements
"Container Killed by ResourceManager"));
}
break;
- case DECREASE_CONTAINERS_RESOURCE:
- CMgrDecreaseContainersResourceEvent containersDecreasedEvent =
- (CMgrDecreaseContainersResourceEvent) event;
+ case UPDATE_CONTAINERS:
+ CMgrUpdateContainersEvent containersDecreasedEvent =
+ (CMgrUpdateContainersEvent) event;
for (org.apache.hadoop.yarn.api.records.Container container
- : containersDecreasedEvent.getContainersToDecrease()) {
+ : containersDecreasedEvent.getContainersToUpdate()) {
try {
ContainerTokenIdentifier containerTokenIdentifier =
BuilderUtils.newContainerTokenIdentifier(
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[47/50] [abbrv] hadoop git commit: HDFS-10899. Add functionality to
re-encrypt EDEKs.
Posted by as...@apache.org.
HDFS-10899. Add functionality to re-encrypt EDEKs.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1000a2af
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1000a2af
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1000a2af
Branch: refs/heads/YARN-5972
Commit: 1000a2af04b24c123a3b08168f36b4e90420cab7
Parents: 26d8c8f
Author: Xiao Chen <xi...@apache.org>
Authored: Wed Aug 23 17:05:47 2017 -0700
Committer: Xiao Chen <xi...@apache.org>
Committed: Wed Aug 23 17:06:16 2017 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/hdfs/DFSClient.java | 20 +
.../hadoop/hdfs/DistributedFileSystem.java | 34 +
.../apache/hadoop/hdfs/client/HdfsAdmin.java | 29 +
.../hadoop/hdfs/protocol/ClientProtocol.java | 25 +
.../hadoop/hdfs/protocol/HdfsConstants.java | 7 +
.../hdfs/protocol/ReencryptionStatus.java | 216 ++
.../protocol/ReencryptionStatusIterator.java | 58 +
.../hdfs/protocol/ZoneReencryptionStatus.java | 257 +++
.../ClientNamenodeProtocolTranslatorPB.java | 39 +
.../hadoop/hdfs/protocolPB/PBHelperClient.java | 136 +-
.../src/main/proto/ClientNamenodeProtocol.proto | 4 +
.../src/main/proto/encryption.proto | 41 +
.../src/main/proto/hdfs.proto | 14 +
.../org/apache/hadoop/hdfs/DFSConfigKeys.java | 12 +
...tNamenodeProtocolServerSideTranslatorPB.java | 36 +
.../namenode/EncryptionFaultInjector.java | 9 +
.../server/namenode/EncryptionZoneManager.java | 351 +++-
.../server/namenode/FSDirEncryptionZoneOp.java | 238 ++-
.../hdfs/server/namenode/FSDirWriteFileOp.java | 4 +-
.../hdfs/server/namenode/FSDirXAttrOp.java | 7 +
.../hdfs/server/namenode/FSDirectory.java | 11 +-
.../hdfs/server/namenode/FSNamesystem.java | 86 +
.../hdfs/server/namenode/NameNodeRpcServer.java | 27 +
.../server/namenode/ReencryptionHandler.java | 940 +++++++++
.../server/namenode/ReencryptionUpdater.java | 523 +++++
.../apache/hadoop/hdfs/tools/CryptoAdmin.java | 134 +-
.../src/main/resources/hdfs-default.xml | 52 +
.../src/site/markdown/TransparentEncryption.md | 45 +-
.../hdfs/server/namenode/TestReencryption.java | 1847 ++++++++++++++++++
.../namenode/TestReencryptionHandler.java | 197 ++
.../src/test/resources/testCryptoConf.xml | 80 +
31 files changed, 5443 insertions(+), 36 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1000a2af/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
index 47c14e2..9239df3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
@@ -121,6 +121,7 @@ import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants.ReencryptAction;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.RollingUpgradeAction;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
@@ -131,11 +132,13 @@ import org.apache.hadoop.hdfs.protocol.NSQuotaExceededException;
import org.apache.hadoop.hdfs.protocol.OpenFileEntry;
import org.apache.hadoop.hdfs.protocol.OpenFilesIterator;
import org.apache.hadoop.hdfs.protocol.QuotaByStorageTypeExceededException;
+import org.apache.hadoop.hdfs.protocol.ReencryptionStatusIterator;
import org.apache.hadoop.hdfs.protocol.RollingUpgradeInfo;
import org.apache.hadoop.hdfs.protocol.SnapshotAccessControlException;
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus;
import org.apache.hadoop.hdfs.protocol.UnresolvedPathException;
+import org.apache.hadoop.hdfs.protocol.ZoneReencryptionStatus;
import org.apache.hadoop.hdfs.protocol.datatransfer.DataTransferProtoUtil;
import org.apache.hadoop.hdfs.protocol.datatransfer.IOStreamPair;
import org.apache.hadoop.hdfs.protocol.datatransfer.ReplaceDatanodeOnFailure;
@@ -2639,6 +2642,23 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
return new EncryptionZoneIterator(namenode, tracer);
}
+ public void reencryptEncryptionZone(String zone, ReencryptAction action)
+ throws IOException {
+ checkOpen();
+ try (TraceScope ignored = newPathTraceScope("reencryptEncryptionZone",
+ zone)) {
+ namenode.reencryptEncryptionZone(zone, action);
+ } catch (RemoteException re) {
+ throw re.unwrapRemoteException(AccessControlException.class,
+ SafeModeException.class, UnresolvedPathException.class);
+ }
+ }
+
+ public RemoteIterator<ZoneReencryptionStatus> listReencryptionStatus()
+ throws IOException {
+ checkOpen();
+ return new ReencryptionStatusIterator(namenode, tracer);
+ }
public void setErasureCodingPolicy(String src, String ecPolicyName)
throws IOException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1000a2af/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
index ceec2b3..f3605fa 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
@@ -83,11 +83,13 @@ import org.apache.hadoop.hdfs.protocol.DirectoryListing;
import org.apache.hadoop.hdfs.protocol.EncryptionZone;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants.ReencryptAction;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.RollingUpgradeAction;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.hdfs.protocol.HdfsLocatedFileStatus;
import org.apache.hadoop.hdfs.protocol.OpenFileEntry;
+import org.apache.hadoop.hdfs.protocol.ZoneReencryptionStatus;
import org.apache.hadoop.hdfs.protocol.RollingUpgradeInfo;
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus;
@@ -2314,6 +2316,38 @@ public class DistributedFileSystem extends FileSystem {
}
/* HDFS only */
+ public void reencryptEncryptionZone(final Path zone,
+ final ReencryptAction action) throws IOException {
+ final Path absF = fixRelativePart(zone);
+ new FileSystemLinkResolver<Void>() {
+ @Override
+ public Void doCall(final Path p) throws IOException {
+ dfs.reencryptEncryptionZone(getPathName(p), action);
+ return null;
+ }
+
+ @Override
+ public Void next(final FileSystem fs, final Path p) throws IOException {
+ if (fs instanceof DistributedFileSystem) {
+ DistributedFileSystem myDfs = (DistributedFileSystem) fs;
+ myDfs.reencryptEncryptionZone(p, action);
+ return null;
+ }
+ throw new UnsupportedOperationException(
+ "Cannot call reencryptEncryptionZone"
+ + " on a symlink to a non-DistributedFileSystem: " + zone
+ + " -> " + p);
+ }
+ }.resolve(this, absF);
+ }
+
+ /* HDFS only */
+ public RemoteIterator<ZoneReencryptionStatus> listReencryptionStatus()
+ throws IOException {
+ return dfs.listReencryptionStatus();
+ }
+
+ /* HDFS only */
public FileEncryptionInfo getFileEncryptionInfo(final Path path)
throws IOException {
Path absF = fixRelativePart(path);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1000a2af/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java
index abf341e..85a7efe 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/client/HdfsAdmin.java
@@ -49,6 +49,8 @@ import org.apache.hadoop.hdfs.protocol.CachePoolInfo;
import org.apache.hadoop.hdfs.protocol.EncryptionZone;
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants.ReencryptAction;
+import org.apache.hadoop.hdfs.protocol.ZoneReencryptionStatus;
import org.apache.hadoop.hdfs.protocol.OpenFileEntry;
import org.apache.hadoop.security.AccessControlException;
@@ -370,6 +372,33 @@ public class HdfsAdmin {
}
/**
+ * Performs re-encryption action for a given encryption zone.
+ *
+ * @param zone the root of the encryption zone
+ * @param action the re-encrypt action
+ * @throws IOException If any error occurs when handling re-encrypt action.
+ */
+ public void reencryptEncryptionZone(final Path zone,
+ final ReencryptAction action) throws IOException {
+ dfs.reencryptEncryptionZone(zone, action);
+ }
+
+ /**
+ * Returns a RemoteIterator which can be used to list all re-encryption
+ * information. For large numbers of re-encryptions, the iterator will fetch
+ * the list in a number of small batches.
+ * <p>
+ * Since the list is fetched in batches, it does not represent a
+ * consistent snapshot of the entire list of encryption zones.
+ * <p>
+ * This method can only be called by HDFS superusers.
+ */
+ public RemoteIterator<ZoneReencryptionStatus> listReencryptionStatus()
+ throws IOException {
+ return dfs.listReencryptionStatus();
+ }
+
+ /**
* Returns the FileEncryptionInfo on the HdfsFileStatus for the given path.
* The return value can be null if the path points to a directory, or a file
* that is not in an encryption zone.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1000a2af/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
index b0e85e5..b550467 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
@@ -41,6 +41,7 @@ import org.apache.hadoop.fs.permission.AclStatus;
import org.apache.hadoop.fs.permission.FsAction;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.inotify.EventBatchList;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants.ReencryptAction;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.RollingUpgradeAction;
import org.apache.hadoop.hdfs.security.token.block.DataEncryptionKey;
import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
@@ -1444,6 +1445,30 @@ public interface ClientProtocol {
long prevId) throws IOException;
/**
+ * Used to implement re-encryption of encryption zones.
+ *
+ * @param zone the encryption zone to re-encrypt.
+ * @param action the action for the re-encryption.
+ * @throws IOException
+ */
+ @AtMostOnce
+ void reencryptEncryptionZone(String zone, ReencryptAction action)
+ throws IOException;
+
+ /**
+ * Used to implement cursor-based batched listing of
+ * {@ZoneReencryptionStatus}s.
+ *
+ * @param prevId ID of the last item in the previous batch. If there is no
+ * previous batch, a negative value can be used.
+ * @return Batch of encryption zones.
+ * @throws IOException
+ */
+ @Idempotent
+ BatchedEntries<ZoneReencryptionStatus> listReencryptionStatus(long prevId)
+ throws IOException;
+
+ /**
* Set xattr of a file or directory.
* The name must be prefixed with the namespace followed by ".". For example,
* "user.attr".
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1000a2af/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java
index 2681f12..8c44293 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/HdfsConstants.java
@@ -144,6 +144,13 @@ public final class HdfsConstants {
ALL, LIVE, DEAD, DECOMMISSIONING, ENTERING_MAINTENANCE, IN_MAINTENANCE
}
+ /**
+ * Re-encrypt encryption zone actions.
+ */
+ public enum ReencryptAction {
+ CANCEL, START
+ }
+
/* Hidden constructor */
protected HdfsConstants() {
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1000a2af/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ReencryptionStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ReencryptionStatus.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ReencryptionStatus.java
new file mode 100644
index 0000000..e83ab52
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ReencryptionStatus.java
@@ -0,0 +1,216 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.protocol;
+
+import com.google.common.annotations.VisibleForTesting;
+import com.google.common.base.Preconditions;
+import com.google.common.collect.Lists;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.fs.BatchedRemoteIterator.BatchedListEntries;
+import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.ReencryptionInfoProto;
+import org.apache.hadoop.hdfs.protocol.ZoneReencryptionStatus.State;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.util.Map;
+import java.util.NavigableMap;
+import java.util.TreeMap;
+
+/**
+ * A class representing information about re-encrypting encryption zones. It
+ * contains a collection of @{code ZoneReencryptionStatus} for each EZ.
+ * <p>
+ * FSDirectory lock is used for synchronization (except test-only methods, which
+ * are not protected).
+ */
+@InterfaceAudience.Private
+public final class ReencryptionStatus {
+
+ public static final Logger LOG =
+ LoggerFactory.getLogger(ReencryptionStatus.class);
+
+ public static final BatchedListEntries<ZoneReencryptionStatus> EMPTY_LIST =
+ new BatchedListEntries<>(Lists.newArrayList(), false);
+
+ /**
+ * The zones that were submitted for re-encryption. This should preserve
+ * the order of submission.
+ */
+ private final TreeMap<Long, ZoneReencryptionStatus> zoneStatuses;
+ // Metrics
+ private long zonesReencrypted;
+
+ public ReencryptionStatus() {
+ zoneStatuses = new TreeMap<>();
+ }
+
+ @VisibleForTesting
+ public ReencryptionStatus(ReencryptionStatus rhs) {
+ if (rhs != null) {
+ this.zoneStatuses = new TreeMap<>(rhs.zoneStatuses);
+ this.zonesReencrypted = rhs.zonesReencrypted;
+ } else {
+ zoneStatuses = new TreeMap<>();
+ }
+ }
+
+ @VisibleForTesting
+ public void resetMetrics() {
+ zonesReencrypted = 0;
+ for (Map.Entry<Long, ZoneReencryptionStatus> entry : zoneStatuses
+ .entrySet()) {
+ entry.getValue().resetMetrics();
+ }
+ }
+
+ public ZoneReencryptionStatus getZoneStatus(final Long zondId) {
+ return zoneStatuses.get(zondId);
+ }
+
+ public void markZoneForRetry(final Long zoneId) {
+ final ZoneReencryptionStatus zs = zoneStatuses.get(zoneId);
+ Preconditions.checkNotNull(zs, "Cannot find zone " + zoneId);
+ LOG.info("Zone {} will retry re-encryption", zoneId);
+ zs.setState(State.Submitted);
+ }
+
+ public void markZoneStarted(final Long zoneId) {
+ final ZoneReencryptionStatus zs = zoneStatuses.get(zoneId);
+ Preconditions.checkNotNull(zs, "Cannot find zone " + zoneId);
+ LOG.info("Zone {} starts re-encryption processing", zoneId);
+ zs.setState(State.Processing);
+ }
+
+ public void markZoneCompleted(final Long zoneId) {
+ final ZoneReencryptionStatus zs = zoneStatuses.get(zoneId);
+ Preconditions.checkNotNull(zs, "Cannot find zone " + zoneId);
+ LOG.info("Zone {} completed re-encryption.", zoneId);
+ zs.setState(State.Completed);
+ zonesReencrypted++;
+ }
+
+ public Long getNextUnprocessedZone() {
+ for (Map.Entry<Long, ZoneReencryptionStatus> entry : zoneStatuses
+ .entrySet()) {
+ if (entry.getValue().getState() == State.Submitted) {
+ return entry.getKey();
+ }
+ }
+ return null;
+ }
+
+ public boolean hasRunningZone(final Long zoneId) {
+ return zoneStatuses.containsKey(zoneId)
+ && zoneStatuses.get(zoneId).getState() != State.Completed;
+ }
+
+ /**
+ * @param zoneId
+ * @return true if this is a zone is added.
+ */
+ private boolean addZoneIfNecessary(final Long zoneId, final String name,
+ final ReencryptionInfoProto reProto) {
+ if (!zoneStatuses.containsKey(zoneId)) {
+ LOG.debug("Adding zone {} for re-encryption status", zoneId);
+ Preconditions.checkNotNull(reProto);
+ final ZoneReencryptionStatus.Builder builder =
+ new ZoneReencryptionStatus.Builder();
+ builder.id(zoneId).zoneName(name)
+ .ezKeyVersionName(reProto.getEzKeyVersionName())
+ .submissionTime(reProto.getSubmissionTime())
+ .canceled(reProto.getCanceled())
+ .filesReencrypted(reProto.getNumReencrypted())
+ .fileReencryptionFailures(reProto.getNumFailures());
+ if (reProto.hasCompletionTime()) {
+ builder.completionTime(reProto.getCompletionTime());
+ builder.state(State.Completed);
+ zonesReencrypted++;
+ } else {
+ builder.state(State.Submitted);
+ }
+ if (reProto.hasLastFile()) {
+ builder.lastCheckpointFile(reProto.getLastFile());
+ }
+ return zoneStatuses.put(zoneId, builder.build()) == null;
+ }
+ return false;
+ }
+
+ public void updateZoneStatus(final Long zoneId, final String zonePath,
+ final ReencryptionInfoProto reProto) {
+ Preconditions.checkArgument(zoneId != null, "zoneId can't be null");
+ if (addZoneIfNecessary(zoneId, zonePath, reProto)) {
+ return;
+ }
+ final ZoneReencryptionStatus zs = getZoneStatus(zoneId);
+ assert zs != null;
+ if (reProto.hasCompletionTime()) {
+ zs.markZoneCompleted(reProto);
+ } else if (!reProto.hasLastFile() && !reProto.hasCompletionTime()) {
+ zs.markZoneSubmitted(reProto);
+ } else {
+ zs.updateZoneProcess(reProto);
+ }
+ }
+
+ public boolean removeZone(final Long zoneId) {
+ LOG.debug("Removing re-encryption status of zone {} ", zoneId);
+ return zoneStatuses.remove(zoneId) != null;
+ }
+
+ @VisibleForTesting
+ public int zonesQueued() {
+ int ret = 0;
+ for (Map.Entry<Long, ZoneReencryptionStatus> entry : zoneStatuses
+ .entrySet()) {
+ if (entry.getValue().getState() == State.Submitted) {
+ ret++;
+ }
+ }
+ return ret;
+ }
+
+ @VisibleForTesting
+ public int zonesTotal() {
+ return zoneStatuses.size();
+ }
+
+ @VisibleForTesting
+ public long getNumZonesReencrypted() {
+ return zonesReencrypted;
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ for (Map.Entry<Long, ZoneReencryptionStatus> entry : zoneStatuses
+ .entrySet()) {
+ sb.append("[zone:" + entry.getKey());
+ sb.append(" state:" + entry.getValue().getState());
+ sb.append(" lastProcessed:" + entry.getValue().getLastCheckpointFile());
+ sb.append(" filesReencrypted:" + entry.getValue().getFilesReencrypted());
+ sb.append(" fileReencryptionFailures:" + entry.getValue()
+ .getNumReencryptionFailures() + "]");
+ }
+ return sb.toString();
+ }
+
+ public NavigableMap<Long, ZoneReencryptionStatus> getZoneStatuses() {
+ return zoneStatuses;
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1000a2af/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ReencryptionStatusIterator.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ReencryptionStatusIterator.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ReencryptionStatusIterator.java
new file mode 100644
index 0000000..c8a8857
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ReencryptionStatusIterator.java
@@ -0,0 +1,58 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hdfs.protocol;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.fs.BatchedRemoteIterator;
+import org.apache.htrace.core.TraceScope;
+import org.apache.htrace.core.Tracer;
+
+import java.io.IOException;
+
+/**
+ * ReencryptionStatusIterator is a remote iterator that iterates over the
+ * reencryption status of encryption zones.
+ * It supports retrying in case of namenode failover.
+ */
+@InterfaceAudience.Private
+public class ReencryptionStatusIterator
+ extends BatchedRemoteIterator<Long, ZoneReencryptionStatus> {
+
+ private final ClientProtocol namenode;
+ private final Tracer tracer;
+
+ public ReencryptionStatusIterator(ClientProtocol namenode, Tracer tracer) {
+ super((long) 0);
+ this.namenode = namenode;
+ this.tracer = tracer;
+ }
+
+ @Override
+ public BatchedEntries<ZoneReencryptionStatus> makeRequest(Long prevId)
+ throws IOException {
+ try (TraceScope ignored = tracer.newScope("listReencryptionStatus")) {
+ return namenode.listReencryptionStatus(prevId);
+ }
+ }
+
+ @Override
+ public Long elementToPrevKey(ZoneReencryptionStatus entry) {
+ return entry.getId();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1000a2af/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ZoneReencryptionStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ZoneReencryptionStatus.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ZoneReencryptionStatus.java
new file mode 100644
index 0000000..9022b9f
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ZoneReencryptionStatus.java
@@ -0,0 +1,257 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.protocol;
+
+import com.google.common.base.Preconditions;
+import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.ReencryptionInfoProto;
+
+/**
+ * A class representing information about re-encryption of an encryption zone.
+ * <p>
+ * FSDirectory lock is used for synchronization (except test-only methods, which
+ * are not protected).
+ */
+public class ZoneReencryptionStatus {
+ /**
+ * State of re-encryption.
+ */
+ public enum State {
+ /**
+ * Submitted for re-encryption but hasn't been picked up.
+ * This is the initial state.
+ */
+ Submitted,
+ /**
+ * Currently re-encrypting.
+ */
+ Processing,
+ /**
+ * Re-encryption completed.
+ */
+ Completed
+ }
+
+ private long id;
+ private String zoneName;
+ /**
+ * The re-encryption status of the zone. Note this is a in-memory only
+ * variable. On failover it will always be submitted, or completed if
+ * completionTime != 0;
+ */
+ private State state;
+ private String ezKeyVersionName;
+ private long submissionTime;
+ private long completionTime;
+ private boolean canceled;
+ /**
+ * Name of last file processed. It's important to record name (not inode)
+ * because we want to restore to the position even if the inode is removed.
+ */
+ private String lastCheckpointFile;
+ private long filesReencrypted;
+ private long numReencryptionFailures;
+
+ /**
+ * Builder of {@link ZoneReencryptionStatus}.
+ */
+ public static final class Builder {
+ private long id;
+ private String zoneName;
+ private State state;
+ private String ezKeyVersionName;
+ private long submissionTime;
+ private long completionTime;
+ private boolean canceled;
+ private String lastCheckpointFile;
+ private long filesReencrypted;
+ private long fileReencryptionFailures;
+
+ public Builder() {
+ }
+
+ public Builder id(final long inodeid) {
+ id = inodeid;
+ return this;
+ }
+
+ public Builder zoneName(final String ezName) {
+ zoneName = ezName;
+ return this;
+ }
+
+ public Builder state(final State st) {
+ state = st;
+ return this;
+ }
+
+ public Builder ezKeyVersionName(final String ezkvn) {
+ ezKeyVersionName = ezkvn;
+ return this;
+ }
+
+ public Builder submissionTime(final long submission) {
+ submissionTime = submission;
+ return this;
+ }
+
+ public Builder completionTime(final long completion) {
+ completionTime = completion;
+ return this;
+ }
+
+ public Builder canceled(final boolean isCanceled) {
+ canceled = isCanceled;
+ return this;
+ }
+
+ public Builder lastCheckpointFile(final String lastFile) {
+ lastCheckpointFile = lastFile;
+ return this;
+ }
+
+ public Builder filesReencrypted(final long numReencrypted) {
+ filesReencrypted = numReencrypted;
+ return this;
+ }
+
+ public Builder fileReencryptionFailures(final long numFailures) {
+ fileReencryptionFailures = numFailures;
+ return this;
+ }
+
+ public ZoneReencryptionStatus build() {
+ ZoneReencryptionStatus ret = new ZoneReencryptionStatus();
+ Preconditions.checkArgument(id != 0, "no inode id set.");
+ Preconditions.checkNotNull(state, "no state id set.");
+ Preconditions.checkNotNull(ezKeyVersionName, "no keyVersionName set.");
+ Preconditions
+ .checkArgument(submissionTime != 0, "no submission time set.");
+ ret.id = this.id;
+ ret.zoneName = this.zoneName;
+ ret.state = this.state;
+ ret.ezKeyVersionName = this.ezKeyVersionName;
+ ret.submissionTime = this.submissionTime;
+ ret.completionTime = this.completionTime;
+ ret.canceled = this.canceled;
+ ret.lastCheckpointFile = this.lastCheckpointFile;
+ ret.filesReencrypted = this.filesReencrypted;
+ ret.numReencryptionFailures = this.fileReencryptionFailures;
+ return ret;
+ }
+ }
+
+ public ZoneReencryptionStatus() {
+ reset();
+ }
+
+ void resetMetrics() {
+ filesReencrypted = 0;
+ numReencryptionFailures = 0;
+ }
+
+ public long getId() {
+ return id;
+ }
+
+ public String getZoneName() {
+ return zoneName;
+ }
+
+ void setState(final State s) {
+ state = s;
+ }
+
+ public State getState() {
+ return state;
+ }
+
+ public String getEzKeyVersionName() {
+ return ezKeyVersionName;
+ }
+
+ public long getSubmissionTime() {
+ return submissionTime;
+ }
+
+ public long getCompletionTime() {
+ return completionTime;
+ }
+
+ public boolean isCanceled() {
+ return canceled;
+ }
+
+ public String getLastCheckpointFile() {
+ return lastCheckpointFile;
+ }
+
+ public long getFilesReencrypted() {
+ return filesReencrypted;
+ }
+
+ public long getNumReencryptionFailures() {
+ return numReencryptionFailures;
+ }
+
+ public void reset() {
+ state = State.Submitted;
+ ezKeyVersionName = null;
+ submissionTime = 0;
+ completionTime = 0;
+ canceled = false;
+ lastCheckpointFile = null;
+ resetMetrics();
+ }
+
+ /**
+ * Set the zone name. The zone name is resolved from inode id and set during
+ * a listReencryptionStatus call, for the crypto admin to consume.
+ */
+ public void setZoneName(final String name) {
+ Preconditions.checkNotNull(name == null);
+ zoneName = name;
+ }
+
+ public void cancel() {
+ canceled = true;
+ }
+
+ void markZoneCompleted(final ReencryptionInfoProto proto) {
+ state = ZoneReencryptionStatus.State.Completed;
+ completionTime = proto.getCompletionTime();
+ lastCheckpointFile = null;
+ canceled = proto.getCanceled();
+ filesReencrypted = proto.getNumReencrypted();
+ numReencryptionFailures = proto.getNumFailures();
+ }
+
+ void markZoneSubmitted(final ReencryptionInfoProto proto) {
+ reset();
+ state = ZoneReencryptionStatus.State.Submitted;
+ ezKeyVersionName = proto.getEzKeyVersionName();
+ submissionTime = proto.getSubmissionTime();
+ filesReencrypted = proto.getNumReencrypted();
+ numReencryptionFailures = proto.getNumFailures();
+ }
+
+ void updateZoneProcess(final ReencryptionInfoProto proto) {
+ lastCheckpointFile = proto.getLastFile();
+ filesReencrypted = proto.getNumReencrypted();
+ numReencryptionFailures = proto.getNumFailures();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1000a2af/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
index ac06c1a..ec7d93f 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolTranslatorPB.java
@@ -66,6 +66,7 @@ import org.apache.hadoop.hdfs.protocol.EncryptionZone;
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants.ReencryptAction;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.RollingUpgradeAction;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
@@ -74,6 +75,7 @@ import org.apache.hadoop.hdfs.protocol.LocatedBlock;
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
import org.apache.hadoop.hdfs.protocol.BlocksStats;
import org.apache.hadoop.hdfs.protocol.OpenFileEntry;
+import org.apache.hadoop.hdfs.protocol.ZoneReencryptionStatus;
import org.apache.hadoop.hdfs.protocol.RollingUpgradeInfo;
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus;
@@ -180,6 +182,10 @@ import org.apache.hadoop.hdfs.protocol.proto.EncryptionZonesProtos.CreateEncrypt
import org.apache.hadoop.hdfs.protocol.proto.EncryptionZonesProtos.EncryptionZoneProto;
import org.apache.hadoop.hdfs.protocol.proto.EncryptionZonesProtos.GetEZForPathRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.EncryptionZonesProtos.ListEncryptionZonesRequestProto;
+import org.apache.hadoop.hdfs.protocol.proto.EncryptionZonesProtos.ListReencryptionStatusRequestProto;
+import org.apache.hadoop.hdfs.protocol.proto.EncryptionZonesProtos.ListReencryptionStatusResponseProto;
+import org.apache.hadoop.hdfs.protocol.proto.EncryptionZonesProtos.ZoneReencryptionStatusProto;
+import org.apache.hadoop.hdfs.protocol.proto.EncryptionZonesProtos.ReencryptEncryptionZoneRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.AddErasureCodingPoliciesRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.AddErasureCodingPoliciesResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.GetErasureCodingPoliciesRequestProto;
@@ -1545,6 +1551,39 @@ public class ClientNamenodeProtocolTranslatorPB implements
}
@Override
+ public void reencryptEncryptionZone(String zone, ReencryptAction action)
+ throws IOException {
+ final ReencryptEncryptionZoneRequestProto.Builder builder =
+ ReencryptEncryptionZoneRequestProto.newBuilder();
+ builder.setZone(zone).setAction(PBHelperClient.convert(action));
+ ReencryptEncryptionZoneRequestProto req = builder.build();
+ try {
+ rpcProxy.reencryptEncryptionZone(null, req);
+ } catch (ServiceException e) {
+ throw ProtobufHelper.getRemoteException(e);
+ }
+ }
+
+ @Override
+ public BatchedEntries<ZoneReencryptionStatus> listReencryptionStatus(long id)
+ throws IOException {
+ final ListReencryptionStatusRequestProto req =
+ ListReencryptionStatusRequestProto.newBuilder().setId(id).build();
+ try {
+ ListReencryptionStatusResponseProto response =
+ rpcProxy.listReencryptionStatus(null, req);
+ List<ZoneReencryptionStatus> elements =
+ Lists.newArrayListWithCapacity(response.getStatusesCount());
+ for (ZoneReencryptionStatusProto p : response.getStatusesList()) {
+ elements.add(PBHelperClient.convert(p));
+ }
+ return new BatchedListEntries<>(elements, response.getHasMore());
+ } catch (ServiceException e) {
+ throw ProtobufHelper.getRemoteException(e);
+ }
+ }
+
+ @Override
public void setXAttr(String src, XAttr xAttr, EnumSet<XAttrSetFlag> flag)
throws IOException {
SetXAttrRequestProto req = SetXAttrRequestProto.newBuilder()
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1000a2af/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
index 5b1a687..30a3108 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocolPB/PBHelperClient.java
@@ -83,6 +83,7 @@ import org.apache.hadoop.hdfs.protocol.ExtendedBlock;
import org.apache.hadoop.hdfs.protocol.FsPermissionExtension;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.DatanodeReportType;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants.ReencryptAction;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.RollingUpgradeAction;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
@@ -99,6 +100,7 @@ import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffReportEntry;
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport.DiffType;
import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus;
import org.apache.hadoop.hdfs.protocol.SystemErasureCodingPolicies;
+import org.apache.hadoop.hdfs.protocol.ZoneReencryptionStatus;
import org.apache.hadoop.hdfs.protocol.proto.AclProtos.AclEntryProto;
import org.apache.hadoop.hdfs.protocol.proto.AclProtos.AclEntryProto.AclEntryScopeProto;
import org.apache.hadoop.hdfs.protocol.proto.AclProtos.AclEntryProto.AclEntryTypeProto;
@@ -129,6 +131,9 @@ import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.SafeMo
import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.ShortCircuitShmIdProto;
import org.apache.hadoop.hdfs.protocol.proto.DataTransferProtos.ShortCircuitShmSlotProto;
import org.apache.hadoop.hdfs.protocol.proto.EncryptionZonesProtos.EncryptionZoneProto;
+import org.apache.hadoop.hdfs.protocol.proto.EncryptionZonesProtos.ReencryptActionProto;
+import org.apache.hadoop.hdfs.protocol.proto.EncryptionZonesProtos.ReencryptionStateProto;
+import org.apache.hadoop.hdfs.protocol.proto.EncryptionZonesProtos.ZoneReencryptionStatusProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.AccessModeProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.AddECPolicyResponseProto;
@@ -157,6 +162,7 @@ import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.LocatedBlockProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.LocatedBlockProto.Builder;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.LocatedBlocksProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.QuotaUsageProto;
+import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.ReencryptionInfoProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.RollingUpgradeStatusProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.SnapshotDiffReportEntryProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.SnapshotDiffReportProto;
@@ -165,6 +171,7 @@ import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.SnapshottableDirectorySt
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.StorageReportProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.StorageTypeProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.StorageTypesProto;
+import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.ZoneEncryptionInfoProto;
import org.apache.hadoop.hdfs.protocol.proto.InotifyProtos;
import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos.GetXAttrsResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos.ListXAttrsResponseProto;
@@ -1678,6 +1685,17 @@ public class PBHelperClient {
return builder.build();
}
+ public static ReencryptActionProto convert(ReencryptAction a) {
+ switch (a) {
+ case CANCEL:
+ return ReencryptActionProto.CANCEL_REENCRYPT;
+ case START:
+ return ReencryptActionProto.START_REENCRYPT;
+ default:
+ throw new IllegalArgumentException("Unexpected value: " + a);
+ }
+ }
+
public static RollingUpgradeActionProto convert(RollingUpgradeAction a) {
switch (a) {
case QUERY:
@@ -2282,6 +2300,17 @@ public class PBHelperClient {
}
}
+ public static ReencryptAction convert(ReencryptActionProto a) {
+ switch (a) {
+ case CANCEL_REENCRYPT:
+ return ReencryptAction.CANCEL;
+ case START_REENCRYPT:
+ return ReencryptAction.START;
+ default:
+ throw new IllegalArgumentException("Unexpected value: " + a);
+ }
+ }
+
public static RollingUpgradeAction convert(RollingUpgradeActionProto a) {
switch (a) {
case QUERY:
@@ -2733,16 +2762,24 @@ public class PBHelperClient {
.build();
}
- public static HdfsProtos.ZoneEncryptionInfoProto convert(
- CipherSuite suite, CryptoProtocolVersion version, String keyName) {
+ public static ZoneEncryptionInfoProto convert(CipherSuite suite,
+ CryptoProtocolVersion version, String keyName) {
+ return convert(suite, version, keyName, null);
+ }
+
+ public static ZoneEncryptionInfoProto convert(CipherSuite suite,
+ CryptoProtocolVersion version, String keyName,
+ ReencryptionInfoProto proto) {
if (suite == null || version == null || keyName == null) {
return null;
}
- return HdfsProtos.ZoneEncryptionInfoProto.newBuilder()
- .setSuite(convert(suite))
- .setCryptoProtocolVersion(convert(version))
- .setKeyName(keyName)
- .build();
+ ZoneEncryptionInfoProto.Builder builder =
+ ZoneEncryptionInfoProto.newBuilder().setSuite(convert(suite))
+ .setCryptoProtocolVersion(convert(version)).setKeyName(keyName);
+ if (proto != null) {
+ builder.setReencryptionProto(proto);
+ }
+ return builder.build();
}
public static FileEncryptionInfo convert(
@@ -2759,6 +2796,91 @@ public class PBHelperClient {
ezKeyVersionName);
}
+ public static ReencryptionInfoProto convert(String ezkvn, Long submissionTime,
+ boolean isCanceled, long numReencrypted, long numFailures,
+ Long completionTime, String lastFile) {
+ if (ezkvn == null || submissionTime == null) {
+ return null;
+ }
+ ReencryptionInfoProto.Builder builder =
+ ReencryptionInfoProto.newBuilder().setEzKeyVersionName(ezkvn)
+ .setSubmissionTime(submissionTime).setCanceled(isCanceled)
+ .setNumReencrypted(numReencrypted).setNumFailures(numFailures);
+ if (completionTime != null) {
+ builder.setCompletionTime(completionTime);
+ }
+ if (lastFile != null) {
+ builder.setLastFile(lastFile);
+ }
+ return builder.build();
+ }
+
+ public static ZoneReencryptionStatusProto convert(ZoneReencryptionStatus zs) {
+ ZoneReencryptionStatusProto.Builder builder =
+ ZoneReencryptionStatusProto.newBuilder()
+ .setId(zs.getId())
+ .setPath(zs.getZoneName())
+ .setEzKeyVersionName(zs.getEzKeyVersionName())
+ .setSubmissionTime(zs.getSubmissionTime())
+ .setCanceled(zs.isCanceled())
+ .setNumReencrypted(zs.getFilesReencrypted())
+ .setNumFailures(zs.getNumReencryptionFailures());
+ switch (zs.getState()) {
+ case Submitted:
+ builder.setState(ReencryptionStateProto.SUBMITTED);
+ break;
+ case Processing:
+ builder.setState(ReencryptionStateProto.PROCESSING);
+ break;
+ case Completed:
+ builder.setState(ReencryptionStateProto.COMPLETED);
+ break;
+ default:
+ throw new IllegalArgumentException("Unknown state " + zs.getState());
+ }
+ final long completion = zs.getCompletionTime();
+ if (completion != 0) {
+ builder.setCompletionTime(completion);
+ }
+ final String file = zs.getLastCheckpointFile();
+ if (file != null) {
+ builder.setLastFile(file);
+ }
+ return builder.build();
+ }
+
+ public static ZoneReencryptionStatus convert(
+ ZoneReencryptionStatusProto proto) {
+ ZoneReencryptionStatus.State state;
+ switch (proto.getState()) {
+ case SUBMITTED:
+ state = ZoneReencryptionStatus.State.Submitted;
+ break;
+ case PROCESSING:
+ state = ZoneReencryptionStatus.State.Processing;
+ break;
+ case COMPLETED:
+ state = ZoneReencryptionStatus.State.Completed;
+ break;
+ default:
+ throw new IllegalArgumentException("Unknown state " + proto.getState());
+ }
+ ZoneReencryptionStatus.Builder builder = new ZoneReencryptionStatus.
+ Builder().
+ id(proto.getId()).zoneName(proto.getPath()).state(state)
+ .ezKeyVersionName(proto.getEzKeyVersionName())
+ .submissionTime(proto.getSubmissionTime()).canceled(proto.getCanceled())
+ .filesReencrypted(proto.getNumReencrypted())
+ .fileReencryptionFailures(proto.getNumFailures());
+ if (proto.hasCompletionTime()) {
+ builder.completionTime(proto.getCompletionTime());
+ }
+ if (proto.hasLastFile()) {
+ builder.lastCheckpointFile(proto.getLastFile());
+ }
+ return builder.build();
+ }
+
public static DatanodeInfo[] convert(DatanodeInfosProto datanodeInfosProto) {
List<DatanodeInfoProto> proto = datanodeInfosProto.getDatanodesList();
DatanodeInfo[] infos = new DatanodeInfo[proto.size()];
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1000a2af/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto
index 4f44c5e..3f108fa 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/ClientNamenodeProtocol.proto
@@ -941,6 +941,10 @@ service ClientNamenodeProtocol {
returns(CreateEncryptionZoneResponseProto);
rpc listEncryptionZones(ListEncryptionZonesRequestProto)
returns(ListEncryptionZonesResponseProto);
+ rpc reencryptEncryptionZone(ReencryptEncryptionZoneRequestProto)
+ returns(ReencryptEncryptionZoneResponseProto);
+ rpc listReencryptionStatus(ListReencryptionStatusRequestProto)
+ returns(ListReencryptionStatusResponseProto);
rpc getEZForPath(GetEZForPathRequestProto)
returns(GetEZForPathResponseProto);
rpc setErasureCodingPolicy(SetErasureCodingPolicyRequestProto)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1000a2af/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/encryption.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/encryption.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/encryption.proto
index 68b2f3a..75d3a0e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/encryption.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/encryption.proto
@@ -58,6 +58,47 @@ message ListEncryptionZonesResponseProto {
required bool hasMore = 2;
}
+enum ReencryptActionProto {
+ CANCEL_REENCRYPT = 1;
+ START_REENCRYPT = 2;
+}
+
+message ReencryptEncryptionZoneRequestProto {
+ required ReencryptActionProto action = 1;
+ required string zone = 2;
+}
+
+message ReencryptEncryptionZoneResponseProto {
+}
+
+message ListReencryptionStatusRequestProto {
+ required int64 id = 1;
+}
+
+enum ReencryptionStateProto {
+ SUBMITTED = 1;
+ PROCESSING = 2;
+ COMPLETED = 3;
+}
+
+message ZoneReencryptionStatusProto {
+ required int64 id = 1;
+ required string path = 2;
+ required ReencryptionStateProto state = 3;
+ required string ezKeyVersionName = 4;
+ required int64 submissionTime = 5;
+ required bool canceled = 6;
+ required int64 numReencrypted = 7;
+ required int64 numFailures = 8;
+ optional int64 completionTime = 9;
+ optional string lastFile = 10;
+}
+
+message ListReencryptionStatusResponseProto {
+ repeated ZoneReencryptionStatusProto statuses = 1;
+ required bool hasMore = 2;
+}
+
message GetEZForPathRequestProto {
required string src = 1;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1000a2af/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
index 7109980..59381bc 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/proto/hdfs.proto
@@ -313,6 +313,20 @@ message ZoneEncryptionInfoProto {
required CipherSuiteProto suite = 1;
required CryptoProtocolVersionProto cryptoProtocolVersion = 2;
required string keyName = 3;
+ optional ReencryptionInfoProto reencryptionProto = 4;
+}
+
+/**
+ * Re-encryption information for an encryption zone
+ */
+message ReencryptionInfoProto {
+ required string ezKeyVersionName = 1;
+ required uint64 submissionTime = 2;
+ required bool canceled = 3;
+ required int64 numReencrypted = 4;
+ required int64 numFailures = 5;
+ optional uint64 completionTime = 6;
+ optional string lastFile = 7;
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1000a2af/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
index f4c383e..7f60000 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/DFSConfigKeys.java
@@ -881,6 +881,8 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
HdfsClientConfigKeys.DFS_DATA_TRANSFER_SASL_PROPS_RESOLVER_CLASS_KEY;
public static final int DFS_NAMENODE_LIST_ENCRYPTION_ZONES_NUM_RESPONSES_DEFAULT = 100;
public static final String DFS_NAMENODE_LIST_ENCRYPTION_ZONES_NUM_RESPONSES = "dfs.namenode.list.encryption.zones.num.responses";
+ public static final int DFS_NAMENODE_LIST_REENCRYPTION_STATUS_NUM_RESPONSES_DEFAULT = 100;
+ public static final String DFS_NAMENODE_LIST_REENCRYPTION_STATUS_NUM_RESPONSES_KEY = "dfs.namenode.list.reencryption.status.num.responses";
public static final String DFS_NAMENODE_LIST_OPENFILES_NUM_RESPONSES =
"dfs.namenode.list.openfiles.num.responses";
public static final int DFS_NAMENODE_LIST_OPENFILES_NUM_RESPONSES_DEFAULT =
@@ -889,6 +891,16 @@ public class DFSConfigKeys extends CommonConfigurationKeys {
public static final int DFS_NAMENODE_EDEKCACHELOADER_INTERVAL_MS_DEFAULT = 1000;
public static final String DFS_NAMENODE_EDEKCACHELOADER_INITIAL_DELAY_MS_KEY = "dfs.namenode.edekcacheloader.initial.delay.ms";
public static final int DFS_NAMENODE_EDEKCACHELOADER_INITIAL_DELAY_MS_DEFAULT = 3000;
+ public static final String DFS_NAMENODE_REENCRYPT_SLEEP_INTERVAL_KEY = "dfs.namenode.reencrypt.sleep.interval";
+ public static final String DFS_NAMENODE_REENCRYPT_SLEEP_INTERVAL_DEFAULT = "1m";
+ public static final String DFS_NAMENODE_REENCRYPT_BATCH_SIZE_KEY = "dfs.namenode.reencrypt.batch.size";
+ public static final int DFS_NAMENODE_REENCRYPT_BATCH_SIZE_DEFAULT = 1000;
+ public static final String DFS_NAMENODE_REENCRYPT_THROTTLE_LIMIT_HANDLER_RATIO_KEY = "dfs.namenode.reencrypt.throttle.limit.handler.ratio";
+ public static final double DFS_NAMENODE_REENCRYPT_THROTTLE_LIMIT_HANDLER_RATIO_DEFAULT = 1.0;
+ public static final String DFS_NAMENODE_REENCRYPT_THROTTLE_LIMIT_UPDATER_RATIO_KEY = "dfs.namenode.reencrypt.throttle.limit.updater.ratio";
+ public static final double DFS_NAMENODE_REENCRYPT_THROTTLE_LIMIT_UPDATER_RATIO_DEFAULT = 1.0;
+ public static final String DFS_NAMENODE_REENCRYPT_EDEK_THREADS_KEY = "dfs.namenode.reencrypt.edek.threads";
+ public static final int DFS_NAMENODE_REENCRYPT_EDEK_THREADS_DEFAULT = 10;
// Journal-node related configs. These are read on the JN side.
public static final String DFS_JOURNALNODE_EDITS_DIR_KEY = "dfs.journalnode.edits.dir";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1000a2af/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
index a446276..44d5216 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/protocolPB/ClientNamenodeProtocolServerSideTranslatorPB.java
@@ -55,6 +55,7 @@ import org.apache.hadoop.hdfs.protocol.OpenFileEntry;
import org.apache.hadoop.hdfs.protocol.RollingUpgradeInfo;
import org.apache.hadoop.hdfs.protocol.SnapshotDiffReport;
import org.apache.hadoop.hdfs.protocol.SnapshottableDirectoryStatus;
+import org.apache.hadoop.hdfs.protocol.ZoneReencryptionStatus;
import org.apache.hadoop.hdfs.protocol.proto.AclProtos.GetAclStatusRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.AclProtos.GetAclStatusResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.AclProtos.ModifyAclEntriesRequestProto;
@@ -221,8 +222,12 @@ import org.apache.hadoop.hdfs.protocol.proto.EncryptionZonesProtos.GetEZForPathR
import org.apache.hadoop.hdfs.protocol.proto.EncryptionZonesProtos.GetEZForPathRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.EncryptionZonesProtos.ListEncryptionZonesResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.EncryptionZonesProtos.ListEncryptionZonesRequestProto;
+import org.apache.hadoop.hdfs.protocol.proto.EncryptionZonesProtos.ListReencryptionStatusRequestProto;
+import org.apache.hadoop.hdfs.protocol.proto.EncryptionZonesProtos.ListReencryptionStatusResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.AddErasureCodingPoliciesRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.AddErasureCodingPoliciesResponseProto;
+import org.apache.hadoop.hdfs.protocol.proto.EncryptionZonesProtos.ReencryptEncryptionZoneRequestProto;
+import org.apache.hadoop.hdfs.protocol.proto.EncryptionZonesProtos.ReencryptEncryptionZoneResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.GetErasureCodingPoliciesRequestProto;
import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.GetErasureCodingPoliciesResponseProto;
import org.apache.hadoop.hdfs.protocol.proto.ErasureCodingProtos.GetErasureCodingPolicyRequestProto;
@@ -1483,6 +1488,37 @@ public class ClientNamenodeProtocolServerSideTranslatorPB implements
}
@Override
+ public ReencryptEncryptionZoneResponseProto reencryptEncryptionZone(
+ RpcController controller, ReencryptEncryptionZoneRequestProto req)
+ throws ServiceException {
+ try {
+ server.reencryptEncryptionZone(req.getZone(),
+ PBHelperClient.convert(req.getAction()));
+ return ReencryptEncryptionZoneResponseProto.newBuilder().build();
+ } catch (IOException e) {
+ throw new ServiceException(e);
+ }
+ }
+
+ public ListReencryptionStatusResponseProto listReencryptionStatus(
+ RpcController controller, ListReencryptionStatusRequestProto req)
+ throws ServiceException {
+ try {
+ BatchedEntries<ZoneReencryptionStatus> entries = server
+ .listReencryptionStatus(req.getId());
+ ListReencryptionStatusResponseProto.Builder builder =
+ ListReencryptionStatusResponseProto.newBuilder();
+ builder.setHasMore(entries.hasMore());
+ for (int i=0; i<entries.size(); i++) {
+ builder.addStatuses(PBHelperClient.convert(entries.get(i)));
+ }
+ return builder.build();
+ } catch (IOException e) {
+ throw new ServiceException(e);
+ }
+ }
+
+ @Override
public SetErasureCodingPolicyResponseProto setErasureCodingPolicy(
RpcController controller, SetErasureCodingPolicyRequestProto req)
throws ServiceException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1000a2af/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionFaultInjector.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionFaultInjector.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionFaultInjector.java
index 104d8c3..e4a035e 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionFaultInjector.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionFaultInjector.java
@@ -42,4 +42,13 @@ public class EncryptionFaultInjector {
@VisibleForTesting
public void startFileAfterGenerateKey() throws IOException {}
+
+ @VisibleForTesting
+ public void reencryptEncryptedKeys() throws IOException {}
+
+ @VisibleForTesting
+ public void reencryptUpdaterProcessOneTask() throws IOException {}
+
+ @VisibleForTesting
+ public void reencryptUpdaterProcessCheckpoint() throws IOException {}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1000a2af/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
index 96e189b..d6302ba 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/EncryptionZoneManager.java
@@ -19,32 +19,45 @@ package org.apache.hadoop.hdfs.server.namenode;
import java.io.FileNotFoundException;
import java.io.IOException;
+import java.util.ArrayList;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.ThreadFactory;
import java.util.EnumSet;
import java.util.List;
import java.util.Map;
import java.util.NavigableMap;
import java.util.TreeMap;
+import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.collect.Lists;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.crypto.CipherSuite;
import org.apache.hadoop.crypto.CryptoProtocolVersion;
+import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension;
+import org.apache.hadoop.fs.ParentNotDirectoryException;
import org.apache.hadoop.fs.UnresolvedLinkException;
import org.apache.hadoop.fs.XAttr;
import org.apache.hadoop.fs.XAttrSetFlag;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.XAttrHelper;
import org.apache.hadoop.hdfs.protocol.EncryptionZone;
+import org.apache.hadoop.hdfs.protocol.ReencryptionStatus;
import org.apache.hadoop.hdfs.protocol.SnapshotAccessControlException;
+import org.apache.hadoop.hdfs.protocol.ZoneReencryptionStatus;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos;
import org.apache.hadoop.hdfs.protocolPB.PBHelperClient;
import org.apache.hadoop.hdfs.server.namenode.FSDirectory.DirOp;
+import org.apache.hadoop.security.AccessControlException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import static org.apache.hadoop.fs.BatchedRemoteIterator.BatchedListEntries;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_LIST_REENCRYPTION_STATUS_NUM_RESPONSES_DEFAULT;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_LIST_REENCRYPTION_STATUS_NUM_RESPONSES_KEY;
import static org.apache.hadoop.hdfs.server.common.HdfsServerConstants
.CRYPTO_XATTR_ENCRYPTION_ZONE;
@@ -57,7 +70,7 @@ import static org.apache.hadoop.hdfs.server.common.HdfsServerConstants
*/
public class EncryptionZoneManager {
- public static Logger LOG = LoggerFactory.getLogger(EncryptionZoneManager
+ public static final Logger LOG = LoggerFactory.getLogger(EncryptionZoneManager
.class);
/**
@@ -99,6 +112,91 @@ public class EncryptionZoneManager {
private TreeMap<Long, EncryptionZoneInt> encryptionZones = null;
private final FSDirectory dir;
private final int maxListEncryptionZonesResponses;
+ private final int maxListRecncryptionStatusResponses;
+
+ private ThreadFactory reencryptionThreadFactory;
+ private ExecutorService reencryptHandlerExecutor;
+ private ReencryptionHandler reencryptionHandler;
+ // Reencryption status is kept here to decouple status listing (which should
+ // work as long as NN is up), with the actual handler (which only exists if
+ // keyprovider exists)
+ private final ReencryptionStatus reencryptionStatus;
+
+ public static final BatchedListEntries<ZoneReencryptionStatus> EMPTY_LIST =
+ new BatchedListEntries<>(new ArrayList<ZoneReencryptionStatus>(), false);
+
+ @VisibleForTesting
+ public void pauseReencryptForTesting() {
+ reencryptionHandler.pauseForTesting();
+ }
+
+ @VisibleForTesting
+ public void resumeReencryptForTesting() {
+ reencryptionHandler.resumeForTesting();
+ }
+
+ @VisibleForTesting
+ public void pauseForTestingAfterNthSubmission(final int count) {
+ reencryptionHandler.pauseForTestingAfterNthSubmission(count);
+ }
+
+ @VisibleForTesting
+ public void pauseReencryptUpdaterForTesting() {
+ reencryptionHandler.pauseUpdaterForTesting();
+ }
+
+ @VisibleForTesting
+ public void resumeReencryptUpdaterForTesting() {
+ reencryptionHandler.resumeUpdaterForTesting();
+ }
+
+ @VisibleForTesting
+ public void pauseForTestingAfterNthCheckpoint(final String zone,
+ final int count) throws IOException {
+ INodesInPath iip;
+ dir.readLock();
+ try {
+ iip = dir.resolvePath(dir.getPermissionChecker(), zone, DirOp.READ);
+ } finally {
+ dir.readUnlock();
+ }
+ reencryptionHandler
+ .pauseForTestingAfterNthCheckpoint(iip.getLastINode().getId(), count);
+ }
+
+ @VisibleForTesting
+ public void resetMetricsForTesting() {
+ reencryptionStatus.resetMetrics();
+ }
+
+ @VisibleForTesting
+ public ReencryptionStatus getReencryptionStatus() {
+ return reencryptionStatus;
+ }
+
+ @VisibleForTesting
+ public ZoneReencryptionStatus getZoneStatus(final String zone)
+ throws IOException {
+ final FSPermissionChecker pc = dir.getPermissionChecker();
+ final INode inode;
+ dir.getFSNamesystem().readLock();
+ dir.readLock();
+ try {
+ final INodesInPath iip = dir.resolvePath(pc, zone, DirOp.READ);
+ inode = iip.getLastINode();
+ if (inode == null) {
+ return null;
+ }
+ return getReencryptionStatus().getZoneStatus(inode.getId());
+ } finally {
+ dir.readUnlock();
+ dir.getFSNamesystem().readUnlock();
+ }
+ }
+
+ FSDirectory getFSDirectory() {
+ return dir;
+ }
/**
* Construct a new EncryptionZoneManager.
@@ -115,6 +213,50 @@ public class EncryptionZoneManager {
DFSConfigKeys.DFS_NAMENODE_LIST_ENCRYPTION_ZONES_NUM_RESPONSES + " " +
"must be a positive integer."
);
+ if (getProvider() != null) {
+ reencryptionHandler = new ReencryptionHandler(this, conf);
+ reencryptionThreadFactory = new ThreadFactoryBuilder().setDaemon(true)
+ .setNameFormat("reencryptionHandlerThread #%d").build();
+ }
+ maxListRecncryptionStatusResponses =
+ conf.getInt(DFS_NAMENODE_LIST_REENCRYPTION_STATUS_NUM_RESPONSES_KEY,
+ DFS_NAMENODE_LIST_REENCRYPTION_STATUS_NUM_RESPONSES_DEFAULT);
+ Preconditions.checkArgument(maxListRecncryptionStatusResponses >= 0,
+ DFS_NAMENODE_LIST_REENCRYPTION_STATUS_NUM_RESPONSES_KEY +
+ " must be a positive integer."
+ );
+ reencryptionStatus = new ReencryptionStatus();
+ }
+
+ KeyProviderCryptoExtension getProvider() {
+ return dir.getProvider();
+ }
+
+ void startReencryptThreads() {
+ if (getProvider() == null) {
+ return;
+ }
+ Preconditions.checkNotNull(reencryptionHandler);
+ reencryptHandlerExecutor =
+ Executors.newSingleThreadExecutor(reencryptionThreadFactory);
+ reencryptHandlerExecutor.execute(reencryptionHandler);
+ reencryptionHandler.startUpdaterThread();
+ }
+
+ void stopReencryptThread() {
+ if (getProvider() == null || reencryptionHandler == null) {
+ return;
+ }
+ dir.writeLock();
+ try {
+ reencryptionHandler.stopThreads();
+ } finally {
+ dir.writeUnlock();
+ }
+ if (reencryptHandlerExecutor != null) {
+ reencryptHandlerExecutor.shutdownNow();
+ reencryptHandlerExecutor = null;
+ }
}
/**
@@ -157,7 +299,13 @@ public class EncryptionZoneManager {
void removeEncryptionZone(Long inodeId) {
assert dir.hasWriteLock();
if (hasCreatedEncryptionZone()) {
- encryptionZones.remove(inodeId);
+ if (encryptionZones.remove(inodeId) == null
+ || !getReencryptionStatus().hasRunningZone(inodeId)) {
+ return;
+ }
+ if (reencryptionHandler != null) {
+ reencryptionHandler.removeZone(inodeId);
+ }
}
}
@@ -173,13 +321,17 @@ public class EncryptionZoneManager {
}
/**
- * Returns the path of the EncryptionZoneInt.
+ * Returns the full path from an INode id.
* <p/>
* Called while holding the FSDirectory lock.
*/
- private String getFullPathName(EncryptionZoneInt ezi) {
+ String getFullPathName(Long nodeId) {
assert dir.hasReadLock();
- return dir.getInode(ezi.getINodeId()).getFullPathName();
+ INode inode = dir.getInode(nodeId);
+ if (inode == null) {
+ return null;
+ }
+ return inode.getFullPathName();
}
/**
@@ -247,7 +399,8 @@ public class EncryptionZoneManager {
if (ezi == null) {
return null;
} else {
- return new EncryptionZone(ezi.getINodeId(), getFullPathName(ezi),
+ return new EncryptionZone(ezi.getINodeId(),
+ getFullPathName(ezi.getINodeId()),
ezi.getSuite(), ezi.getVersion(), ezi.getKeyName());
}
}
@@ -284,8 +437,8 @@ public class EncryptionZoneManager {
if (srcInEZ) {
if (srcParentEZI != dstParentEZI) {
- final String srcEZPath = getFullPathName(srcParentEZI);
- final String dstEZPath = getFullPathName(dstParentEZI);
+ final String srcEZPath = getFullPathName(srcParentEZI.getINodeId());
+ final String dstEZPath = getFullPathName(dstParentEZI.getINodeId());
final StringBuilder sb = new StringBuilder(srcIIP.getPath());
sb.append(" can't be moved from encryption zone ");
sb.append(srcEZPath);
@@ -294,6 +447,24 @@ public class EncryptionZoneManager {
sb.append(".");
throw new IOException(sb.toString());
}
+ checkMoveValidityForReencryption(srcIIP.getPath(),
+ srcParentEZI.getINodeId());
+ } else if (dstInEZ) {
+ checkMoveValidityForReencryption(dstIIP.getPath(),
+ dstParentEZI.getINodeId());
+ }
+ }
+
+ private void checkMoveValidityForReencryption(final String pathName,
+ final long zoneId) throws IOException {
+ assert dir.hasReadLock();
+ final ZoneReencryptionStatus zs = reencryptionStatus.getZoneStatus(zoneId);
+ if (zs != null && zs.getState() != ZoneReencryptionStatus.State.Completed) {
+ final StringBuilder sb = new StringBuilder(pathName);
+ sb.append(" can't be moved because encryption zone ");
+ sb.append(getFullPathName(zoneId));
+ sb.append(" is currently under re-encryption");
+ throw new IOException(sb.toString());
}
}
@@ -364,19 +535,13 @@ public class EncryptionZoneManager {
/*
Skip EZs that are only present in snapshots. Re-resolve the path to
see if the path's current inode ID matches EZ map's INode ID.
-
+
INode#getFullPathName simply calls getParent recursively, so will return
- the INode's parents at the time it was snapshotted. It will not
+ the INode's parents at the time it was snapshotted. It will not
contain a reference INode.
*/
- final String pathName = getFullPathName(ezi);
- INode inode = dir.getInode(ezi.getINodeId());
- INode lastINode = null;
- if (inode.getParent() != null || inode.isRoot()) {
- INodesInPath iip = dir.getINodesInPath(pathName, DirOp.READ_LINK);
- lastINode = iip.getLastINode();
- }
- if (lastINode == null || lastINode.getId() != ezi.getINodeId()) {
+ final String pathName = getFullPathName(ezi.getINodeId());
+ if (!pathResolvesToId(ezi.getINodeId(), pathName)) {
continue;
}
// Add the EZ to the result list
@@ -392,6 +557,156 @@ public class EncryptionZoneManager {
}
/**
+ * Resolves the path to inode id, then check if it's the same as the inode id
+ * passed in. This is necessary to filter out zones in snapshots.
+ * @param zoneId
+ * @param zonePath
+ * @return true if path resolve to the id, false if not.
+ * @throws UnresolvedLinkException
+ */
+ private boolean pathResolvesToId(final long zoneId, final String zonePath)
+ throws UnresolvedLinkException, AccessControlException,
+ ParentNotDirectoryException {
+ assert dir.hasReadLock();
+ INode inode = dir.getInode(zoneId);
+ if (inode == null) {
+ return false;
+ }
+ INode lastINode = null;
+ if (inode.getParent() != null || inode.isRoot()) {
+ INodesInPath iip = dir.getINodesInPath(zonePath, DirOp.READ_LINK);
+ lastINode = iip.getLastINode();
+ }
+ if (lastINode == null || lastINode.getId() != zoneId) {
+ return false;
+ }
+ return true;
+ }
+
+ /**
+ * Re-encrypts the given encryption zone path. If the given path is not the
+ * root of an encryption zone, an exception is thrown.
+ */
+ XAttr reencryptEncryptionZone(final INodesInPath zoneIIP,
+ final String keyVersionName) throws IOException {
+ assert dir.hasWriteLock();
+ if (reencryptionHandler == null) {
+ throw new IOException("No key provider configured, re-encryption "
+ + "operation is rejected");
+ }
+ final INode inode = zoneIIP.getLastINode();
+ final String zoneName = zoneIIP.getPath();
+ checkEncryptionZoneRoot(inode, zoneName);
+ if (getReencryptionStatus().hasRunningZone(inode.getId())) {
+ throw new IOException("Zone " + zoneName
+ + " is already submitted for re-encryption.");
+ }
+ LOG.info("Zone {}({}) is submitted for re-encryption.", zoneName,
+ inode.getId());
+ XAttr ret = FSDirEncryptionZoneOp
+ .updateReencryptionSubmitted(dir, zoneIIP, keyVersionName);
+ reencryptionHandler.notifyNewSubmission();
+ return ret;
+ }
+
+ /**
+ * Cancels the currently-running re-encryption of the given encryption zone.
+ * If the given path is not the root of an encryption zone,
+ * * an exception is thrown.
+ */
+ List<XAttr> cancelReencryptEncryptionZone(final INodesInPath zoneIIP)
+ throws IOException {
+ assert dir.hasWriteLock();
+ if (reencryptionHandler == null) {
+ throw new IOException("No key provider configured, re-encryption "
+ + "operation is rejected");
+ }
+ final long zoneId = zoneIIP.getLastINode().getId();
+ final String zoneName = zoneIIP.getPath();
+ checkEncryptionZoneRoot(zoneIIP.getLastINode(), zoneName);
+ reencryptionHandler.cancelZone(zoneId, zoneName);
+ LOG.info("Cancelled zone {}({}) for re-encryption.", zoneName, zoneId);
+ return FSDirEncryptionZoneOp.updateReencryptionFinish(dir, zoneIIP,
+ reencryptionStatus.getZoneStatus(zoneId));
+ }
+
+ /**
+ * Cursor-based listing of zone re-encryption status.
+ * <p/>
+ * Called while holding the FSDirectory lock.
+ */
+ BatchedListEntries<ZoneReencryptionStatus> listReencryptionStatus(
+ final long prevId) throws IOException {
+ assert dir.hasReadLock();
+ if (!hasCreatedEncryptionZone()) {
+ return ReencryptionStatus.EMPTY_LIST;
+ }
+
+ NavigableMap<Long, ZoneReencryptionStatus> stats =
+ reencryptionStatus.getZoneStatuses();
+
+ if (stats.isEmpty()) {
+ return EMPTY_LIST;
+ }
+
+ NavigableMap<Long, ZoneReencryptionStatus> tailMap =
+ stats.tailMap(prevId, false);
+ final int numResp =
+ Math.min(maxListRecncryptionStatusResponses, tailMap.size());
+ final List<ZoneReencryptionStatus> ret =
+ Lists.newArrayListWithExpectedSize(numResp);
+ int count = 0;
+ for (ZoneReencryptionStatus zs : tailMap.values()) {
+ final String name = getFullPathName(zs.getId());
+ if (name == null || !pathResolvesToId(zs.getId(), name)) {
+ continue;
+ }
+ zs.setZoneName(name);
+ ret.add(zs);
+ ++count;
+ if (count >= numResp) {
+ break;
+ }
+ }
+ final boolean hasMore = (numResp < tailMap.size());
+ return new BatchedListEntries<>(ret, hasMore);
+ }
+
+ /**
+ * Return whether an INode is an encryption zone root.
+ */
+ boolean isEncryptionZoneRoot(final INode inode, final String name)
+ throws FileNotFoundException {
+ assert dir.hasReadLock();
+ if (inode == null) {
+ throw new FileNotFoundException("INode does not exist for " + name);
+ }
+ if (!inode.isDirectory()) {
+ return false;
+ }
+ if (!hasCreatedEncryptionZone()
+ || !encryptionZones.containsKey(inode.getId())) {
+ return false;
+ }
+ return true;
+ }
+
+ /**
+ * Return whether an INode is an encryption zone root.
+ *
+ * @param inode the zone inode
+ * @throws IOException if the inode is not a directory,
+ * or is a directory but not the root of an EZ.
+ */
+ void checkEncryptionZoneRoot(final INode inode, final String name)
+ throws IOException {
+ if (!isEncryptionZoneRoot(inode, name)) {
+ throw new IOException("Path " + name + " is not the root of an"
+ + " encryption zone.");
+ }
+ }
+
+ /**
* @return number of encryption zones.
*/
public int getNumEncryptionZones() {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[11/50] [abbrv] hadoop git commit: HDFS-11082. Provide replicated EC
policy to replicate files. Contributed by SammiChen.
Posted by as...@apache.org.
HDFS-11082. Provide replicated EC policy to replicate files. Contributed by SammiChen.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/96b3a6b9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/96b3a6b9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/96b3a6b9
Branch: refs/heads/YARN-5972
Commit: 96b3a6b9721e922d33fadc2459b561a85dbf9b8e
Parents: 08aaa4b
Author: Andrew Wang <wa...@apache.org>
Authored: Wed Aug 16 22:17:06 2017 -0700
Committer: Andrew Wang <wa...@apache.org>
Committed: Wed Aug 16 22:17:06 2017 -0700
----------------------------------------------------------------------
.../io/erasurecode/ErasureCodeConstants.java | 8 ++
.../java/org/apache/hadoop/hdfs/DFSClient.java | 3 +-
.../hadoop/hdfs/DistributedFileSystem.java | 6 +-
.../hadoop/hdfs/protocol/ClientProtocol.java | 6 +-
.../hdfs/protocol/ErasureCodingPolicy.java | 5 ++
.../protocol/SystemErasureCodingPolicies.java | 14 ++++
.../namenode/ErasureCodingPolicyManager.java | 13 ++-
.../server/namenode/FSDirErasureCodingOp.java | 13 ++-
.../hdfs/server/namenode/FSDirWriteFileOp.java | 2 +-
.../org/apache/hadoop/hdfs/tools/ECAdmin.java | 24 +++++-
.../src/site/markdown/HDFSErasureCoding.md | 16 ++--
.../hadoop/hdfs/TestErasureCodingPolicies.java | 81 ++++++++++++++++++
.../hdfs/server/namenode/TestFSImage.java | 87 ++++++++++++++++++++
.../test/resources/testErasureCodingConf.xml | 78 +++++++++++++++++-
14 files changed, 331 insertions(+), 25 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/96b3a6b9/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeConstants.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeConstants.java
index e0d7946..d3c3b6b 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeConstants.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/io/erasurecode/ErasureCodeConstants.java
@@ -30,6 +30,7 @@ public final class ErasureCodeConstants {
public static final String RS_LEGACY_CODEC_NAME = "rs-legacy";
public static final String XOR_CODEC_NAME = "xor";
public static final String HHXOR_CODEC_NAME = "hhxor";
+ public static final String REPLICATION_CODEC_NAME = "replication";
public static final ECSchema RS_6_3_SCHEMA = new ECSchema(
RS_CODEC_NAME, 6, 3);
@@ -45,4 +46,11 @@ public final class ErasureCodeConstants {
public static final ECSchema RS_10_4_SCHEMA = new ECSchema(
RS_CODEC_NAME, 10, 4);
+
+ public static final ECSchema REPLICATION_1_2_SCHEMA = new ECSchema(
+ REPLICATION_CODEC_NAME, 1, 2);
+
+ public static final byte USER_DEFINED_POLICY_START_ID = (byte) 64;
+ public static final byte REPLICATION_POLICY_ID = (byte) 63;
+ public static final String REPLICATION_POLICY_NAME = REPLICATION_CODEC_NAME;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/96b3a6b9/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
index 969522d..47c14e2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClient.java
@@ -3044,7 +3044,8 @@ public class DFSClient implements java.io.Closeable, RemotePeerFactory,
*
* @param src path to get the information for
* @return Returns the policy information if file or directory on the path is
- * erasure coded, null otherwise
+ * erasure coded, null otherwise. Null will be returned if directory or file
+ * has REPLICATION policy.
* @throws IOException
*/
http://git-wip-us.apache.org/repos/asf/hadoop/blob/96b3a6b9/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
index 8f82d03..ceec2b3 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DistributedFileSystem.java
@@ -2540,7 +2540,8 @@ public class DistributedFileSystem extends FileSystem {
*
* @param path The path of the file or directory
* @return Returns the policy information if file or directory on the path
- * is erasure coded, null otherwise
+ * is erasure coded, null otherwise. Null will be returned if directory or
+ * file has REPLICATION policy.
* @throws IOException
*/
public ErasureCodingPolicy getErasureCodingPolicy(final Path path)
@@ -2567,7 +2568,8 @@ public class DistributedFileSystem extends FileSystem {
}
/**
- * Retrieve all the erasure coding policies supported by this file system.
+ * Retrieve all the erasure coding policies supported by this file system,
+ * excluding REPLICATION policy.
*
* @return all erasure coding policies supported by this file system.
* @throws IOException
http://git-wip-us.apache.org/repos/asf/hadoop/blob/96b3a6b9/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
index eb9380d..b0e85e5 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ClientProtocol.java
@@ -1588,7 +1588,8 @@ public interface ClientProtocol {
/**
- * Get the erasure coding policies loaded in Namenode.
+ * Get the erasure coding policies loaded in Namenode, excluding REPLICATION
+ * policy.
*
* @throws IOException
*/
@@ -1604,7 +1605,8 @@ public interface ClientProtocol {
Map<String, String> getErasureCodingCodecs() throws IOException;
/**
- * Get the information about the EC policy for the path.
+ * Get the information about the EC policy for the path. Null will be returned
+ * if directory or file has REPLICATION policy.
*
* @param src path to get the info for
* @throws IOException
http://git-wip-us.apache.org/repos/asf/hadoop/blob/96b3a6b9/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ErasureCodingPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ErasureCodingPolicy.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ErasureCodingPolicy.java
index 7afc377..501b67c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ErasureCodingPolicy.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/ErasureCodingPolicy.java
@@ -25,6 +25,7 @@ import org.apache.commons.lang.builder.HashCodeBuilder;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.io.erasurecode.ECSchema;
+import org.apache.hadoop.io.erasurecode.ErasureCodeConstants;
/**
* A policy about how to write/read/code an erasure coding file.
@@ -107,6 +108,10 @@ public final class ErasureCodingPolicy implements Serializable {
this.id = id;
}
+ public boolean isReplicationPolicy() {
+ return (id == ErasureCodeConstants.REPLICATION_POLICY_ID);
+ }
+
@Override
public boolean equals(Object o) {
if (o == null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/96b3a6b9/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SystemErasureCodingPolicies.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SystemErasureCodingPolicies.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SystemErasureCodingPolicies.java
index 2cd838b..f0efe76 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SystemErasureCodingPolicies.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/protocol/SystemErasureCodingPolicies.java
@@ -68,6 +68,13 @@ public final class SystemErasureCodingPolicies {
new ErasureCodingPolicy(ErasureCodeConstants.RS_10_4_SCHEMA,
DEFAULT_CELLSIZE, RS_10_4_POLICY_ID);
+ // REPLICATION policy is always enabled.
+ private static final ErasureCodingPolicy REPLICATION_POLICY =
+ new ErasureCodingPolicy(ErasureCodeConstants.REPLICATION_POLICY_NAME,
+ ErasureCodeConstants.REPLICATION_1_2_SCHEMA,
+ DEFAULT_CELLSIZE,
+ ErasureCodeConstants.REPLICATION_POLICY_ID);
+
private static final List<ErasureCodingPolicy> SYS_POLICIES =
Collections.unmodifiableList(Arrays.asList(
SYS_POLICY1, SYS_POLICY2, SYS_POLICY3, SYS_POLICY4,
@@ -118,4 +125,11 @@ public final class SystemErasureCodingPolicies {
public static ErasureCodingPolicy getByName(String name) {
return SYSTEM_POLICIES_BY_NAME.get(name);
}
+
+ /**
+ * Get the special REPLICATION policy.
+ */
+ public static ErasureCodingPolicy getReplicationPolicy() {
+ return REPLICATION_POLICY;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/96b3a6b9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java
index 18b8e8a..404a0aa 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingPolicyManager.java
@@ -27,6 +27,7 @@ import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.io.erasurecode.CodecUtil;
+import org.apache.hadoop.io.erasurecode.ErasureCodeConstants;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -48,7 +49,6 @@ public final class ErasureCodingPolicyManager {
public static Logger LOG = LoggerFactory.getLogger(
ErasureCodingPolicyManager.class);
- private static final byte USER_DEFINED_POLICY_START_ID = (byte) 64;
private int maxCellSize =
DFSConfigKeys.DFS_NAMENODE_EC_POLICIES_MAX_CELLSIZE_DEFAULT;
@@ -157,7 +157,13 @@ public final class ErasureCodingPolicyManager {
* Get enabled policy by policy name.
*/
public ErasureCodingPolicy getEnabledPolicyByName(String name) {
- return enabledPoliciesByName.get(name);
+ ErasureCodingPolicy ecPolicy = enabledPoliciesByName.get(name);
+ if (ecPolicy == null) {
+ if (name.equalsIgnoreCase(ErasureCodeConstants.REPLICATION_POLICY_NAME)) {
+ ecPolicy = SystemErasureCodingPolicies.getReplicationPolicy();
+ }
+ }
+ return ecPolicy;
}
/**
@@ -257,7 +263,8 @@ public final class ErasureCodingPolicyManager {
private byte getNextAvailablePolicyID() {
byte currentId = this.userPoliciesByID.keySet().stream()
- .max(Byte::compareTo).orElse(USER_DEFINED_POLICY_START_ID);
+ .max(Byte::compareTo).orElse(
+ ErasureCodeConstants.USER_DEFINED_POLICY_START_ID);
return (byte) (currentId + 1);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/96b3a6b9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java
index 7895433..426b42b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirErasureCodingOp.java
@@ -62,7 +62,7 @@ final class FSDirErasureCodingOp {
/**
* Check if the ecPolicyName is valid and enabled, return the corresponding
- * EC policy if is.
+ * EC policy if is, including the REPLICATION EC policy.
* @param fsn namespace
* @param ecPolicyName name of EC policy to be checked
* @return an erasure coding policy if ecPolicyName is valid and enabled
@@ -295,7 +295,12 @@ final class FSDirErasureCodingOp {
if (iip.getLastINode() == null) {
throw new FileNotFoundException("Path not found: " + iip.getPath());
}
- return getErasureCodingPolicyForPath(fsd, iip);
+
+ ErasureCodingPolicy ecPolicy = getErasureCodingPolicyForPath(fsd, iip);
+ if (ecPolicy != null && ecPolicy.isReplicationPolicy()) {
+ ecPolicy = null;
+ }
+ return ecPolicy;
}
/**
@@ -312,7 +317,8 @@ final class FSDirErasureCodingOp {
}
/**
- * Get the erasure coding policy. This does not do any permission checking.
+ * Get the erasure coding policy, including the REPLICATION policy. This does
+ * not do any permission checking.
*
* @param fsn namespace
* @param iip inodes in the path containing the file
@@ -350,6 +356,7 @@ final class FSDirErasureCodingOp {
return CodecRegistry.getInstance().getCodec2CoderCompactMap();
}
+ //return erasure coding policy for path, including REPLICATION policy
private static ErasureCodingPolicy getErasureCodingPolicyForPath(
FSDirectory fsd, INodesInPath iip) throws IOException {
Preconditions.checkNotNull(iip, "INodes cannot be null");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/96b3a6b9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
index a62cddd..7ab05d7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSDirWriteFileOp.java
@@ -541,7 +541,7 @@ class FSDirWriteFileOp {
ecPolicy = FSDirErasureCodingOp.unprotectedGetErasureCodingPolicy(
fsd.getFSNamesystem(), existing);
}
- if (ecPolicy != null) {
+ if (ecPolicy != null && (!ecPolicy.isReplicationPolicy())) {
isStriped = true;
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/96b3a6b9/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java
index 17a84f9..55d85ff 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/ECAdmin.java
@@ -25,6 +25,7 @@ import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.protocol.AddECPolicyResponse;
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
import org.apache.hadoop.hdfs.util.ECPolicyLoader;
+import org.apache.hadoop.io.erasurecode.ErasureCodeConstants;
import org.apache.hadoop.tools.TableListing;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.Tool;
@@ -309,7 +310,8 @@ public class ECAdmin extends Configured implements Tool {
@Override
public String getShortUsage() {
- return "[" + getName() + " -path <path> -policy <policy>]\n";
+ return "[" + getName() +
+ " -path <path> [-policy <policy>] [-replicate]]\n";
}
@Override
@@ -318,9 +320,13 @@ public class ECAdmin extends Configured implements Tool {
listing.addRow("<path>", "The path of the file/directory to set " +
"the erasure coding policy");
listing.addRow("<policy>", "The name of the erasure coding policy");
+ listing.addRow("-replicate",
+ "force 3x replication scheme on the directory");
return getShortUsage() + "\n" +
"Set the erasure coding policy for a file/directory.\n\n" +
- listing.toString();
+ listing.toString() + "\n" +
+ "-replicate and -policy are optional arguments. They cannot been " +
+ "used at the same time";
}
@Override
@@ -332,14 +338,24 @@ public class ECAdmin extends Configured implements Tool {
return 1;
}
- final String ecPolicyName = StringUtils.popOptionWithArgument("-policy",
+ String ecPolicyName = StringUtils.popOptionWithArgument("-policy",
args);
+ final boolean replicate = StringUtils.popOption("-replicate", args);
if (args.size() > 0) {
System.err.println(getName() + ": Too many arguments");
return 1;
}
+ if (replicate) {
+ if (ecPolicyName != null) {
+ System.err.println(getName() +
+ ": -replicate and -policy cannot been used at the same time");
+ return 2;
+ }
+ ecPolicyName = ErasureCodeConstants.REPLICATION_POLICY_NAME;
+ }
+
final Path p = new Path(path);
final DistributedFileSystem dfs = AdminHelper.getDFS(p.toUri(), conf);
try {
@@ -353,7 +369,7 @@ public class ECAdmin extends Configured implements Tool {
}
} catch (Exception e) {
System.err.println(AdminHelper.prettifyException(e));
- return 2;
+ return 3;
}
return 0;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/96b3a6b9/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md
index 4a48c2a..786b512 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/HDFSErasureCoding.md
@@ -65,9 +65,11 @@ Architecture
2. _The size of a striping cell._ This determines the granularity of striped reads and writes, including buffer sizes and encoding work.
- Policies are named *codec*-*num data blocks*-*num parity blocks*-*cell size*. Currently, five built-in policies are supported: `RS-3-2-64k`, `RS-6-3-64k`, `RS-10-4-64k`, `RS-LEGACY-6-3-64k`, and `XOR-2-1-64k`.
+ Policies are named *codec*-*num data blocks*-*num parity blocks*-*cell size*. Currently, six built-in policies are supported: `RS-3-2-64k`, `RS-6-3-64k`, `RS-10-4-64k`, `RS-LEGACY-6-3-64k`, `XOR-2-1-64k` and `REPLICATION`.
- By default, all built-in erasure coding policies are disabled.
+ `REPLICATION` is a special policy. It can only be set on directory, to force the directory to adopt 3x replication scheme, instead of inheriting its ancestor's erasure coding policy. This policy makes it possible to interleave 3x replication scheme directory with erasure coding directory.
+
+ `REPLICATION` policy is always enabled. For other built-in policies, unless they are configured in `dfs.namenode.ec.policies.enabled` property, otherwise they are disabled by default.
Similar to HDFS storage policies, erasure coding policies are set on a directory. When a file is created, it inherits the EC policy of its nearest ancestor directory.
@@ -112,7 +114,7 @@ Deployment
what EC policies can be set by clients. It does not affect the behavior of already set file or directory-level EC policies.
By default, all built-in erasure coding policies are disabled. Typically, the cluster administrator will enable set of policies by including them
- in the `dfs .namenode.ec.policies.enabled` configuration based on the size of the cluster and the desired fault-tolerance properties. For instance,
+ in the `dfs.namenode.ec.policies.enabled` configuration based on the size of the cluster and the desired fault-tolerance properties. For instance,
for a cluster with 9 racks, a policy like `RS-10-4-64k` will not preserve rack-level fault-tolerance, and `RS-6-3-64k` or `RS-3-2-64k` might
be more appropriate. If the administrator only cares about node-level fault-tolerance, `RS-10-4-64k` would still be appropriate as long as
there are at least 14 DataNodes in the cluster.
@@ -153,7 +155,7 @@ Deployment
HDFS provides an `ec` subcommand to perform administrative commands related to erasure coding.
hdfs ec [generic options]
- [-setPolicy -policy <policyName> -path <path>]
+ [-setPolicy -path <path> [-policy <policyName>] [-replicate]]
[-getPolicy -path <path>]
[-unsetPolicy -path <path>]
[-listPolicies]
@@ -165,7 +167,7 @@ Deployment
Below are the details about each command.
- * `[-setPolicy -policy <policyName> -path <path>]`
+ * `[-setPolicy -path <path> [-policy <policyName>] [-replicate]]`
Sets an erasure coding policy on a directory at the specified path.
@@ -175,6 +177,10 @@ Below are the details about each command.
This parameter can be omitted if a 'dfs.namenode.ec.system.default.policy' configuration is set.
The EC policy of the path will be set with the default value in configuration.
+ `-replicate` apply the special `REPLICATION` policy on the directory, force the directory to adopt 3x replication scheme.
+
+ `-replicate` and `-policy <policyName>` are optional arguments. They cannot be specified at the same time.
+
* `[-getPolicy -path <path>]`
http://git-wip-us.apache.org/repos/asf/hadoop/blob/96b3a6b9/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java
index 22e118f..47cdf23 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingPolicies.java
@@ -732,4 +732,85 @@ public class TestErasureCodingPolicies {
}
});
}
+
+ @Test
+ public void testReplicationPolicy() throws Exception {
+ ErasureCodingPolicy replicaPolicy =
+ SystemErasureCodingPolicies.getReplicationPolicy();
+
+ final Path rootDir = new Path("/striped");
+ final Path replicaDir = new Path(rootDir, "replica");
+ final Path subReplicaDir = new Path(replicaDir, "replica");
+ final Path replicaFile = new Path(replicaDir, "file");
+ final Path subReplicaFile = new Path(subReplicaDir, "file");
+
+ fs.mkdirs(rootDir);
+ fs.setErasureCodingPolicy(rootDir, ecPolicy.getName());
+
+ // 1. At first, child directory will inherit parent's EC policy
+ fs.mkdirs(replicaDir);
+ fs.createFile(replicaFile).build().close();
+ HdfsFileStatus fileStatus = (HdfsFileStatus)fs.getFileStatus(replicaFile);
+ assertEquals("File should inherit EC policy.", ecPolicy, fileStatus
+ .getErasureCodingPolicy());
+ assertEquals("File should be a EC file.", true, fileStatus
+ .isErasureCoded());
+ assertEquals("File should have the same EC policy as its ancestor.",
+ ecPolicy, fs.getErasureCodingPolicy(replicaFile));
+ fs.delete(replicaFile, false);
+
+ // 2. Set replication policy on child directory, then get back the policy
+ fs.setErasureCodingPolicy(replicaDir, replicaPolicy.getName());
+ ErasureCodingPolicy temp = fs.getErasureCodingPolicy(replicaDir);
+ assertEquals("Directory should hide replication EC policy.",
+ null, temp);
+
+ // 3. New file will be replication file. Please be noted that replication
+ // policy only set on directory, not on file
+ fs.createFile(replicaFile).build().close();
+ assertEquals("Replication file should have default replication factor.",
+ fs.getDefaultReplication(),
+ fs.getFileStatus(replicaFile).getReplication());
+ fs.setReplication(replicaFile, (short) 2);
+ assertEquals("File should have replication factor as expected.",
+ 2, fs.getFileStatus(replicaFile).getReplication());
+ fileStatus = (HdfsFileStatus)fs.getFileStatus(replicaFile);
+ assertEquals("File should not have EC policy.", null, fileStatus
+ .getErasureCodingPolicy());
+ assertEquals("File should not be a EC file.", false,
+ fileStatus.isErasureCoded());
+ ErasureCodingPolicy ecPolicyOnFile = fs.getErasureCodingPolicy(replicaFile);
+ assertEquals("File should not have EC policy.", null, ecPolicyOnFile);
+ fs.delete(replicaFile, false);
+
+ // 4. New directory under replication directory, is also replication
+ // directory
+ fs.mkdirs(subReplicaDir);
+ assertEquals("Directory should inherit hiding replication EC policy.",
+ null, fs.getErasureCodingPolicy(subReplicaDir));
+ fs.createFile(subReplicaFile).build().close();
+ assertEquals("File should have default replication factor.",
+ fs.getDefaultReplication(),
+ fs.getFileStatus(subReplicaFile).getReplication());
+ fileStatus = (HdfsFileStatus)fs.getFileStatus(subReplicaFile);
+ assertEquals("File should not have EC policy.", null,
+ fileStatus.getErasureCodingPolicy());
+ assertEquals("File should not be a EC file.", false,
+ fileStatus.isErasureCoded());
+ assertEquals("File should not have EC policy.", null,
+ fs.getErasureCodingPolicy(subReplicaFile));
+ fs.delete(subReplicaFile, false);
+
+ // 5. Unset replication policy on directory, new file will be EC file
+ fs.unsetErasureCodingPolicy(replicaDir);
+ fs.createFile(subReplicaFile).build().close();
+ fileStatus = (HdfsFileStatus)fs.getFileStatus(subReplicaFile);
+ assertEquals("File should inherit EC policy.", ecPolicy,
+ fileStatus.getErasureCodingPolicy());
+ assertEquals("File should be a EC file.", true,
+ fileStatus.isErasureCoded());
+ assertEquals("File should have the same EC policy as its ancestor",
+ ecPolicy, fs.getErasureCodingPolicy(subReplicaFile));
+ fs.delete(subReplicaFile, false);
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/96b3a6b9/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java
index 22c40fb..9256056 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java
@@ -723,4 +723,91 @@ public class TestFSImage {
.getBlockType());
assertEquals(defaultBlockType, BlockType.CONTIGUOUS);
}
+
+ /**
+ * Test if a INodeFile under a replication EC policy directory
+ * can be saved by FSImageSerialization and loaded by FSImageFormat#Loader.
+ */
+ @Test
+ public void testSaveAndLoadFileUnderReplicationPolicyDir()
+ throws IOException {
+ Configuration conf = new Configuration();
+ DFSTestUtil.enableAllECPolicies(conf);
+ MiniDFSCluster cluster = null;
+ try {
+ cluster = new MiniDFSCluster.Builder(conf).build();
+ cluster.waitActive();
+ FSNamesystem fsn = cluster.getNamesystem();
+ DistributedFileSystem fs = cluster.getFileSystem();
+ ErasureCodingPolicy replicaPolicy =
+ SystemErasureCodingPolicies.getReplicationPolicy();
+ ErasureCodingPolicy defaultEcPolicy =
+ StripedFileTestUtil.getDefaultECPolicy();
+
+ final Path ecDir = new Path("/ec");
+ final Path replicaDir = new Path(ecDir, "replica");
+ final Path replicaFile1 = new Path(replicaDir, "f1");
+ final Path replicaFile2 = new Path(replicaDir, "f2");
+
+ // create root directory
+ fs.mkdir(ecDir, null);
+ fs.setErasureCodingPolicy(ecDir, defaultEcPolicy.getName());
+
+ // create directory, and set replication Policy
+ fs.mkdir(replicaDir, null);
+ fs.setErasureCodingPolicy(replicaDir, replicaPolicy.getName());
+
+ // create an empty file f1
+ fs.create(replicaFile1).close();
+
+ // create an under-construction file f2
+ FSDataOutputStream out = fs.create(replicaFile2, (short) 2);
+ out.writeBytes("hello");
+ ((DFSOutputStream) out.getWrappedStream()).hsync(EnumSet
+ .of(SyncFlag.UPDATE_LENGTH));
+
+ // checkpoint
+ fs.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
+ fs.saveNamespace();
+ fs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE);
+
+ cluster.restartNameNode();
+ cluster.waitActive();
+ fs = cluster.getFileSystem();
+
+ assertTrue(fs.getFileStatus(ecDir).isDirectory());
+ assertTrue(fs.getFileStatus(replicaDir).isDirectory());
+ assertTrue(fs.exists(replicaFile1));
+ assertTrue(fs.exists(replicaFile2));
+
+ // check directories
+ assertEquals("Directory should have default EC policy.",
+ defaultEcPolicy, fs.getErasureCodingPolicy(ecDir));
+ assertEquals("Directory should hide replication EC policy.",
+ null, fs.getErasureCodingPolicy(replicaDir));
+
+ // check file1
+ assertEquals("File should not have EC policy.", null,
+ fs.getErasureCodingPolicy(replicaFile1));
+ // check internals of file2
+ INodeFile file2Node =
+ fsn.dir.getINode4Write(replicaFile2.toString()).asFile();
+ assertEquals("hello".length(), file2Node.computeFileSize());
+ assertTrue(file2Node.isUnderConstruction());
+ BlockInfo[] blks = file2Node.getBlocks();
+ assertEquals(1, blks.length);
+ assertEquals(BlockUCState.UNDER_CONSTRUCTION, blks[0].getBlockUCState());
+ assertEquals("File should return expected replication factor.",
+ 2, blks[0].getReplication());
+ assertEquals("File should not have EC policy.", null,
+ fs.getErasureCodingPolicy(replicaFile2));
+ // check lease manager
+ Lease lease = fsn.leaseManager.getLease(file2Node);
+ Assert.assertNotNull(lease);
+ } finally {
+ if (cluster != null) {
+ cluster.shutdown();
+ }
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/96b3a6b9/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml
index c68c6d6..1baf355 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testErasureCodingConf.xml
@@ -101,7 +101,7 @@
</comparator>
<comparator>
<type>SubstringComparator</type>
- <expected-output>[-setPolicy -path <path> -policy <policy>]</expected-output>
+ <expected-output>[-setPolicy -path <path> [-policy <policy>] [-replicate]]</expected-output>
</comparator>
</comparators>
</test>
@@ -238,6 +238,29 @@
</test>
<test>
+ <description>setPolicy : set replication policy on a directory</description>
+ <test-commands>
+ <command>-fs NAMENODE -mkdir /ecdir</command>
+ <ec-admin-command>-fs NAMENODE -setPolicy -policy RS-6-3-64k -path /ecdir</ec-admin-command>
+ <command>-fs NAMENODE -mkdir /ecdir/replica</command>
+ <ec-admin-command>-fs NAMENODE -setPolicy -replicate -path /ecdir/replica</ec-admin-command>
+ <command>-fs NAMENODE -touchz /ecdir/replica/file</command>
+ <ec-admin-command>-fs NAMENODE -getPolicy -path /ecdir/replica/file</ec-admin-command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm /ecdir/replica/file</command>
+ <command>-fs NAMENODE -rmdir /ecdir/replica</command>
+ <command>-fs NAMENODE -rmdir /ecdir</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>is unspecified</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+
+ <test>
<description>unsetPolicy : unset policy and get</description>
<test-commands>
<command>-fs NAMENODE -mkdir /ecdir</command>
@@ -453,7 +476,7 @@
<!-- Test illegal parameters -->
<test>
- <description>setPolicy : illegal parameters - path is missing</description>
+ <description>setPolicy : illegal parameters - path option is missing</description>
<test-commands>
<command>-fs NAMENODE -mkdir /ecdir</command>
<ec-admin-command>-fs NAMENODE -setPolicy</ec-admin-command>
@@ -470,7 +493,7 @@
</test>
<test>
- <description>setPolicy : illegal parameters - policy name is missing</description>
+ <description>setPolicy : illegal parameters - path name is missing</description>
<test-commands>
<command>-fs NAMENODE -mkdir /ecdir</command>
<ec-admin-command>-fs NAMENODE -setPolicy -path</ec-admin-command>
@@ -487,7 +510,7 @@
</test>
<test>
- <description>setPolicy : illegal parameters - too many arguments</description>
+ <description>setPolicy : illegal parameters - too many arguments case 1</description>
<test-commands>
<command>-fs NAMENODE -mkdir /ecdir</command>
<ec-admin-command>-fs NAMENODE -setPolicy -path /ecdir1 -policy RS-3-2-64k /ecdir2</ec-admin-command>
@@ -504,6 +527,23 @@
</test>
<test>
+ <description>setPolicy : illegal parameters - too many arguments case 2</description>
+ <test-commands>
+ <command>-fs NAMENODE -mkdir /ecdir</command>
+ <ec-admin-command>-fs NAMENODE -setPolicy -path /ecdir1 -policy RS-3-2-64k -replicate /ecdir2</ec-admin-command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rmdir /ecdir</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>-setPolicy: Too many arguments</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+
+ <test>
<description>setPolicy : illegal parameters - invalidpolicy</description>
<test-commands>
<command>-fs NAMENODE -mkdir /ecdir</command>
@@ -553,6 +593,36 @@
</test>
<test>
+ <description>setPolicy : illegal parameters - wrong spelling replicate </description>
+ <test-commands>
+ <ec-admin-command>-fs NAMENODE -setPolicy -path /ecdir -replica</ec-admin-command>
+ </test-commands>
+ <cleanup-commands>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>-setPolicy: Too many arguments</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+
+ <test>
+ <description>setPolicy : illegal parameters - replicate and policy coexist</description>
+ <test-commands>
+ <ec-admin-command>-fs NAMENODE -setPolicy -path /ecdir -policy RS-3-2-64k -replicate</ec-admin-command>
+ </test-commands>
+ <cleanup-commands>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>-replicate and -policy cannot been used at the same time</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+
+ <test>
<description>setPolicy : set erasure coding policy without given a specific policy name</description>
<test-commands>
<command>-fs NAMENODE -mkdir /ecdir</command>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[24/50] [abbrv] hadoop git commit: YARN-6979. Add flag to notify all
types of container updates to NM via NodeHeartbeatResponse. (Kartheek
Muthyala via asuresh)
Posted by as...@apache.org.
YARN-6979. Add flag to notify all types of container updates to NM via NodeHeartbeatResponse. (Kartheek Muthyala via asuresh)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8410d862
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8410d862
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8410d862
Branch: refs/heads/YARN-5972
Commit: 8410d862d3a72740f461ef91dddb5325955e1ca5
Parents: 436c263
Author: Arun Suresh <as...@apache.org>
Authored: Sun Aug 20 07:54:09 2017 -0700
Committer: Arun Suresh <as...@apache.org>
Committed: Sun Aug 20 07:54:09 2017 -0700
----------------------------------------------------------------------
.../hadoop/yarn/sls/nodemanager/NodeInfo.java | 2 +-
.../yarn/sls/scheduler/RMNodeWrapper.java | 2 +-
.../hadoop/yarn/conf/YarnConfiguration.java | 4 +
.../src/main/resources/yarn-default.xml | 8 +
.../protocolrecords/NodeHeartbeatResponse.java | 6 +-
.../impl/pb/NodeHeartbeatResponsePBImpl.java | 42 ++---
.../yarn_server_common_service_protos.proto | 3 +
.../hadoop/yarn/TestYarnServerApiClasses.java | 6 +-
.../nodemanager/NodeStatusUpdaterImpl.java | 2 +-
.../containermanager/ContainerManagerImpl.java | 35 ++--
.../scheduler/ContainerScheduler.java | 1 +
.../resourcemanager/ResourceTrackerService.java | 2 +-
.../rmcontainer/RMContainerImpl.java | 8 +-
.../server/resourcemanager/rmnode/RMNode.java | 6 +-
.../rmnode/RMNodeDecreaseContainerEvent.java | 39 -----
.../resourcemanager/rmnode/RMNodeEventType.java | 2 +-
.../resourcemanager/rmnode/RMNodeImpl.java | 29 ++--
.../rmnode/RMNodeUpdateContainerEvent.java | 44 +++++
.../scheduler/AbstractYarnScheduler.java | 11 ++
.../scheduler/SchedulerApplicationAttempt.java | 39 +++--
.../yarn/server/resourcemanager/MockNodes.java | 2 +-
...pportunisticContainerAllocatorAMService.java | 168 +++++++++++++++++++
.../capacity/TestContainerResizing.java | 7 +-
.../capacity/TestIncreaseAllocationExpirer.java | 4 +-
24 files changed, 346 insertions(+), 126 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8410d862/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
index 8962aba..e71ddff 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/nodemanager/NodeInfo.java
@@ -179,7 +179,7 @@ public class NodeInfo {
}
@Override
- public void updateNodeHeartbeatResponseForContainersDecreasing(
+ public void updateNodeHeartbeatResponseForUpdatedContainers(
NodeHeartbeatResponse response) {
// TODO Auto-generated method stub
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8410d862/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
index d7b159c..6b7ac3c 100644
--- a/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
+++ b/hadoop-tools/hadoop-sls/src/main/java/org/apache/hadoop/yarn/sls/scheduler/RMNodeWrapper.java
@@ -168,7 +168,7 @@ public class RMNodeWrapper implements RMNode {
}
@Override
- public void updateNodeHeartbeatResponseForContainersDecreasing(
+ public void updateNodeHeartbeatResponseForUpdatedContainers(
NodeHeartbeatResponse response) {
// TODO Auto-generated method stub
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8410d862/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 8515e0a..86f45b8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -167,6 +167,10 @@ public class YarnConfiguration extends Configuration {
public static final String RM_APPLICATION_MASTER_SERVICE_PROCESSORS =
RM_PREFIX + "application-master-service.processors";
+ public static final String RM_AUTO_UPDATE_CONTAINERS =
+ RM_PREFIX + "auto-update.containers";
+ public static final boolean DEFAULT_RM_AUTO_UPDATE_CONTAINERS = false;
+
/** The actual bind address for the RM.*/
public static final String RM_BIND_HOST =
RM_PREFIX + "bind-host";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8410d862/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
index dbf115b..f93de44 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
@@ -73,6 +73,14 @@
</property>
<property>
+ <description>
+ If set to true, then ALL container updates will be automatically sent to
+ the NM in the next heartbeat</description>
+ <name>yarn.resourcemanager.auto-update.containers</name>
+ <value>false</value>
+ </property>
+
+ <property>
<description>The number of threads used to handle applications manager requests.</description>
<name>yarn.resourcemanager.client.thread-count</name>
<value>50</value>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8410d862/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatResponse.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatResponse.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatResponse.java
index 7568bbb..3b0ec10 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatResponse.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/NodeHeartbeatResponse.java
@@ -104,10 +104,10 @@ public abstract class NodeHeartbeatResponse {
public abstract void setResource(Resource resource);
- public abstract List<Container> getContainersToDecrease();
+ public abstract List<Container> getContainersToUpdate();
- public abstract void addAllContainersToDecrease(
- Collection<Container> containersToDecrease);
+ public abstract void addAllContainersToUpdate(
+ Collection<Container> containersToUpdate);
public abstract ContainerQueuingLimit getContainerQueuingLimit();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8410d862/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatResponsePBImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatResponsePBImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatResponsePBImpl.java
index 51c1a78..46c2b0b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatResponsePBImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/NodeHeartbeatResponsePBImpl.java
@@ -75,7 +75,7 @@ public class NodeHeartbeatResponsePBImpl extends NodeHeartbeatResponse {
private MasterKey containerTokenMasterKey = null;
private MasterKey nmTokenMasterKey = null;
private ContainerQueuingLimit containerQueuingLimit = null;
- private List<Container> containersToDecrease = null;
+ private List<Container> containersToUpdate = null;
private List<SignalContainerRequest> containersToSignal = null;
public NodeHeartbeatResponsePBImpl() {
@@ -119,8 +119,8 @@ public class NodeHeartbeatResponsePBImpl extends NodeHeartbeatResponse {
if (this.systemCredentials != null) {
addSystemCredentialsToProto();
}
- if (this.containersToDecrease != null) {
- addContainersToDecreaseToProto();
+ if (this.containersToUpdate != null) {
+ addContainersToUpdateToProto();
}
if (this.containersToSignal != null) {
addContainersToSignalToProto();
@@ -499,39 +499,39 @@ public class NodeHeartbeatResponsePBImpl extends NodeHeartbeatResponse {
builder.addAllApplicationsToCleanup(iterable);
}
- private void initContainersToDecrease() {
- if (this.containersToDecrease != null) {
+ private void initContainersToUpdate() {
+ if (this.containersToUpdate != null) {
return;
}
NodeHeartbeatResponseProtoOrBuilder p = viaProto ? proto : builder;
- List<ContainerProto> list = p.getContainersToDecreaseList();
- this.containersToDecrease = new ArrayList<>();
+ List<ContainerProto> list = p.getContainersToUpdateList();
+ this.containersToUpdate = new ArrayList<>();
for (ContainerProto c : list) {
- this.containersToDecrease.add(convertFromProtoFormat(c));
+ this.containersToUpdate.add(convertFromProtoFormat(c));
}
}
@Override
- public List<Container> getContainersToDecrease() {
- initContainersToDecrease();
- return this.containersToDecrease;
+ public List<Container> getContainersToUpdate() {
+ initContainersToUpdate();
+ return this.containersToUpdate;
}
@Override
- public void addAllContainersToDecrease(
- final Collection<Container> containersToDecrease) {
- if (containersToDecrease == null) {
+ public void addAllContainersToUpdate(
+ final Collection<Container> containersToBeUpdated) {
+ if (containersToBeUpdated == null) {
return;
}
- initContainersToDecrease();
- this.containersToDecrease.addAll(containersToDecrease);
+ initContainersToUpdate();
+ this.containersToUpdate.addAll(containersToBeUpdated);
}
- private void addContainersToDecreaseToProto() {
+ private void addContainersToUpdateToProto() {
maybeInitBuilder();
- builder.clearContainersToDecrease();
- if (this.containersToDecrease == null) {
+ builder.clearContainersToUpdate();
+ if (this.containersToUpdate == null) {
return;
}
Iterable<ContainerProto> iterable = new
@@ -539,7 +539,7 @@ public class NodeHeartbeatResponsePBImpl extends NodeHeartbeatResponse {
@Override
public Iterator<ContainerProto> iterator() {
return new Iterator<ContainerProto>() {
- private Iterator<Container> iter = containersToDecrease.iterator();
+ private Iterator<Container> iter = containersToUpdate.iterator();
@Override
public boolean hasNext() {
return iter.hasNext();
@@ -555,7 +555,7 @@ public class NodeHeartbeatResponsePBImpl extends NodeHeartbeatResponse {
};
}
};
- builder.addAllContainersToDecrease(iterable);
+ builder.addAllContainersToUpdate(iterable);
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8410d862/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto
index edb2d9c..4e05fba 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_service_protos.proto
@@ -111,11 +111,14 @@ message NodeHeartbeatResponseProto {
repeated ContainerIdProto containers_to_be_removed_from_nm = 9;
repeated SystemCredentialsForAppsProto system_credentials_for_apps = 10;
optional bool areNodeLabelsAcceptedByRM = 11 [default = false];
+ // to be deprecated in favour of containers_to_update
repeated ContainerProto containers_to_decrease = 12;
repeated SignalContainerRequestProto containers_to_signal = 13;
optional ResourceProto resource = 14;
optional ContainerQueuingLimitProto container_queuing_limit = 15;
repeated AppCollectorsMapProto app_collectors_map = 16;
+ // to be used in place of containers_to_decrease
+ repeated ContainerProto containers_to_update = 17;
}
message ContainerQueuingLimitProto {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8410d862/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestYarnServerApiClasses.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestYarnServerApiClasses.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestYarnServerApiClasses.java
index b670c36..8c0c73a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestYarnServerApiClasses.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/TestYarnServerApiClasses.java
@@ -180,14 +180,14 @@ public class TestYarnServerApiClasses {
@Test
public void testNodeHeartbeatResponsePBImplWithDecreasedContainers() {
NodeHeartbeatResponsePBImpl original = new NodeHeartbeatResponsePBImpl();
- original.addAllContainersToDecrease(
+ original.addAllContainersToUpdate(
Arrays.asList(getDecreasedContainer(1, 2, 2048, 2),
getDecreasedContainer(2, 3, 1024, 1)));
NodeHeartbeatResponsePBImpl copy =
new NodeHeartbeatResponsePBImpl(original.getProto());
- assertEquals(1, copy.getContainersToDecrease().get(0)
+ assertEquals(1, copy.getContainersToUpdate().get(0)
.getId().getContainerId());
- assertEquals(1024, copy.getContainersToDecrease().get(1)
+ assertEquals(1024, copy.getContainersToUpdate().get(1)
.getResource().getMemorySize());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8410d862/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
index b5ec383..1d9256f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
@@ -1099,7 +1099,7 @@ public class NodeStatusUpdaterImpl extends AbstractService implements
parseCredentials(systemCredentials));
}
List<org.apache.hadoop.yarn.api.records.Container>
- containersToDecrease = response.getContainersToDecrease();
+ containersToDecrease = response.getContainersToUpdate();
if (!containersToDecrease.isEmpty()) {
dispatcher.getEventHandler().handle(
new CMgrDecreaseContainersResourceEvent(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8410d862/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
index a1e8ca0..12931bc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
@@ -166,6 +166,7 @@ import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
+import java.util.EnumSet;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -1241,10 +1242,19 @@ public class ContainerManagerImpl extends CompositeService implements
org.apache.hadoop.yarn.server.nodemanager.
containermanager.container.ContainerState currentState =
container.getContainerState();
- if (currentState != org.apache.hadoop.yarn.server.
- nodemanager.containermanager.container.ContainerState.RUNNING &&
- currentState != org.apache.hadoop.yarn.server.
- nodemanager.containermanager.container.ContainerState.SCHEDULED) {
+ EnumSet<org.apache.hadoop.yarn.server.nodemanager.containermanager
+ .container.ContainerState> allowedStates = EnumSet.of(
+ org.apache.hadoop.yarn.server.nodemanager.containermanager.container
+ .ContainerState.RUNNING,
+ org.apache.hadoop.yarn.server.nodemanager.containermanager.container
+ .ContainerState.SCHEDULED,
+ org.apache.hadoop.yarn.server.nodemanager.containermanager.container
+ .ContainerState.LOCALIZING,
+ org.apache.hadoop.yarn.server.nodemanager.containermanager.container
+ .ContainerState.REINITIALIZING,
+ org.apache.hadoop.yarn.server.nodemanager.containermanager.container
+ .ContainerState.RELAUNCHING);
+ if (!allowedStates.contains(currentState)) {
throw RPCUtil.getRemoteException("Container " + containerId.toString()
+ " is in " + currentState.name() + " state."
+ " Resource can only be changed when a container is in"
@@ -1279,17 +1289,12 @@ public class ContainerManagerImpl extends CompositeService implements
org.apache.hadoop.yarn.api.records.Container.newInstance(
containerId, null, null, targetResource, null, null,
currentExecType);
- } else {
- increasedContainer =
- org.apache.hadoop.yarn.api.records.Container.newInstance(
- containerId, null, null, currentResource, null, null,
- targetExecType);
- }
- if (context.getIncreasedContainers().putIfAbsent(containerId,
- increasedContainer) != null){
- throw RPCUtil.getRemoteException("Container " + containerId.toString()
- + " resource is being increased -or- " +
- "is undergoing ExecutionType promoted.");
+ if (context.getIncreasedContainers().putIfAbsent(containerId,
+ increasedContainer) != null){
+ throw RPCUtil.getRemoteException("Container " + containerId.toString()
+ + " resource is being increased -or- " +
+ "is undergoing ExecutionType promoted.");
+ }
}
}
this.readLock.lock();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8410d862/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java
index 19b4505..644bdae 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerScheduler.java
@@ -173,6 +173,7 @@ public class ContainerScheduler extends AbstractService implements
new ChangeMonitoringContainerResourceEvent(containerId,
updateEvent.getUpdatedToken().getResource()));
} else {
+ // Is Queued or localizing..
updateEvent.getContainer().setContainerTokenIdentifier(
updateEvent.getUpdatedToken());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8410d862/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
index aa7f524..e6f2bb2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
@@ -551,7 +551,7 @@ public class ResourceTrackerService extends AbstractService implements
getResponseId() + 1, NodeAction.NORMAL, null, null, null, null,
nextHeartBeatInterval);
rmNode.updateNodeHeartbeatResponseForCleanup(nodeHeartBeatResponse);
- rmNode.updateNodeHeartbeatResponseForContainersDecreasing(
+ rmNode.updateNodeHeartbeatResponseForUpdatedContainers(
nodeHeartBeatResponse);
populateKeys(request, nodeHeartBeatResponse);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8410d862/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
index 1e9463a..f49db7e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmcontainer/RMContainerImpl.java
@@ -51,8 +51,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptE
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptEventType;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.event.RMAppAttemptContainerFinishedEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeCleanContainerEvent;
-import org.apache.hadoop.yarn.server.resourcemanager.rmnode
- .RMNodeDecreaseContainerEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeUpdateContainerEvent;
import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey;
import org.apache.hadoop.yarn.state.InvalidStateTransitionException;
import org.apache.hadoop.yarn.state.MultipleArcTransition;
@@ -284,7 +283,6 @@ public class RMContainerImpl implements RMContainer {
@Override
public RMContainerState getState() {
this.readLock.lock();
-
try {
return this.stateMachine.getCurrentState();
} finally {
@@ -598,7 +596,7 @@ public class RMContainerImpl implements RMContainer {
RMContainerUpdatesAcquiredEvent acquiredEvent =
(RMContainerUpdatesAcquiredEvent) event;
if (acquiredEvent.isIncreasedContainer()) {
- // If container is increased but not acquired by AM, we will start
+ // If container is increased but not started by AM, we will start
// containerAllocationExpirer for this container in this transition.
container.containerAllocationExpirer.register(
new AllocationExpirationInfo(event.getContainerId(), true));
@@ -641,7 +639,7 @@ public class RMContainerImpl implements RMContainer {
container.lastConfirmedResource = rmContainerResource;
container.containerAllocationExpirer.unregister(
new AllocationExpirationInfo(event.getContainerId()));
- container.eventHandler.handle(new RMNodeDecreaseContainerEvent(
+ container.eventHandler.handle(new RMNodeUpdateContainerEvent(
container.nodeId,
Collections.singletonList(container.getContainer())));
} else if (Resources.fitsIn(nmContainerResource, rmContainerResource)) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8410d862/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java
index 86f8679..ab15c95 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java
@@ -144,7 +144,7 @@ public interface RMNode {
* applications to clean up for this node.
* @param response the {@link NodeHeartbeatResponse} to update
*/
- public void updateNodeHeartbeatResponseForCleanup(NodeHeartbeatResponse response);
+ void updateNodeHeartbeatResponseForCleanup(NodeHeartbeatResponse response);
public NodeHeartbeatResponse getLastNodeHeartBeatResponse();
@@ -169,9 +169,9 @@ public interface RMNode {
public Set<String> getNodeLabels();
/**
- * Update containers to be decreased
+ * Update containers to be updated
*/
- public void updateNodeHeartbeatResponseForContainersDecreasing(
+ void updateNodeHeartbeatResponseForUpdatedContainers(
NodeHeartbeatResponse response);
public List<Container> pullNewlyIncreasedContainers();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8410d862/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeDecreaseContainerEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeDecreaseContainerEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeDecreaseContainerEvent.java
deleted file mode 100644
index 62925ad..0000000
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeDecreaseContainerEvent.java
+++ /dev/null
@@ -1,39 +0,0 @@
-/**
-* Licensed to the Apache Software Foundation (ASF) under one
-* or more contributor license agreements. See the NOTICE file
-* distributed with this work for additional information
-* regarding copyright ownership. The ASF licenses this file
-* to you under the Apache License, Version 2.0 (the
-* "License"); you may not use this file except in compliance
-* with the License. You may obtain a copy of the License at
-*
-* http://www.apache.org/licenses/LICENSE-2.0
-*
-* Unless required by applicable law or agreed to in writing, software
-* distributed under the License is distributed on an "AS IS" BASIS,
-* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-* See the License for the specific language governing permissions and
-* limitations under the License.
-*/
-
-package org.apache.hadoop.yarn.server.resourcemanager.rmnode;
-
-import java.util.List;
-
-import org.apache.hadoop.yarn.api.records.Container;
-import org.apache.hadoop.yarn.api.records.NodeId;
-
-public class RMNodeDecreaseContainerEvent extends RMNodeEvent {
- final List<Container> toBeDecreasedContainers;
-
- public RMNodeDecreaseContainerEvent(NodeId nodeId,
- List<Container> toBeDecreasedContainers) {
- super(nodeId, RMNodeEventType.DECREASE_CONTAINER);
-
- this.toBeDecreasedContainers = toBeDecreasedContainers;
- }
-
- public List<Container> getToBeDecreasedContainers() {
- return toBeDecreasedContainers;
- }
-}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8410d862/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeEventType.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeEventType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeEventType.java
index b28fef3..a3b2ed7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeEventType.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeEventType.java
@@ -42,7 +42,7 @@ public enum RMNodeEventType {
// Source: Container
CONTAINER_ALLOCATED,
CLEANUP_CONTAINER,
- DECREASE_CONTAINER,
+ UPDATE_CONTAINER,
// Source: ClientRMService
SIGNAL_CONTAINER,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8410d862/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
index 1f121f8..1bdaa98 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
@@ -171,7 +171,7 @@ public class RMNodeImpl implements RMNode, EventHandler<RMNodeEvent> {
private final List<ApplicationId> runningApplications =
new ArrayList<ApplicationId>();
- private final Map<ContainerId, Container> toBeDecreasedContainers =
+ private final Map<ContainerId, Container> toBeUpdatedContainers =
new HashMap<>();
private final Map<ContainerId, Container> nmReportedIncreasedContainers =
@@ -228,8 +228,8 @@ public class RMNodeImpl implements RMNode, EventHandler<RMNodeEvent> {
.addTransition(NodeState.RUNNING, NodeState.RUNNING,
RMNodeEventType.RESOURCE_UPDATE, new UpdateNodeResourceWhenRunningTransition())
.addTransition(NodeState.RUNNING, NodeState.RUNNING,
- RMNodeEventType.DECREASE_CONTAINER,
- new DecreaseContainersTransition())
+ RMNodeEventType.UPDATE_CONTAINER,
+ new UpdateContainersTransition())
.addTransition(NodeState.RUNNING, NodeState.RUNNING,
RMNodeEventType.SIGNAL_CONTAINER, new SignalContainerTransition())
.addTransition(NodeState.RUNNING, NodeState.SHUTDOWN,
@@ -614,18 +614,18 @@ public class RMNodeImpl implements RMNode, EventHandler<RMNodeEvent> {
};
@VisibleForTesting
- public Collection<Container> getToBeDecreasedContainers() {
- return toBeDecreasedContainers.values();
+ public Collection<Container> getToBeUpdatedContainers() {
+ return toBeUpdatedContainers.values();
}
@Override
- public void updateNodeHeartbeatResponseForContainersDecreasing(
+ public void updateNodeHeartbeatResponseForUpdatedContainers(
NodeHeartbeatResponse response) {
this.writeLock.lock();
try {
- response.addAllContainersToDecrease(toBeDecreasedContainers.values());
- toBeDecreasedContainers.clear();
+ response.addAllContainersToUpdate(toBeUpdatedContainers.values());
+ toBeUpdatedContainers.clear();
} finally {
this.writeLock.unlock();
}
@@ -1031,16 +1031,19 @@ public class RMNodeImpl implements RMNode, EventHandler<RMNodeEvent> {
RMNodeFinishedContainersPulledByAMEvent) event).getContainers());
}
}
-
- public static class DecreaseContainersTransition
+
+ /**
+ * Transition to Update a container.
+ */
+ public static class UpdateContainersTransition
implements SingleArcTransition<RMNodeImpl, RMNodeEvent> {
@Override
public void transition(RMNodeImpl rmNode, RMNodeEvent event) {
- RMNodeDecreaseContainerEvent de = (RMNodeDecreaseContainerEvent) event;
+ RMNodeUpdateContainerEvent de = (RMNodeUpdateContainerEvent) event;
- for (Container c : de.getToBeDecreasedContainers()) {
- rmNode.toBeDecreasedContainers.put(c.getId(), c);
+ for (Container c : de.getToBeUpdatedContainers()) {
+ rmNode.toBeUpdatedContainers.put(c.getId(), c);
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8410d862/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeUpdateContainerEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeUpdateContainerEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeUpdateContainerEvent.java
new file mode 100644
index 0000000..73af563
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeUpdateContainerEvent.java
@@ -0,0 +1,44 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.yarn.server.resourcemanager.rmnode;
+
+import java.util.List;
+
+import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.NodeId;
+
+/**
+ * This class is used to create update container event
+ * for the containers running on a node.
+ *
+ */
+public class RMNodeUpdateContainerEvent extends RMNodeEvent {
+ private List<Container> toBeUpdatedContainers;
+
+ public RMNodeUpdateContainerEvent(NodeId nodeId,
+ List<Container> toBeUpdatedContainers) {
+ super(nodeId, RMNodeEventType.UPDATE_CONTAINER);
+
+ this.toBeUpdatedContainers = toBeUpdatedContainers;
+ }
+
+ public List<Container> getToBeUpdatedContainers() {
+ return toBeUpdatedContainers;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8410d862/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
index 79caab0..c3879dd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
@@ -150,6 +150,10 @@ public abstract class AbstractYarnScheduler
*/
protected final ReentrantReadWriteLock.WriteLock writeLock;
+ // If set to true, then ALL container updates will be automatically sent to
+ // the NM in the next heartbeat.
+ private boolean autoUpdateContainers = false;
+
/**
* Construct the service.
*
@@ -178,6 +182,9 @@ public abstract class AbstractYarnScheduler
configuredMaximumAllocationWaitTime);
maxClusterLevelAppPriority = getMaxPriorityFromConf(conf);
createReleaseCache();
+ autoUpdateContainers =
+ conf.getBoolean(YarnConfiguration.RM_AUTO_UPDATE_CONTAINERS,
+ YarnConfiguration.DEFAULT_RM_AUTO_UPDATE_CONTAINERS);
super.serviceInit(conf);
}
@@ -235,6 +242,10 @@ public abstract class AbstractYarnScheduler
return nodeTracker.getNodes(nodeFilter);
}
+ public boolean shouldContainersBeAutoUpdated() {
+ return this.autoUpdateContainers;
+ }
+
@Override
public Resource getClusterResource() {
return nodeTracker.getClusterCapacity();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8410d862/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
index 397d507..cc14a1e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
@@ -74,8 +74,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerStat
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerUpdatesAcquiredEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeCleanContainerEvent;
-import org.apache.hadoop.yarn.server.resourcemanager.rmnode
- .RMNodeDecreaseContainerEvent;
+import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeUpdateContainerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.SchedulingMode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.SchedulingPlacementSet;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.SchedulableEntity;
@@ -663,20 +662,38 @@ public class SchedulerApplicationAttempt implements SchedulableEntity {
+ " an updated container " + container.getId(), e);
return null;
}
-
- if (updateType == null ||
- ContainerUpdateType.PROMOTE_EXECUTION_TYPE == updateType ||
- ContainerUpdateType.DEMOTE_EXECUTION_TYPE == updateType) {
+
+ if (updateType == null) {
+ // This is a newly allocated container
rmContainer.handle(new RMContainerEvent(
rmContainer.getContainerId(), RMContainerEventType.ACQUIRED));
} else {
- rmContainer.handle(new RMContainerUpdatesAcquiredEvent(
- rmContainer.getContainerId(),
- ContainerUpdateType.INCREASE_RESOURCE == updateType));
- if (ContainerUpdateType.DECREASE_RESOURCE == updateType) {
+ // Resource increase is handled as follows:
+ // If the AM does not use the updated token to increase the container
+ // for a configured period of time, the RM will automatically rollback
+ // the update by performing a container decrease. This rollback (which
+ // essentially is another resource decrease update) is notified to the
+ // NM heartbeat response. If autoUpdate flag is set, then AM does not
+ // need to do anything - same code path as resource decrease.
+ //
+ // Resource Decrease is always automatic: the AM never has to do
+ // anything. It is always via NM heartbeat response.
+ //
+ // ExecutionType updates (both Promotion and Demotion) are either
+ // always automatic (if the flag is set) or the AM has to explicitly
+ // call updateContainer() on the NM. There is no expiry
+ boolean autoUpdate =
+ ContainerUpdateType.DECREASE_RESOURCE == updateType ||
+ ((AbstractYarnScheduler)rmContext.getScheduler())
+ .shouldContainersBeAutoUpdated();
+ if (autoUpdate) {
this.rmContext.getDispatcher().getEventHandler().handle(
- new RMNodeDecreaseContainerEvent(rmContainer.getNodeId(),
+ new RMNodeUpdateContainerEvent(rmContainer.getNodeId(),
Collections.singletonList(rmContainer.getContainer())));
+ } else {
+ rmContainer.handle(new RMContainerUpdatesAcquiredEvent(
+ rmContainer.getContainerId(),
+ ContainerUpdateType.INCREASE_RESOURCE == updateType));
}
}
return container;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8410d862/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
index 91170d1..7f58711 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
@@ -242,7 +242,7 @@ public class MockNodes {
}
@Override
- public void updateNodeHeartbeatResponseForContainersDecreasing(
+ public void updateNodeHeartbeatResponseForUpdatedContainers(
NodeHeartbeatResponse response) {
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8410d862/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestOpportunisticContainerAllocatorAMService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestOpportunisticContainerAllocatorAMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestOpportunisticContainerAllocatorAMService.java
index 6819395..b885118 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestOpportunisticContainerAllocatorAMService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestOpportunisticContainerAllocatorAMService.java
@@ -43,11 +43,13 @@ import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.api.records.UpdateContainerRequest;
+import org.apache.hadoop.yarn.api.records.UpdatedContainer;
import org.apache.hadoop.yarn.server.api.DistributedSchedulingAMProtocolPB;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
import org.apache.hadoop.yarn.server.api.protocolrecords.DistributedSchedulingAllocateRequest;
import org.apache.hadoop.yarn.server.api.protocolrecords.DistributedSchedulingAllocateResponse;
+import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatResponse;
import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterDistributedSchedulingAMResponse;
import org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterRequest;
import org.apache.hadoop.yarn.api.protocolrecords.FinishApplicationMasterResponse;
@@ -122,6 +124,21 @@ public class TestOpportunisticContainerAllocatorAMService {
rm.start();
}
+ public void createAndStartRMWithAutoUpdateContainer() {
+ CapacitySchedulerConfiguration csConf =
+ new CapacitySchedulerConfiguration();
+ YarnConfiguration conf = new YarnConfiguration(csConf);
+ conf.setBoolean(YarnConfiguration.RM_AUTO_UPDATE_CONTAINERS, true);
+ conf.setClass(YarnConfiguration.RM_SCHEDULER, CapacityScheduler.class,
+ ResourceScheduler.class);
+ conf.setBoolean(
+ YarnConfiguration.OPPORTUNISTIC_CONTAINER_ALLOCATION_ENABLED, true);
+ conf.setInt(
+ YarnConfiguration.NM_CONTAINER_QUEUING_SORTING_NODES_INTERVAL_MS, 100);
+ rm = new MockRM(conf);
+ rm.start();
+ }
+
@After
public void stopRM() {
if (rm != null) {
@@ -548,6 +565,157 @@ public class TestOpportunisticContainerAllocatorAMService {
verifyMetrics(metrics, 7168, 7, 1024, 1, 1);
}
+ @Test(timeout = 600000)
+ public void testContainerAutoUpdateContainer() throws Exception {
+ rm.stop();
+ createAndStartRMWithAutoUpdateContainer();
+ MockNM nm1 = new MockNM("h1:1234", 4096, rm.getResourceTrackerService());
+ nm1.registerNode();
+
+ OpportunisticContainerAllocatorAMService amservice =
+ (OpportunisticContainerAllocatorAMService) rm
+ .getApplicationMasterService();
+ RMApp app1 = rm.submitApp(1 * GB, "app", "user", null, "default");
+ ApplicationAttemptId attemptId =
+ app1.getCurrentAppAttempt().getAppAttemptId();
+ MockAM am1 = MockRM.launchAndRegisterAM(app1, rm, nm1);
+ ResourceScheduler scheduler = rm.getResourceScheduler();
+ RMNode rmNode1 = rm.getRMContext().getRMNodes().get(nm1.getNodeId());
+
+ nm1.nodeHeartbeat(true);
+
+ ((RMNodeImpl) rmNode1)
+ .setOpportunisticContainersStatus(getOppurtunisticStatus(-1, 100));
+
+ OpportunisticContainerContext ctxt =
+ ((CapacityScheduler) scheduler).getApplicationAttempt(attemptId)
+ .getOpportunisticContainerContext();
+ // Send add and update node events to AM Service.
+ amservice.handle(new NodeAddedSchedulerEvent(rmNode1));
+ amservice.handle(new NodeUpdateSchedulerEvent(rmNode1));
+
+ nm1.nodeHeartbeat(true);
+ Thread.sleep(1000);
+
+ AllocateResponse allocateResponse = am1.allocate(Arrays.asList(
+ ResourceRequest.newInstance(Priority.newInstance(1), "*",
+ Resources.createResource(1 * GB), 2, true, null,
+ ExecutionTypeRequest
+ .newInstance(ExecutionType.OPPORTUNISTIC, true))), null);
+ List<Container> allocatedContainers =
+ allocateResponse.getAllocatedContainers();
+ Assert.assertEquals(2, allocatedContainers.size());
+ Container container = allocatedContainers.get(0);
+ // Start Container in NM
+ nm1.nodeHeartbeat(Arrays.asList(ContainerStatus
+ .newInstance(container.getId(), ExecutionType.OPPORTUNISTIC,
+ ContainerState.RUNNING, "", 0)), true);
+ Thread.sleep(200);
+
+ // Verify that container is actually running wrt the RM..
+ RMContainer rmContainer = ((CapacityScheduler) scheduler)
+ .getApplicationAttempt(container.getId().getApplicationAttemptId())
+ .getRMContainer(container.getId());
+ Assert.assertEquals(RMContainerState.RUNNING, rmContainer.getState());
+
+ // Send Promotion req... this should result in update error
+ // Since the container doesn't exist anymore..
+ allocateResponse = am1.sendContainerUpdateRequest(Arrays.asList(
+ UpdateContainerRequest.newInstance(0, container.getId(),
+ ContainerUpdateType.PROMOTE_EXECUTION_TYPE, null,
+ ExecutionType.GUARANTEED)));
+
+ nm1.nodeHeartbeat(Arrays.asList(ContainerStatus
+ .newInstance(container.getId(), ExecutionType.OPPORTUNISTIC,
+ ContainerState.RUNNING, "", 0)), true);
+ Thread.sleep(200);
+ // Get the update response on next allocate
+ allocateResponse = am1.allocate(new ArrayList<>(), new ArrayList<>());
+ // Check the update response from YARNRM
+ Assert.assertEquals(1, allocateResponse.getUpdatedContainers().size());
+ UpdatedContainer uc = allocateResponse.getUpdatedContainers().get(0);
+ Assert.assertEquals(container.getId(), uc.getContainer().getId());
+ Assert.assertEquals(ExecutionType.GUARANTEED,
+ uc.getContainer().getExecutionType());
+ // Check that the container is updated in NM through NM heartbeat response
+ NodeHeartbeatResponse response = nm1.nodeHeartbeat(true);
+ Assert.assertEquals(1, response.getContainersToUpdate().size());
+ Container containersFromNM = response.getContainersToUpdate().get(0);
+ Assert.assertEquals(container.getId(), containersFromNM.getId());
+ Assert.assertEquals(ExecutionType.GUARANTEED,
+ containersFromNM.getExecutionType());
+
+ //Increase resources
+ allocateResponse = am1.sendContainerUpdateRequest(Arrays.asList(
+ UpdateContainerRequest.newInstance(1, container.getId(),
+ ContainerUpdateType.INCREASE_RESOURCE,
+ Resources.createResource(2 * GB, 1), null)));
+ response = nm1.nodeHeartbeat(Arrays.asList(ContainerStatus
+ .newInstance(container.getId(), ExecutionType.GUARANTEED,
+ ContainerState.RUNNING, "", 0)), true);
+
+ Thread.sleep(200);
+ if (allocateResponse.getUpdatedContainers().size() == 0) {
+ allocateResponse = am1.allocate(new ArrayList<>(), new ArrayList<>());
+ }
+ Assert.assertEquals(1, allocateResponse.getUpdatedContainers().size());
+ uc = allocateResponse.getUpdatedContainers().get(0);
+ Assert.assertEquals(container.getId(), uc.getContainer().getId());
+ Assert.assertEquals(Resource.newInstance(2 * GB, 1),
+ uc.getContainer().getResource());
+
+ // Check that the container resources are increased in
+ // NM through NM heartbeat response
+ if (response.getContainersToUpdate().size() == 0) {
+ response = nm1.nodeHeartbeat(true);
+ }
+ Assert.assertEquals(1, response.getContainersToUpdate().size());
+ Assert.assertEquals(Resource.newInstance(2 * GB, 1),
+ response.getContainersToUpdate().get(0).getResource());
+
+ //Decrease resources
+ allocateResponse = am1.sendContainerUpdateRequest(Arrays.asList(
+ UpdateContainerRequest.newInstance(2, container.getId(),
+ ContainerUpdateType.DECREASE_RESOURCE,
+ Resources.createResource(1 * GB, 1), null)));
+ Assert.assertEquals(1, allocateResponse.getUpdatedContainers().size());
+
+ // Check that the container resources are decreased
+ // in NM through NM heartbeat response
+ response = nm1.nodeHeartbeat(true);
+ Assert.assertEquals(1, response.getContainersToUpdate().size());
+ Assert.assertEquals(Resource.newInstance(1 * GB, 1),
+ response.getContainersToUpdate().get(0).getResource());
+
+ nm1.nodeHeartbeat(true);
+ // DEMOTE the container
+ allocateResponse = am1.sendContainerUpdateRequest(Arrays.asList(
+ UpdateContainerRequest.newInstance(3, container.getId(),
+ ContainerUpdateType.DEMOTE_EXECUTION_TYPE, null,
+ ExecutionType.OPPORTUNISTIC)));
+
+ response = nm1.nodeHeartbeat(Arrays.asList(ContainerStatus
+ .newInstance(container.getId(), ExecutionType.GUARANTEED,
+ ContainerState.RUNNING, "", 0)), true);
+ Thread.sleep(200);
+ if (allocateResponse.getUpdatedContainers().size() == 0) {
+ // Get the update response on next allocate
+ allocateResponse = am1.allocate(new ArrayList<>(), new ArrayList<>());
+ }
+ // Check the update response from YARNRM
+ Assert.assertEquals(1, allocateResponse.getUpdatedContainers().size());
+ uc = allocateResponse.getUpdatedContainers().get(0);
+ Assert.assertEquals(ExecutionType.OPPORTUNISTIC,
+ uc.getContainer().getExecutionType());
+ // Check that the container is updated in NM through NM heartbeat response
+ if (response.getContainersToUpdate().size() == 0) {
+ response = nm1.nodeHeartbeat(true);
+ }
+ Assert.assertEquals(1, response.getContainersToUpdate().size());
+ Assert.assertEquals(ExecutionType.OPPORTUNISTIC,
+ response.getContainersToUpdate().get(0).getExecutionType());
+ }
+
private void verifyMetrics(QueueMetrics metrics, long availableMB,
int availableVirtualCores, long allocatedMB,
int allocatedVirtualCores, int allocatedContainers) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8410d862/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerResizing.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerResizing.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerResizing.java
index b4b05ed..291a74e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerResizing.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerResizing.java
@@ -51,8 +51,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainerStat
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNode;
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeImpl;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ResourceScheduler;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler
- .SchedulerApplicationAttempt;
+
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerNode;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica.FiCaSchedulerApp;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica
@@ -60,11 +59,9 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.fica
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.AppAttemptRemovedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.PlacementSet;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.SimplePlacementSet;
import org.apache.hadoop.yarn.util.resource.Resources;
import org.junit.Assert;
import org.junit.Before;
-import org.junit.Ignore;
import org.junit.Test;
public class TestContainerResizing {
@@ -205,7 +202,7 @@ public class TestContainerResizing {
RMNodeImpl rmNode =
(RMNodeImpl) rm1.getRMContext().getRMNodes().get(nm1.getNodeId());
Collection<Container> decreasedContainers =
- rmNode.getToBeDecreasedContainers();
+ rmNode.getToBeUpdatedContainers();
boolean rmNodeReceivedDecreaseContainer = false;
for (Container c : decreasedContainers) {
if (c.getId().equals(containerId1)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8410d862/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestIncreaseAllocationExpirer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestIncreaseAllocationExpirer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestIncreaseAllocationExpirer.java
index 184e854..a76ed64 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestIncreaseAllocationExpirer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestIncreaseAllocationExpirer.java
@@ -319,7 +319,7 @@ public class TestIncreaseAllocationExpirer {
verifyAvailableResourceOfSchedulerNode(rm1, nm1.getNodeId(), 16 * GB);
// Verify NM receives the decrease message (3G)
List<Container> containersToDecrease =
- nm1.nodeHeartbeat(true).getContainersToDecrease();
+ nm1.nodeHeartbeat(true).getContainersToUpdate();
Assert.assertEquals(1, containersToDecrease.size());
Assert.assertEquals(
3 * GB, containersToDecrease.get(0).getResource().getMemorySize());
@@ -435,7 +435,7 @@ public class TestIncreaseAllocationExpirer {
.getAllocatedResource().getMemorySize());
// Verify NM receives 2 decrease message
List<Container> containersToDecrease =
- nm1.nodeHeartbeat(true).getContainersToDecrease();
+ nm1.nodeHeartbeat(true).getContainersToUpdate();
Assert.assertEquals(2, containersToDecrease.size());
// Sort the list to make sure containerId3 is the first
Collections.sort(containersToDecrease);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[48/50] [abbrv] hadoop git commit: YARN-7090.
testRMRestartAfterNodeLabelDisabled get failed when CapacityScheduler is
configured. Contributed by Wangda Tan.
Posted by as...@apache.org.
YARN-7090. testRMRestartAfterNodeLabelDisabled get failed when CapacityScheduler is configured. Contributed by Wangda Tan.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/652dd434
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/652dd434
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/652dd434
Branch: refs/heads/YARN-5972
Commit: 652dd434d96cd1a1fe25cd8c636b1859c29f462b
Parents: 1000a2a
Author: Junping Du <ju...@apache.org>
Authored: Wed Aug 23 18:06:29 2017 -0700
Committer: Junping Du <ju...@apache.org>
Committed: Wed Aug 23 18:06:29 2017 -0700
----------------------------------------------------------------------
.../yarn/server/resourcemanager/TestRMRestart.java | 16 ++++++++++++++++
1 file changed, 16 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/652dd434/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
index 3130ad1..4b4dffd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMRestart.java
@@ -2504,6 +2504,22 @@ public class TestRMRestart extends ParameterizedSchedulerTestBase {
@Test(timeout = 60000)
public void testRMRestartAfterNodeLabelDisabled() throws Exception {
+ if (getSchedulerType() != SchedulerType.CAPACITY) {
+ return;
+ }
+
+ // Initial FS node label store root dir to a random tmp dir
+ File nodeLabelFsStoreDir = new File("target",
+ this.getClass().getSimpleName()
+ + "-testRMRestartAfterNodeLabelDisabled");
+ if (nodeLabelFsStoreDir.exists()) {
+ FileUtils.deleteDirectory(nodeLabelFsStoreDir);
+ }
+ nodeLabelFsStoreDir.deleteOnExit();
+ String nodeLabelFsStoreDirURI = nodeLabelFsStoreDir.toURI().toString();
+ conf.set(YarnConfiguration.FS_NODE_LABELS_STORE_ROOT_DIR,
+ nodeLabelFsStoreDirURI);
+
conf.setBoolean(YarnConfiguration.NODE_LABELS_ENABLED, true);
MockRM rm1 = new MockRM(
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[05/50] [abbrv] hadoop git commit: YARN-6965. Duplicate instantiation
in FairSchedulerQueueInfo. Contributed by Masahiro Tanaka.
Posted by as...@apache.org.
YARN-6965. Duplicate instantiation in FairSchedulerQueueInfo. Contributed by Masahiro Tanaka.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/588c190a
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/588c190a
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/588c190a
Branch: refs/heads/YARN-5972
Commit: 588c190afd49bdbd5708f7805bf6c68f09fee142
Parents: 75dd866
Author: Akira Ajisaka <aa...@apache.org>
Authored: Wed Aug 16 14:06:22 2017 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Wed Aug 16 14:06:22 2017 +0900
----------------------------------------------------------------------
.../server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java | 1 -
1 file changed, 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/588c190a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java
index a4607c2..79339c7 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java
@@ -99,7 +99,6 @@ public class FairSchedulerQueueInfo {
steadyFairResources = new ResourceInfo(queue.getSteadyFairShare());
fairResources = new ResourceInfo(queue.getFairShare());
minResources = new ResourceInfo(queue.getMinShare());
- maxResources = new ResourceInfo(queue.getMaxShare());
maxResources = new ResourceInfo(
Resources.componentwiseMin(queue.getMaxShare(),
scheduler.getClusterResource()));
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[36/50] [abbrv] hadoop git commit: HADOOP-14705. Add batched
interface reencryptEncryptedKeys to KMS.
Posted by as...@apache.org.
HADOOP-14705. Add batched interface reencryptEncryptedKeys to KMS.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4ec5acc7
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4ec5acc7
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4ec5acc7
Branch: refs/heads/YARN-5972
Commit: 4ec5acc70418a3f2327cf83ecae1789a057fdd99
Parents: 27ab5f7
Author: Xiao Chen <xi...@apache.org>
Authored: Tue Aug 22 07:46:46 2017 -0700
Committer: Xiao Chen <xi...@apache.org>
Committed: Tue Aug 22 07:47:39 2017 -0700
----------------------------------------------------------------------
.../crypto/key/KeyProviderCryptoExtension.java | 147 ++++++++++++++++---
.../crypto/key/kms/KMSClientProvider.java | 145 +++++++-----------
.../hadoop/crypto/key/kms/KMSRESTConstants.java | 1 +
.../key/kms/LoadBalancingKMSClientProvider.java | 20 +++
.../java/org/apache/hadoop/util/KMSUtil.java | 134 +++++++++++++++++
.../key/TestKeyProviderCryptoExtension.java | 113 ++++++++++----
...rKeyGeneratorKeyProviderCryptoExtension.java | 6 +
.../hadoop/crypto/key/kms/server/KMS.java | 113 +++++++++++---
.../crypto/key/kms/server/KMSJSONReader.java | 8 +-
.../key/kms/server/KMSServerJSONUtils.java | 34 +----
.../hadoop/crypto/key/kms/server/KMSWebApp.java | 18 +++
.../kms/server/KeyAuthorizationKeyProvider.java | 19 +++
.../hadoop-kms/src/site/markdown/index.md.vm | 60 +++++++-
.../hadoop/crypto/key/kms/server/TestKMS.java | 46 +++++-
.../crypto/key/kms/server/TestKMSAudit.java | 7 +-
15 files changed, 673 insertions(+), 198 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4ec5acc7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
index ea5ff28..8c879b3 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/KeyProviderCryptoExtension.java
@@ -22,6 +22,8 @@ import java.io.IOException;
import java.nio.ByteBuffer;
import java.security.GeneralSecurityException;
import java.security.SecureRandom;
+import java.util.List;
+import java.util.ListIterator;
import javax.crypto.Cipher;
import javax.crypto.spec.IvParameterSpec;
@@ -247,6 +249,25 @@ public class KeyProviderCryptoExtension extends
*/
EncryptedKeyVersion reencryptEncryptedKey(EncryptedKeyVersion ekv)
throws IOException, GeneralSecurityException;
+
+ /**
+ * Batched version of {@link #reencryptEncryptedKey(EncryptedKeyVersion)}.
+ * <p>
+ * For each encrypted key version, re-encrypts an encrypted key version,
+ * using its initialization vector and key material, but with the latest
+ * key version name of its key name. If the latest key version name in the
+ * provider is the same as the one encrypted the passed-in encrypted key
+ * version, the same encrypted key version is returned.
+ * <p>
+ * NOTE: The generated key is not stored by the <code>KeyProvider</code>
+ *
+ * @param ekvs List containing the EncryptedKeyVersion's
+ * @throws IOException If any EncryptedKeyVersion could not be re-encrypted
+ * @throws GeneralSecurityException If any EncryptedKeyVersion could not be
+ * re-encrypted because of a cryptographic issue.
+ */
+ void reencryptEncryptedKeys(List<EncryptedKeyVersion> ekvs)
+ throws IOException, GeneralSecurityException;
}
private static class DefaultCryptoExtension implements CryptoExtension {
@@ -315,7 +336,7 @@ public class KeyProviderCryptoExtension extends
.checkNotNull(ekNow, "KeyVersion name '%s' does not exist", ekName);
Preconditions.checkArgument(ekv.getEncryptedKeyVersion().getVersionName()
.equals(KeyProviderCryptoExtension.EEK),
- "encryptedKey version name must be '%s', is '%s'",
+ "encryptedKey version name must be '%s', but found '%s'",
KeyProviderCryptoExtension.EEK,
ekv.getEncryptedKeyVersion().getVersionName());
@@ -336,30 +357,67 @@ public class KeyProviderCryptoExtension extends
}
@Override
- public KeyVersion decryptEncryptedKey(
- EncryptedKeyVersion encryptedKeyVersion) throws IOException,
- GeneralSecurityException {
- // Fetch the encryption key material
- final String encryptionKeyVersionName =
- encryptedKeyVersion.getEncryptionKeyVersionName();
- final KeyVersion encryptionKey =
- keyProvider.getKeyVersion(encryptionKeyVersionName);
- Preconditions.checkNotNull(encryptionKey,
- "KeyVersion name '%s' does not exist", encryptionKeyVersionName);
- Preconditions.checkArgument(
- encryptedKeyVersion.getEncryptedKeyVersion().getVersionName()
- .equals(KeyProviderCryptoExtension.EEK),
- "encryptedKey version name must be '%s', is '%s'",
- KeyProviderCryptoExtension.EEK,
- encryptedKeyVersion.getEncryptedKeyVersion().getVersionName()
- );
+ public void reencryptEncryptedKeys(List<EncryptedKeyVersion> ekvs)
+ throws IOException, GeneralSecurityException {
+ Preconditions.checkNotNull(ekvs, "Input list is null");
+ KeyVersion ekNow = null;
+ Decryptor decryptor = null;
+ Encryptor encryptor = null;
+ try (CryptoCodec cc = CryptoCodec.getInstance(keyProvider.getConf())) {
+ decryptor = cc.createDecryptor();
+ encryptor = cc.createEncryptor();
+ ListIterator<EncryptedKeyVersion> iter = ekvs.listIterator();
+ while (iter.hasNext()) {
+ final EncryptedKeyVersion ekv = iter.next();
+ Preconditions.checkNotNull(ekv, "EncryptedKeyVersion is null");
+ final String ekName = ekv.getEncryptionKeyName();
+ Preconditions.checkNotNull(ekName, "Key name is null");
+ Preconditions.checkNotNull(ekv.getEncryptedKeyVersion(),
+ "EncryptedKeyVersion is null");
+ Preconditions.checkArgument(
+ ekv.getEncryptedKeyVersion().getVersionName()
+ .equals(KeyProviderCryptoExtension.EEK),
+ "encryptedKey version name must be '%s', but found '%s'",
+ KeyProviderCryptoExtension.EEK,
+ ekv.getEncryptedKeyVersion().getVersionName());
+
+ if (ekNow == null) {
+ ekNow = keyProvider.getCurrentKey(ekName);
+ Preconditions
+ .checkNotNull(ekNow, "Key name '%s' does not exist", ekName);
+ } else {
+ Preconditions.checkArgument(ekNow.getName().equals(ekName),
+ "All keys must have the same key name. Expected '%s' "
+ + "but found '%s'", ekNow.getName(), ekName);
+ }
+
+ final String encryptionKeyVersionName =
+ ekv.getEncryptionKeyVersionName();
+ final KeyVersion encryptionKey =
+ keyProvider.getKeyVersion(encryptionKeyVersionName);
+ Preconditions.checkNotNull(encryptionKey,
+ "KeyVersion name '%s' does not exist", encryptionKeyVersionName);
+ if (encryptionKey.equals(ekNow)) {
+ // no-op if same key version
+ continue;
+ }
+
+ final KeyVersion ek =
+ decryptEncryptedKey(decryptor, encryptionKey, ekv);
+ iter.set(generateEncryptedKey(encryptor, ekNow, ek.getMaterial(),
+ ekv.getEncryptedKeyIv()));
+ }
+ }
+ }
+ private KeyVersion decryptEncryptedKey(final Decryptor decryptor,
+ final KeyVersion encryptionKey,
+ final EncryptedKeyVersion encryptedKeyVersion)
+ throws IOException, GeneralSecurityException {
// Encryption key IV is determined from encrypted key's IV
final byte[] encryptionIV =
EncryptedKeyVersion.deriveIV(encryptedKeyVersion.getEncryptedKeyIv());
- CryptoCodec cc = CryptoCodec.getInstance(keyProvider.getConf());
- Decryptor decryptor = cc.createDecryptor();
decryptor.init(encryptionKey.getMaterial(), encryptionIV);
final KeyVersion encryptedKV =
encryptedKeyVersion.getEncryptedKeyVersion();
@@ -372,11 +430,36 @@ public class KeyProviderCryptoExtension extends
bbOut.flip();
byte[] decryptedKey = new byte[keyLen];
bbOut.get(decryptedKey);
- cc.close();
return new KeyVersion(encryptionKey.getName(), EK, decryptedKey);
}
@Override
+ public KeyVersion decryptEncryptedKey(
+ EncryptedKeyVersion encryptedKeyVersion)
+ throws IOException, GeneralSecurityException {
+ // Fetch the encryption key material
+ final String encryptionKeyVersionName =
+ encryptedKeyVersion.getEncryptionKeyVersionName();
+ final KeyVersion encryptionKey =
+ keyProvider.getKeyVersion(encryptionKeyVersionName);
+ Preconditions
+ .checkNotNull(encryptionKey, "KeyVersion name '%s' does not exist",
+ encryptionKeyVersionName);
+ Preconditions.checkArgument(
+ encryptedKeyVersion.getEncryptedKeyVersion().getVersionName()
+ .equals(KeyProviderCryptoExtension.EEK),
+ "encryptedKey version name must be '%s', but found '%s'",
+ KeyProviderCryptoExtension.EEK,
+ encryptedKeyVersion.getEncryptedKeyVersion().getVersionName());
+
+ try (CryptoCodec cc = CryptoCodec.getInstance(keyProvider.getConf())) {
+ final Decryptor decryptor = cc.createDecryptor();
+ return decryptEncryptedKey(decryptor, encryptionKey,
+ encryptedKeyVersion);
+ }
+ }
+
+ @Override
public void warmUpEncryptedKeys(String... keyNames)
throws IOException {
// NO-OP since the default version does not cache any keys
@@ -471,6 +554,28 @@ public class KeyProviderCryptoExtension extends
}
/**
+ * Batched version of {@link #reencryptEncryptedKey(EncryptedKeyVersion)}.
+ * <p>
+ * For each encrypted key version, re-encrypts an encrypted key version,
+ * using its initialization vector and key material, but with the latest
+ * key version name of its key name. If the latest key version name in the
+ * provider is the same as the one encrypted the passed-in encrypted key
+ * version, the same encrypted key version is returned.
+ * <p>
+ * NOTE: The generated key is not stored by the <code>KeyProvider</code>
+ *
+ * @param ekvs List containing the EncryptedKeyVersion's
+ * @return The re-encrypted EncryptedKeyVersion's, in the same order.
+ * @throws IOException If any EncryptedKeyVersion could not be re-encrypted
+ * @throws GeneralSecurityException If any EncryptedKeyVersion could not be
+ * re-encrypted because of a cryptographic issue.
+ */
+ public void reencryptEncryptedKeys(List<EncryptedKeyVersion> ekvs)
+ throws IOException, GeneralSecurityException {
+ getExtension().reencryptEncryptedKeys(ekvs);
+ }
+
+ /**
* Creates a <code>KeyProviderCryptoExtension</code> using a given
* {@link KeyProvider}.
* <p/>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4ec5acc7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
index 20ad58c..af0dd82 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
@@ -70,7 +70,6 @@ import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
-import java.util.LinkedList;
import java.util.List;
import java.util.Map;
import java.util.Queue;
@@ -84,6 +83,13 @@ import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
import com.google.common.base.Strings;
+import static org.apache.hadoop.util.KMSUtil.checkNotEmpty;
+import static org.apache.hadoop.util.KMSUtil.checkNotNull;
+import static org.apache.hadoop.util.KMSUtil.parseJSONEncKeyVersion;
+import static org.apache.hadoop.util.KMSUtil.parseJSONEncKeyVersions;
+import static org.apache.hadoop.util.KMSUtil.parseJSONKeyVersion;
+import static org.apache.hadoop.util.KMSUtil.parseJSONMetadata;
+
/**
* KMS client <code>KeyProvider</code> implementation.
*/
@@ -219,77 +225,11 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension,
}
}
- @SuppressWarnings("rawtypes")
- private static List<EncryptedKeyVersion>
- parseJSONEncKeyVersions(String keyName, List valueList) {
- List<EncryptedKeyVersion> ekvs = new LinkedList<EncryptedKeyVersion>();
- if (!valueList.isEmpty()) {
- for (Object values : valueList) {
- Map valueMap = (Map) values;
- ekvs.add(parseJSONEncKeyVersion(keyName, valueMap));
- }
- }
- return ekvs;
- }
-
- private static EncryptedKeyVersion parseJSONEncKeyVersion(String keyName,
- Map valueMap) {
- String versionName = checkNotNull(
- (String) valueMap.get(KMSRESTConstants.VERSION_NAME_FIELD),
- KMSRESTConstants.VERSION_NAME_FIELD);
-
- byte[] iv = Base64.decodeBase64(checkNotNull(
- (String) valueMap.get(KMSRESTConstants.IV_FIELD),
- KMSRESTConstants.IV_FIELD));
-
- Map encValueMap = checkNotNull((Map)
- valueMap.get(KMSRESTConstants.ENCRYPTED_KEY_VERSION_FIELD),
- KMSRESTConstants.ENCRYPTED_KEY_VERSION_FIELD);
-
- String encVersionName = checkNotNull((String)
- encValueMap.get(KMSRESTConstants.VERSION_NAME_FIELD),
- KMSRESTConstants.VERSION_NAME_FIELD);
-
- byte[] encKeyMaterial = Base64.decodeBase64(checkNotNull((String)
- encValueMap.get(KMSRESTConstants.MATERIAL_FIELD),
- KMSRESTConstants.MATERIAL_FIELD));
-
- return new KMSEncryptedKeyVersion(keyName, versionName, iv,
- encVersionName, encKeyMaterial);
- }
-
- private static KeyVersion parseJSONKeyVersion(Map valueMap) {
- KeyVersion keyVersion = null;
- if (!valueMap.isEmpty()) {
- byte[] material = (valueMap.containsKey(KMSRESTConstants.MATERIAL_FIELD))
- ? Base64.decodeBase64((String) valueMap.get(KMSRESTConstants.MATERIAL_FIELD))
- : null;
- String versionName = (String)valueMap.get(KMSRESTConstants.VERSION_NAME_FIELD);
- String keyName = (String)valueMap.get(KMSRESTConstants.NAME_FIELD);
- keyVersion = new KMSKeyVersion(keyName, versionName, material);
- }
- return keyVersion;
- }
-
- @SuppressWarnings("unchecked")
- private static Metadata parseJSONMetadata(Map valueMap) {
- Metadata metadata = null;
- if (!valueMap.isEmpty()) {
- metadata = new KMSMetadata(
- (String) valueMap.get(KMSRESTConstants.CIPHER_FIELD),
- (Integer) valueMap.get(KMSRESTConstants.LENGTH_FIELD),
- (String) valueMap.get(KMSRESTConstants.DESCRIPTION_FIELD),
- (Map<String, String>) valueMap.get(KMSRESTConstants.ATTRIBUTES_FIELD),
- new Date((Long) valueMap.get(KMSRESTConstants.CREATED_FIELD)),
- (Integer) valueMap.get(KMSRESTConstants.VERSIONS_FIELD));
- }
- return metadata;
- }
-
- private static void writeJson(Map map, OutputStream os) throws IOException {
+ private static void writeJson(Object obj, OutputStream os)
+ throws IOException {
Writer writer = new OutputStreamWriter(os, StandardCharsets.UTF_8);
ObjectMapper jsonMapper = new ObjectMapper();
- jsonMapper.writerWithDefaultPrettyPrinter().writeValue(writer, map);
+ jsonMapper.writerWithDefaultPrettyPrinter().writeValue(writer, obj);
}
/**
@@ -360,25 +300,6 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension,
}
}
- public static <T> T checkNotNull(T o, String name)
- throws IllegalArgumentException {
- if (o == null) {
- throw new IllegalArgumentException("Parameter '" + name +
- "' cannot be null");
- }
- return o;
- }
-
- public static String checkNotEmpty(String s, String name)
- throws IllegalArgumentException {
- checkNotNull(s, name);
- if (s.isEmpty()) {
- throw new IllegalArgumentException("Parameter '" + name +
- "' cannot be empty");
- }
- return s;
- }
-
private String kmsUrl;
private SSLFactory sslFactory;
private ConnectionConfigurator configurator;
@@ -560,12 +481,12 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension,
return conn;
}
- private <T> T call(HttpURLConnection conn, Map jsonOutput,
+ private <T> T call(HttpURLConnection conn, Object jsonOutput,
int expectedResponse, Class<T> klass) throws IOException {
return call(conn, jsonOutput, expectedResponse, klass, authRetry);
}
- private <T> T call(HttpURLConnection conn, Map jsonOutput,
+ private <T> T call(HttpURLConnection conn, Object jsonOutput,
int expectedResponse, Class<T> klass, int authRetryCount)
throws IOException {
T ret = null;
@@ -885,6 +806,48 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension,
}
@Override
+ public void reencryptEncryptedKeys(List<EncryptedKeyVersion> ekvs)
+ throws IOException, GeneralSecurityException {
+ checkNotNull(ekvs, "ekvs");
+ if (ekvs.isEmpty()) {
+ return;
+ }
+ final List<Map> jsonPayload = new ArrayList<>();
+ String keyName = null;
+ for (EncryptedKeyVersion ekv : ekvs) {
+ checkNotNull(ekv.getEncryptionKeyName(), "keyName");
+ checkNotNull(ekv.getEncryptionKeyVersionName(), "versionName");
+ checkNotNull(ekv.getEncryptedKeyIv(), "iv");
+ checkNotNull(ekv.getEncryptedKeyVersion(), "encryptedKey");
+ Preconditions.checkArgument(ekv.getEncryptedKeyVersion().getVersionName()
+ .equals(KeyProviderCryptoExtension.EEK),
+ "encryptedKey version name must be '%s', is '%s'",
+ KeyProviderCryptoExtension.EEK,
+ ekv.getEncryptedKeyVersion().getVersionName());
+ if (keyName == null) {
+ keyName = ekv.getEncryptionKeyName();
+ } else {
+ Preconditions.checkArgument(keyName.equals(ekv.getEncryptionKeyName()),
+ "All EncryptedKey must have the same key name.");
+ }
+ jsonPayload.add(KMSUtil.toJSON(ekv));
+ }
+ final URL url = createURL(KMSRESTConstants.KEY_RESOURCE, keyName,
+ KMSRESTConstants.REENCRYPT_BATCH_SUB_RESOURCE, null);
+ final HttpURLConnection conn = createConnection(url, HTTP_POST);
+ conn.setRequestProperty(CONTENT_TYPE, APPLICATION_JSON_MIME);
+ final List<Map> response =
+ call(conn, jsonPayload, HttpURLConnection.HTTP_OK, List.class);
+ Preconditions.checkArgument(response.size() == ekvs.size(),
+ "Response size is different than input size.");
+ for (int i = 0; i < response.size(); ++i) {
+ final Map item = response.get(i);
+ final EncryptedKeyVersion ekv = parseJSONEncKeyVersion(keyName, item);
+ ekvs.set(i, ekv);
+ }
+ }
+
+ @Override
public List<KeyVersion> getKeyVersions(String name) throws IOException {
checkNotEmpty(name, "name");
URL url = createURL(KMSRESTConstants.KEY_RESOURCE, name,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4ec5acc7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSRESTConstants.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSRESTConstants.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSRESTConstants.java
index 81bdd33..e0197e0 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSRESTConstants.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSRESTConstants.java
@@ -37,6 +37,7 @@ public class KMSRESTConstants {
public static final String EEK_SUB_RESOURCE = "_eek";
public static final String CURRENT_VERSION_SUB_RESOURCE = "_currentversion";
public static final String INVALIDATECACHE_RESOURCE = "_invalidatecache";
+ public static final String REENCRYPT_BATCH_SUB_RESOURCE = "_reencryptbatch";
public static final String KEY = "key";
public static final String EEK_OP = "eek_op";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4ec5acc7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/LoadBalancingKMSClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/LoadBalancingKMSClientProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/LoadBalancingKMSClientProvider.java
index 6e010b1..de4d25a 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/LoadBalancingKMSClientProvider.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/LoadBalancingKMSClientProvider.java
@@ -313,6 +313,26 @@ public class LoadBalancingKMSClientProvider extends KeyProvider implements
}
@Override
+ public void reencryptEncryptedKeys(final List<EncryptedKeyVersion> ekvs)
+ throws IOException, GeneralSecurityException {
+ try {
+ doOp(new ProviderCallable<Void>() {
+ @Override
+ public Void call(KMSClientProvider provider)
+ throws IOException, GeneralSecurityException {
+ provider.reencryptEncryptedKeys(ekvs);
+ return null;
+ }
+ }, nextIdx());
+ } catch (WrapperException we) {
+ if (we.getCause() instanceof GeneralSecurityException) {
+ throw (GeneralSecurityException) we.getCause();
+ }
+ throw new IOException(we.getCause());
+ }
+ }
+
+ @Override
public KeyVersion getKeyVersion(final String versionName) throws IOException {
return doOp(new ProviderCallable<KeyVersion>() {
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4ec5acc7/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/KMSUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/KMSUtil.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/KMSUtil.java
index 5f783a9..c96c6fb 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/KMSUtil.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/KMSUtil.java
@@ -17,15 +17,24 @@
*/
package org.apache.hadoop.util;
+import org.apache.commons.codec.binary.Base64;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.crypto.key.KeyProvider;
+import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension.EncryptedKeyVersion;
import org.apache.hadoop.crypto.key.KeyProviderFactory;
+import org.apache.hadoop.crypto.key.kms.KMSClientProvider;
+import org.apache.hadoop.crypto.key.kms.KMSRESTConstants;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.net.URI;
+import java.util.ArrayList;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
/**
* Utils for KMS.
@@ -71,4 +80,129 @@ public final class KMSUtil {
}
return keyProvider;
}
+
+ @SuppressWarnings("unchecked")
+ public static Map toJSON(KeyProvider.KeyVersion keyVersion) {
+ Map json = new HashMap();
+ if (keyVersion != null) {
+ json.put(KMSRESTConstants.NAME_FIELD,
+ keyVersion.getName());
+ json.put(KMSRESTConstants.VERSION_NAME_FIELD,
+ keyVersion.getVersionName());
+ json.put(KMSRESTConstants.MATERIAL_FIELD,
+ Base64.encodeBase64URLSafeString(
+ keyVersion.getMaterial()));
+ }
+ return json;
+ }
+
+ @SuppressWarnings("unchecked")
+ public static Map toJSON(EncryptedKeyVersion encryptedKeyVersion) {
+ Map json = new HashMap();
+ if (encryptedKeyVersion != null) {
+ json.put(KMSRESTConstants.VERSION_NAME_FIELD,
+ encryptedKeyVersion.getEncryptionKeyVersionName());
+ json.put(KMSRESTConstants.IV_FIELD, Base64
+ .encodeBase64URLSafeString(encryptedKeyVersion.getEncryptedKeyIv()));
+ json.put(KMSRESTConstants.ENCRYPTED_KEY_VERSION_FIELD,
+ toJSON(encryptedKeyVersion.getEncryptedKeyVersion()));
+ }
+ return json;
+ }
+
+ public static <T> T checkNotNull(T o, String name)
+ throws IllegalArgumentException {
+ if (o == null) {
+ throw new IllegalArgumentException("Parameter '" + name +
+ "' cannot be null");
+ }
+ return o;
+ }
+
+ public static String checkNotEmpty(String s, String name)
+ throws IllegalArgumentException {
+ checkNotNull(s, name);
+ if (s.isEmpty()) {
+ throw new IllegalArgumentException("Parameter '" + name +
+ "' cannot be empty");
+ }
+ return s;
+ }
+
+ @SuppressWarnings("rawtypes")
+ public static List<EncryptedKeyVersion>
+ parseJSONEncKeyVersions(String keyName, List valueList) {
+ checkNotNull(valueList, "valueList");
+ List<EncryptedKeyVersion> ekvs = new ArrayList<>(valueList.size());
+ if (!valueList.isEmpty()) {
+ for (Object values : valueList) {
+ Map valueMap = (Map) values;
+ ekvs.add(parseJSONEncKeyVersion(keyName, valueMap));
+ }
+ }
+ return ekvs;
+ }
+
+ @SuppressWarnings("unchecked")
+ public static EncryptedKeyVersion parseJSONEncKeyVersion(String keyName,
+ Map valueMap) {
+ checkNotNull(valueMap, "valueMap");
+ String versionName = checkNotNull(
+ (String) valueMap.get(KMSRESTConstants.VERSION_NAME_FIELD),
+ KMSRESTConstants.VERSION_NAME_FIELD);
+
+ byte[] iv = Base64.decodeBase64(checkNotNull(
+ (String) valueMap.get(KMSRESTConstants.IV_FIELD),
+ KMSRESTConstants.IV_FIELD));
+
+ Map encValueMap = checkNotNull((Map)
+ valueMap.get(KMSRESTConstants.ENCRYPTED_KEY_VERSION_FIELD),
+ KMSRESTConstants.ENCRYPTED_KEY_VERSION_FIELD);
+
+ String encVersionName = checkNotNull((String)
+ encValueMap.get(KMSRESTConstants.VERSION_NAME_FIELD),
+ KMSRESTConstants.VERSION_NAME_FIELD);
+
+ byte[] encKeyMaterial = Base64.decodeBase64(checkNotNull((String)
+ encValueMap.get(KMSRESTConstants.MATERIAL_FIELD),
+ KMSRESTConstants.MATERIAL_FIELD));
+
+ return new KMSClientProvider.KMSEncryptedKeyVersion(keyName, versionName,
+ iv, encVersionName, encKeyMaterial);
+ }
+
+ @SuppressWarnings("unchecked")
+ public static KeyProvider.KeyVersion parseJSONKeyVersion(Map valueMap) {
+ checkNotNull(valueMap, "valueMap");
+ KeyProvider.KeyVersion keyVersion = null;
+ if (!valueMap.isEmpty()) {
+ byte[] material =
+ (valueMap.containsKey(KMSRESTConstants.MATERIAL_FIELD)) ?
+ Base64.decodeBase64(
+ (String) valueMap.get(KMSRESTConstants.MATERIAL_FIELD)) :
+ null;
+ String versionName =
+ (String) valueMap.get(KMSRESTConstants.VERSION_NAME_FIELD);
+ String keyName = (String) valueMap.get(KMSRESTConstants.NAME_FIELD);
+ keyVersion =
+ new KMSClientProvider.KMSKeyVersion(keyName, versionName, material);
+ }
+ return keyVersion;
+ }
+
+ @SuppressWarnings("unchecked")
+ public static KeyProvider.Metadata parseJSONMetadata(Map valueMap) {
+ checkNotNull(valueMap, "valueMap");
+ KeyProvider.Metadata metadata = null;
+ if (!valueMap.isEmpty()) {
+ metadata = new KMSClientProvider.KMSMetadata(
+ (String) valueMap.get(KMSRESTConstants.CIPHER_FIELD),
+ (Integer) valueMap.get(KMSRESTConstants.LENGTH_FIELD),
+ (String) valueMap.get(KMSRESTConstants.DESCRIPTION_FIELD),
+ (Map<String, String>) valueMap.get(KMSRESTConstants.ATTRIBUTES_FIELD),
+ new Date((Long) valueMap.get(KMSRESTConstants.CREATED_FIELD)),
+ (Integer) valueMap.get(KMSRESTConstants.VERSIONS_FIELD));
+ }
+ return metadata;
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4ec5acc7/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderCryptoExtension.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderCryptoExtension.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderCryptoExtension.java
index 32938e3..e897423 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderCryptoExtension.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/crypto/key/TestKeyProviderCryptoExtension.java
@@ -22,6 +22,7 @@ import java.net.URI;
import java.net.URISyntaxException;
import java.security.GeneralSecurityException;
import java.security.SecureRandom;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
@@ -40,7 +41,9 @@ import org.junit.rules.Timeout;
import static org.apache.hadoop.crypto.key.KeyProvider.KeyVersion;
import static org.junit.Assert.assertArrayEquals;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
public class TestKeyProviderCryptoExtension {
@@ -90,13 +93,7 @@ public class TestKeyProviderCryptoExtension {
KeyVersion k1 = kpExt.decryptEncryptedKey(ek1);
assertEquals(KeyProviderCryptoExtension.EK, k1.getVersionName());
assertEquals(encryptionKey.getMaterial().length, k1.getMaterial().length);
- if (Arrays.equals(k1.getMaterial(), encryptionKey.getMaterial())) {
- fail("Encrypted key material should not equal encryption key material");
- }
- if (Arrays.equals(ek1.getEncryptedKeyVersion().getMaterial(),
- encryptionKey.getMaterial())) {
- fail("Encrypted key material should not equal decrypted key material");
- }
+
// Decrypt it again and it should be the same
KeyVersion k1a = kpExt.decryptEncryptedKey(ek1);
assertArrayEquals(k1.getMaterial(), k1a.getMaterial());
@@ -153,9 +150,6 @@ public class TestKeyProviderCryptoExtension {
final KeyVersion k1 = kpExt.decryptEncryptedKey(ek1);
assertEquals(KeyProviderCryptoExtension.EK, k1.getVersionName());
assertEquals(encryptionKey.getMaterial().length, k1.getMaterial().length);
- if (Arrays.equals(k1.getMaterial(), encryptionKey.getMaterial())) {
- fail("Encrypted key material should not equal encryption key material");
- }
// Roll the EK
kpExt.rollNewVersion(ek1.getEncryptionKeyName());
@@ -173,10 +167,7 @@ public class TestKeyProviderCryptoExtension {
assertEquals("Length of encryption key material and EEK material should "
+ "be the same", encryptionKey.getMaterial().length,
ek2.getEncryptedKeyVersion().getMaterial().length);
- if (Arrays.equals(ek2.getEncryptedKeyVersion().getMaterial(),
- encryptionKey.getMaterial())) {
- fail("Encrypted key material should not equal decrypted key material");
- }
+
if (Arrays.equals(ek2.getEncryptedKeyVersion().getMaterial(),
ek1.getEncryptedKeyVersion().getMaterial())) {
fail("Re-encrypted EEK should have different material");
@@ -186,9 +177,6 @@ public class TestKeyProviderCryptoExtension {
final KeyVersion k2 = kpExt.decryptEncryptedKey(ek2);
assertEquals(KeyProviderCryptoExtension.EK, k2.getVersionName());
assertEquals(encryptionKey.getMaterial().length, k2.getMaterial().length);
- if (Arrays.equals(k2.getMaterial(), encryptionKey.getMaterial())) {
- fail("Encrypted key material should not equal encryption key material");
- }
// Re-encrypting the same EEK with the same EK should be deterministic
final KeyProviderCryptoExtension.EncryptedKeyVersion ek2a =
@@ -203,10 +191,7 @@ public class TestKeyProviderCryptoExtension {
assertEquals("Length of encryption key material and EEK material should "
+ "be the same", encryptionKey.getMaterial().length,
ek2a.getEncryptedKeyVersion().getMaterial().length);
- if (Arrays.equals(ek2a.getEncryptedKeyVersion().getMaterial(),
- encryptionKey.getMaterial())) {
- fail("Encrypted key material should not equal decrypted key material");
- }
+
if (Arrays.equals(ek2a.getEncryptedKeyVersion().getMaterial(),
ek1.getEncryptedKeyVersion().getMaterial())) {
fail("Re-encrypted EEK should have different material");
@@ -227,10 +212,6 @@ public class TestKeyProviderCryptoExtension {
assertEquals("Length of encryption key material and EEK material should "
+ "be the same", encryptionKey.getMaterial().length,
ek3.getEncryptedKeyVersion().getMaterial().length);
- if (Arrays.equals(ek3.getEncryptedKeyVersion().getMaterial(),
- encryptionKey.getMaterial())) {
- fail("Encrypted key material should not equal decrypted key material");
- }
if (Arrays.equals(ek3.getEncryptedKeyVersion().getMaterial(),
ek1.getEncryptedKeyVersion().getMaterial())) {
@@ -241,6 +222,78 @@ public class TestKeyProviderCryptoExtension {
}
@Test
+ public void testReencryptEncryptedKeys() throws Exception {
+ List<EncryptedKeyVersion> ekvs = new ArrayList<>(4);
+ // Generate 2 new EEKs @v0 and add to the list
+ ekvs.add(kpExt.generateEncryptedKey(encryptionKey.getName()));
+ ekvs.add(kpExt.generateEncryptedKey(encryptionKey.getName()));
+
+ // Roll the EK
+ kpExt.rollNewVersion(ekvs.get(0).getEncryptionKeyName());
+ // Generate 1 new EEK @v1 add to the list.
+ ekvs.add(kpExt.generateEncryptedKey(encryptionKey.getName()));
+
+ // Roll the EK again
+ kpExt.rollNewVersion(ekvs.get(0).getEncryptionKeyName());
+ // Generate 1 new EEK @v2 add to the list.
+ ekvs.add(kpExt.generateEncryptedKey(encryptionKey.getName()));
+
+ // leave a deep copy of the original, for verification purpose.
+ List<EncryptedKeyVersion> ekvsOrig = new ArrayList<>(ekvs.size());
+ for (EncryptedKeyVersion ekv : ekvs) {
+ ekvsOrig.add(new EncryptedKeyVersion(ekv.getEncryptionKeyName(),
+ ekv.getEncryptionKeyVersionName(), ekv.getEncryptedKeyIv(),
+ ekv.getEncryptedKeyVersion()));
+ }
+
+ // Reencrypt ekvs
+ kpExt.reencryptEncryptedKeys(ekvs);
+
+ // Verify each ekv
+ for (int i = 0; i < ekvs.size(); ++i) {
+ final EncryptedKeyVersion ekv = ekvs.get(i);
+ final EncryptedKeyVersion orig = ekvsOrig.get(i);
+ assertEquals("Version name should be EEK",
+ KeyProviderCryptoExtension.EEK,
+ ekv.getEncryptedKeyVersion().getVersionName());
+ assertEquals("Encryption key name should be " + ENCRYPTION_KEY_NAME,
+ ENCRYPTION_KEY_NAME, ekv.getEncryptionKeyName());
+ assertNotNull("Expected encrypted key material",
+ ekv.getEncryptedKeyVersion().getMaterial());
+ assertEquals("Length of encryption key material and EEK material should "
+ + "be the same", encryptionKey.getMaterial().length,
+ ekv.getEncryptedKeyVersion().getMaterial().length);
+ assertFalse(
+ "Encrypted key material should not equal encryption key material",
+ Arrays.equals(ekv.getEncryptedKeyVersion().getMaterial(),
+ encryptionKey.getMaterial()));
+
+ if (i < 3) {
+ assertFalse("Re-encrypted EEK should have different material",
+ Arrays.equals(ekv.getEncryptedKeyVersion().getMaterial(),
+ orig.getEncryptedKeyVersion().getMaterial()));
+ } else {
+ assertTrue("Re-encrypted EEK should have same material",
+ Arrays.equals(ekv.getEncryptedKeyVersion().getMaterial(),
+ orig.getEncryptedKeyVersion().getMaterial()));
+ }
+
+ // Decrypt the new EEK into an EK and check it
+ final KeyVersion kv = kpExt.decryptEncryptedKey(ekv);
+ assertEquals(KeyProviderCryptoExtension.EK, kv.getVersionName());
+
+ // Decrypt it again and it should be the same
+ KeyVersion kv1 = kpExt.decryptEncryptedKey(ekv);
+ assertArrayEquals(kv.getMaterial(), kv1.getMaterial());
+
+ // Verify decrypting the new EEK and orig EEK gives the same material.
+ final KeyVersion origKv = kpExt.decryptEncryptedKey(orig);
+ assertTrue("Returned EEK and original EEK should both decrypt to the "
+ + "same kv.", Arrays.equals(origKv.getMaterial(), kv.getMaterial()));
+ }
+ }
+
+ @Test
public void testNonDefaultCryptoExtensionSelectionWithCachingKeyProvider()
throws Exception {
Configuration config = new Configuration();
@@ -342,6 +395,11 @@ public class TestKeyProviderCryptoExtension {
}
@Override
+ public void reencryptEncryptedKeys(List<EncryptedKeyVersion> ekvs)
+ throws IOException, GeneralSecurityException {
+ }
+
+ @Override
public KeyVersion decryptEncryptedKey(
EncryptedKeyVersion encryptedKeyVersion)
throws IOException, GeneralSecurityException {
@@ -453,5 +511,10 @@ public class TestKeyProviderCryptoExtension {
throws IOException, GeneralSecurityException {
return ekv;
}
+
+ @Override
+ public void reencryptEncryptedKeys(List<EncryptedKeyVersion> ekvs)
+ throws IOException, GeneralSecurityException {
+ }
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4ec5acc7/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/EagerKeyGeneratorKeyProviderCryptoExtension.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/EagerKeyGeneratorKeyProviderCryptoExtension.java b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/EagerKeyGeneratorKeyProviderCryptoExtension.java
index eb24394..273c673 100644
--- a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/EagerKeyGeneratorKeyProviderCryptoExtension.java
+++ b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/EagerKeyGeneratorKeyProviderCryptoExtension.java
@@ -141,6 +141,12 @@ public class EagerKeyGeneratorKeyProviderCryptoExtension
throws IOException, GeneralSecurityException {
return keyProviderCryptoExtension.reencryptEncryptedKey(ekv);
}
+
+ @Override
+ public void reencryptEncryptedKeys(List<EncryptedKeyVersion> ekvs)
+ throws IOException, GeneralSecurityException {
+ keyProviderCryptoExtension.reencryptEncryptedKeys(ekvs);
+ }
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4ec5acc7/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMS.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMS.java b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMS.java
index 27cc05d..dfc6872 100644
--- a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMS.java
+++ b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMS.java
@@ -17,6 +17,9 @@
*/
package org.apache.hadoop.crypto.key.kms.server;
+import com.google.common.base.Preconditions;
+import com.google.common.base.Stopwatch;
+import org.apache.hadoop.util.KMSUtil;
import org.apache.commons.codec.binary.Base64;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.crypto.key.KeyProvider;
@@ -53,6 +56,9 @@ import java.util.LinkedList;
import java.util.List;
import java.util.Map;
+import static org.apache.hadoop.util.KMSUtil.checkNotEmpty;
+import static org.apache.hadoop.util.KMSUtil.checkNotNull;
+
/**
* Class providing the REST bindings, via Jersey, for the KMS.
*/
@@ -64,7 +70,7 @@ public class KMS {
CREATE_KEY, DELETE_KEY, ROLL_NEW_VERSION, INVALIDATE_CACHE,
GET_KEYS, GET_KEYS_METADATA,
GET_KEY_VERSIONS, GET_METADATA, GET_KEY_VERSION, GET_CURRENT_KEY,
- GENERATE_EEK, DECRYPT_EEK, REENCRYPT_EEK
+ GENERATE_EEK, DECRYPT_EEK, REENCRYPT_EEK, REENCRYPT_EEK_BATCH
}
private KeyProviderCryptoExtension provider;
@@ -72,6 +78,8 @@ public class KMS {
static final Logger LOG = LoggerFactory.getLogger(KMS.class);
+ private static final int MAX_NUM_PER_BATCH = 10000;
+
public KMS() throws Exception {
provider = KMSWebApp.getKeyProvider();
kmsAudit= KMSWebApp.getKMSAudit();
@@ -109,7 +117,7 @@ public class KMS {
KMSWebApp.getAdminCallsMeter().mark();
UserGroupInformation user = HttpUserGroupInformation.get();
final String name = (String) jsonKey.get(KMSRESTConstants.NAME_FIELD);
- KMSClientProvider.checkNotEmpty(name, KMSRESTConstants.NAME_FIELD);
+ checkNotEmpty(name, KMSRESTConstants.NAME_FIELD);
assertAccess(KMSACLs.Type.CREATE, user, KMSOp.CREATE_KEY, name);
String cipher = (String) jsonKey.get(KMSRESTConstants.CIPHER_FIELD);
final String material;
@@ -158,7 +166,7 @@ public class KMS {
if (!KMSWebApp.getACLs().hasAccess(KMSACLs.Type.GET, user)) {
keyVersion = removeKeyMaterial(keyVersion);
}
- Map json = KMSServerJSONUtils.toJSON(keyVersion);
+ Map json = KMSUtil.toJSON(keyVersion);
String requestURL = KMSMDCFilter.getURL();
int idx = requestURL.lastIndexOf(KMSRESTConstants.KEYS_RESOURCE);
requestURL = requestURL.substring(0, idx);
@@ -181,7 +189,7 @@ public class KMS {
KMSWebApp.getAdminCallsMeter().mark();
UserGroupInformation user = HttpUserGroupInformation.get();
assertAccess(KMSACLs.Type.DELETE, user, KMSOp.DELETE_KEY, name);
- KMSClientProvider.checkNotEmpty(name, "name");
+ checkNotEmpty(name, "name");
LOG.debug("Deleting key with name {}.", name);
user.doAs(new PrivilegedExceptionAction<Void>() {
@Override
@@ -212,7 +220,7 @@ public class KMS {
KMSWebApp.getAdminCallsMeter().mark();
UserGroupInformation user = HttpUserGroupInformation.get();
assertAccess(KMSACLs.Type.ROLLOVER, user, KMSOp.ROLL_NEW_VERSION, name);
- KMSClientProvider.checkNotEmpty(name, "name");
+ checkNotEmpty(name, "name");
LOG.debug("Rolling key with name {}.", name);
final String material = (String)
jsonMaterial.get(KMSRESTConstants.MATERIAL_FIELD);
@@ -242,7 +250,7 @@ public class KMS {
if (!KMSWebApp.getACLs().hasAccess(KMSACLs.Type.GET, user)) {
keyVersion = removeKeyMaterial(keyVersion);
}
- Map json = KMSServerJSONUtils.toJSON(keyVersion);
+ Map json = KMSUtil.toJSON(keyVersion);
LOG.trace("Exiting rolloverKey Method.");
return Response.ok().type(MediaType.APPLICATION_JSON).entity(json)
.build();
@@ -260,7 +268,7 @@ public class KMS {
try {
LOG.trace("Entering invalidateCache Method.");
KMSWebApp.getAdminCallsMeter().mark();
- KMSClientProvider.checkNotEmpty(name, "name");
+ checkNotEmpty(name, "name");
UserGroupInformation user = HttpUserGroupInformation.get();
assertAccess(KMSACLs.Type.ROLLOVER, user, KMSOp.INVALIDATE_CACHE, name);
LOG.debug("Invalidating cache with key name {}.", name);
@@ -369,7 +377,7 @@ public class KMS {
try {
LOG.trace("Entering getMetadata method.");
UserGroupInformation user = HttpUserGroupInformation.get();
- KMSClientProvider.checkNotEmpty(name, "name");
+ checkNotEmpty(name, "name");
KMSWebApp.getAdminCallsMeter().mark();
assertAccess(KMSACLs.Type.GET_METADATA, user, KMSOp.GET_METADATA, name);
LOG.debug("Getting metadata for key with name {}.", name);
@@ -403,7 +411,7 @@ public class KMS {
try {
LOG.trace("Entering getCurrentVersion method.");
UserGroupInformation user = HttpUserGroupInformation.get();
- KMSClientProvider.checkNotEmpty(name, "name");
+ checkNotEmpty(name, "name");
KMSWebApp.getKeyCallsMeter().mark();
assertAccess(KMSACLs.Type.GET, user, KMSOp.GET_CURRENT_KEY, name);
LOG.debug("Getting key version for key with name {}.", name);
@@ -417,7 +425,7 @@ public class KMS {
}
);
- Object json = KMSServerJSONUtils.toJSON(keyVersion);
+ Object json = KMSUtil.toJSON(keyVersion);
kmsAudit.ok(user, KMSOp.GET_CURRENT_KEY, name, "");
LOG.trace("Exiting getCurrentVersion method.");
return Response.ok().type(MediaType.APPLICATION_JSON).entity(json)
@@ -436,7 +444,7 @@ public class KMS {
try {
LOG.trace("Entering getKeyVersion method.");
UserGroupInformation user = HttpUserGroupInformation.get();
- KMSClientProvider.checkNotEmpty(versionName, "versionName");
+ checkNotEmpty(versionName, "versionName");
KMSWebApp.getKeyCallsMeter().mark();
assertAccess(KMSACLs.Type.GET, user, KMSOp.GET_KEY_VERSION);
LOG.debug("Getting key with version name {}.", versionName);
@@ -453,7 +461,7 @@ public class KMS {
if (keyVersion != null) {
kmsAudit.ok(user, KMSOp.GET_KEY_VERSION, keyVersion.getName(), "");
}
- Object json = KMSServerJSONUtils.toJSON(keyVersion);
+ Object json = KMSUtil.toJSON(keyVersion);
LOG.trace("Exiting getKeyVersion method.");
return Response.ok().type(MediaType.APPLICATION_JSON).entity(json)
.build();
@@ -477,8 +485,8 @@ public class KMS {
try {
LOG.trace("Entering generateEncryptedKeys method.");
UserGroupInformation user = HttpUserGroupInformation.get();
- KMSClientProvider.checkNotEmpty(name, "name");
- KMSClientProvider.checkNotNull(edekOp, "eekOp");
+ checkNotEmpty(name, "name");
+ checkNotNull(edekOp, "eekOp");
LOG.debug("Generating encrypted key with name {}," +
" the edek Operation is {}.", name, edekOp);
@@ -512,7 +520,7 @@ public class KMS {
kmsAudit.ok(user, KMSOp.GENERATE_EEK, name, "");
retJSON = new ArrayList();
for (EncryptedKeyVersion edek : retEdeks) {
- ((ArrayList) retJSON).add(KMSServerJSONUtils.toJSON(edek));
+ ((ArrayList) retJSON).add(KMSUtil.toJSON(edek));
}
} else {
StringBuilder error;
@@ -537,6 +545,64 @@ public class KMS {
@SuppressWarnings("rawtypes")
@POST
+ @Path(KMSRESTConstants.KEY_RESOURCE + "/{name:.*}/" +
+ KMSRESTConstants.REENCRYPT_BATCH_SUB_RESOURCE)
+ @Consumes(MediaType.APPLICATION_JSON)
+ @Produces(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8)
+ public Response reencryptEncryptedKeys(
+ @PathParam("name") final String name,
+ final List<Map> jsonPayload)
+ throws Exception {
+ LOG.trace("Entering reencryptEncryptedKeys method.");
+ try {
+ final Stopwatch sw = new Stopwatch().start();
+ checkNotEmpty(name, "name");
+ checkNotNull(jsonPayload, "jsonPayload");
+ final UserGroupInformation user = HttpUserGroupInformation.get();
+ KMSWebApp.getReencryptEEKBatchCallsMeter().mark();
+ if (jsonPayload.size() > MAX_NUM_PER_BATCH) {
+ LOG.warn("Payload size {} too big for reencryptEncryptedKeys from"
+ + " user {}.", jsonPayload.size(), user);
+ }
+ assertAccess(KMSACLs.Type.GENERATE_EEK, user, KMSOp.REENCRYPT_EEK_BATCH,
+ name);
+ LOG.debug("Batch reencrypting {} Encrypted Keys for key name {}",
+ jsonPayload.size(), name);
+ final List<EncryptedKeyVersion> ekvs =
+ KMSUtil.parseJSONEncKeyVersions(name, jsonPayload);
+ Preconditions.checkArgument(ekvs.size() == jsonPayload.size(),
+ "EncryptedKey size mismatch after parsing from json");
+ for (EncryptedKeyVersion ekv : ekvs) {
+ Preconditions.checkArgument(name.equals(ekv.getEncryptionKeyName()),
+ "All EncryptedKeys must be under the given key name " + name);
+ }
+
+ user.doAs(new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws Exception {
+ provider.reencryptEncryptedKeys(ekvs);
+ return null;
+ }
+ });
+ List retJSON = new ArrayList<>(ekvs.size());
+ for (EncryptedKeyVersion ekv: ekvs) {
+ retJSON.add(KMSUtil.toJSON(ekv));
+ }
+ kmsAudit.ok(user, KMSOp.REENCRYPT_EEK_BATCH, name,
+ "reencrypted " + ekvs.size() + " keys");
+ LOG.info("reencryptEncryptedKeys {} keys for key {} took {}",
+ jsonPayload.size(), name, sw.stop());
+ LOG.trace("Exiting reencryptEncryptedKeys method.");
+ return Response.ok().type(MediaType.APPLICATION_JSON).entity(retJSON)
+ .build();
+ } catch (Exception e) {
+ LOG.debug("Exception in reencryptEncryptedKeys.", e);
+ throw e;
+ }
+ }
+
+ @SuppressWarnings("rawtypes")
+ @POST
@Path(KMSRESTConstants.KEY_VERSION_RESOURCE + "/{versionName:.*}/" +
KMSRESTConstants.EEK_SUB_RESOURCE)
@Produces(MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8)
@@ -548,8 +614,8 @@ public class KMS {
try {
LOG.trace("Entering decryptEncryptedKey method.");
UserGroupInformation user = HttpUserGroupInformation.get();
- KMSClientProvider.checkNotEmpty(versionName, "versionName");
- KMSClientProvider.checkNotNull(eekOp, "eekOp");
+ checkNotEmpty(versionName, "versionName");
+ checkNotNull(eekOp, "eekOp");
LOG.debug("Decrypting key for {}, the edek Operation is {}.",
versionName, eekOp);
@@ -558,13 +624,14 @@ public class KMS {
String ivStr = (String) jsonPayload.get(KMSRESTConstants.IV_FIELD);
String encMaterialStr =
(String) jsonPayload.get(KMSRESTConstants.MATERIAL_FIELD);
- KMSClientProvider.checkNotNull(ivStr, KMSRESTConstants.IV_FIELD);
+ checkNotNull(ivStr, KMSRESTConstants.IV_FIELD);
final byte[] iv = Base64.decodeBase64(ivStr);
- KMSClientProvider.checkNotNull(encMaterialStr,
+ checkNotNull(encMaterialStr,
KMSRESTConstants.MATERIAL_FIELD);
final byte[] encMaterial = Base64.decodeBase64(encMaterialStr);
Object retJSON;
if (eekOp.equals(KMSRESTConstants.EEK_DECRYPT)) {
+ KMSWebApp.getDecryptEEKCallsMeter().mark();
assertAccess(KMSACLs.Type.DECRYPT_EEK, user, KMSOp.DECRYPT_EEK,
keyName);
@@ -582,9 +649,10 @@ public class KMS {
}
);
- retJSON = KMSServerJSONUtils.toJSON(retKeyVersion);
+ retJSON = KMSUtil.toJSON(retKeyVersion);
kmsAudit.ok(user, KMSOp.DECRYPT_EEK, keyName, "");
} else if (eekOp.equals(KMSRESTConstants.EEK_REENCRYPT)) {
+ KMSWebApp.getReencryptEEKCallsMeter().mark();
assertAccess(KMSACLs.Type.GENERATE_EEK, user, KMSOp.REENCRYPT_EEK,
keyName);
@@ -599,7 +667,7 @@ public class KMS {
}
});
- retJSON = KMSServerJSONUtils.toJSON(retEncryptedKeyVersion);
+ retJSON = KMSUtil.toJSON(retEncryptedKeyVersion);
kmsAudit.ok(user, KMSOp.REENCRYPT_EEK, keyName, "");
} else {
StringBuilder error;
@@ -612,7 +680,6 @@ public class KMS {
LOG.error(error.toString());
throw new IllegalArgumentException(error.toString());
}
- KMSWebApp.getDecryptEEKCallsMeter().mark();
LOG.trace("Exiting handleEncryptedKeyOp method.");
return Response.ok().type(MediaType.APPLICATION_JSON).entity(retJSON)
.build();
@@ -631,7 +698,7 @@ public class KMS {
try {
LOG.trace("Entering getKeyVersions method.");
UserGroupInformation user = HttpUserGroupInformation.get();
- KMSClientProvider.checkNotEmpty(name, "name");
+ checkNotEmpty(name, "name");
KMSWebApp.getKeyCallsMeter().mark();
assertAccess(KMSACLs.Type.GET, user, KMSOp.GET_KEY_VERSIONS, name);
LOG.debug("Getting key versions for key {}", name);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4ec5acc7/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSJSONReader.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSJSONReader.java b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSJSONReader.java
index 14a5451..f6f670b 100644
--- a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSJSONReader.java
+++ b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSJSONReader.java
@@ -31,21 +31,23 @@ import java.io.IOException;
import java.io.InputStream;
import java.lang.annotation.Annotation;
import java.lang.reflect.Type;
+import java.util.List;
import java.util.Map;
@Provider
@Consumes(MediaType.APPLICATION_JSON)
@InterfaceAudience.Private
-public class KMSJSONReader implements MessageBodyReader<Map> {
+public class KMSJSONReader implements MessageBodyReader<Object> {
@Override
public boolean isReadable(Class<?> type, Type genericType,
Annotation[] annotations, MediaType mediaType) {
- return type.isAssignableFrom(Map.class);
+ return type.isAssignableFrom(Map.class) || type
+ .isAssignableFrom(List.class);
}
@Override
- public Map readFrom(Class<Map> type, Type genericType,
+ public Object readFrom(Class<Object> type, Type genericType,
Annotation[] annotations, MediaType mediaType,
MultivaluedMap<String, String> httpHeaders, InputStream entityStream)
throws IOException, WebApplicationException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4ec5acc7/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSServerJSONUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSServerJSONUtils.java b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSServerJSONUtils.java
index 24af81b..d7e6e46 100644
--- a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSServerJSONUtils.java
+++ b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSServerJSONUtils.java
@@ -17,11 +17,10 @@
*/
package org.apache.hadoop.crypto.key.kms.server;
-import org.apache.commons.codec.binary.Base64;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.crypto.key.KeyProvider;
-import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension.EncryptedKeyVersion;
import org.apache.hadoop.crypto.key.kms.KMSRESTConstants;
+import org.apache.hadoop.util.KMSUtil;
import java.util.ArrayList;
import java.util.LinkedHashMap;
@@ -33,48 +32,19 @@ import java.util.Map;
*/
@InterfaceAudience.Private
public class KMSServerJSONUtils {
- @SuppressWarnings("unchecked")
- public static Map toJSON(KeyProvider.KeyVersion keyVersion) {
- Map json = new LinkedHashMap();
- if (keyVersion != null) {
- json.put(KMSRESTConstants.NAME_FIELD,
- keyVersion.getName());
- json.put(KMSRESTConstants.VERSION_NAME_FIELD,
- keyVersion.getVersionName());
- json.put(KMSRESTConstants.MATERIAL_FIELD,
- Base64.encodeBase64URLSafeString(
- keyVersion.getMaterial()));
- }
- return json;
- }
@SuppressWarnings("unchecked")
public static List toJSON(List<KeyProvider.KeyVersion> keyVersions) {
List json = new ArrayList();
if (keyVersions != null) {
for (KeyProvider.KeyVersion version : keyVersions) {
- json.add(toJSON(version));
+ json.add(KMSUtil.toJSON(version));
}
}
return json;
}
@SuppressWarnings("unchecked")
- public static Map toJSON(EncryptedKeyVersion encryptedKeyVersion) {
- Map json = new LinkedHashMap();
- if (encryptedKeyVersion != null) {
- json.put(KMSRESTConstants.VERSION_NAME_FIELD,
- encryptedKeyVersion.getEncryptionKeyVersionName());
- json.put(KMSRESTConstants.IV_FIELD,
- Base64.encodeBase64URLSafeString(
- encryptedKeyVersion.getEncryptedKeyIv()));
- json.put(KMSRESTConstants.ENCRYPTED_KEY_VERSION_FIELD,
- toJSON(encryptedKeyVersion.getEncryptedKeyVersion()));
- }
- return json;
- }
-
- @SuppressWarnings("unchecked")
public static Map toJSON(String keyName, KeyProvider.Metadata meta) {
Map json = new LinkedHashMap();
if (meta != null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4ec5acc7/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSWebApp.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSWebApp.java b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSWebApp.java
index ac24105..9a71fa2 100644
--- a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSWebApp.java
+++ b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KMSWebApp.java
@@ -60,6 +60,10 @@ public class KMSWebApp implements ServletContextListener {
"generate_eek.calls.meter";
private static final String DECRYPT_EEK_METER = METRICS_PREFIX +
"decrypt_eek.calls.meter";
+ private static final String REENCRYPT_EEK_METER = METRICS_PREFIX +
+ "reencrypt_eek.calls.meter";
+ private static final String REENCRYPT_EEK_BATCH_METER = METRICS_PREFIX +
+ "reencrypt_eek_batch.calls.meter";
private static Logger LOG;
private static MetricRegistry metricRegistry;
@@ -72,6 +76,8 @@ public class KMSWebApp implements ServletContextListener {
private static Meter unauthorizedCallsMeter;
private static Meter unauthenticatedCallsMeter;
private static Meter decryptEEKCallsMeter;
+ private static Meter reencryptEEKCallsMeter;
+ private static Meter reencryptEEKBatchCallsMeter;
private static Meter generateEEKCallsMeter;
private static Meter invalidCallsMeter;
private static KMSAudit kmsAudit;
@@ -131,6 +137,10 @@ public class KMSWebApp implements ServletContextListener {
new Meter());
decryptEEKCallsMeter = metricRegistry.register(DECRYPT_EEK_METER,
new Meter());
+ reencryptEEKCallsMeter = metricRegistry.register(REENCRYPT_EEK_METER,
+ new Meter());
+ reencryptEEKBatchCallsMeter = metricRegistry.register(
+ REENCRYPT_EEK_BATCH_METER, new Meter());
adminCallsMeter = metricRegistry.register(ADMIN_CALLS_METER, new Meter());
keyCallsMeter = metricRegistry.register(KEY_CALLS_METER, new Meter());
invalidCallsMeter = metricRegistry.register(INVALID_CALLS_METER,
@@ -239,6 +249,14 @@ public class KMSWebApp implements ServletContextListener {
return decryptEEKCallsMeter;
}
+ public static Meter getReencryptEEKCallsMeter() {
+ return reencryptEEKCallsMeter;
+ }
+
+ public static Meter getReencryptEEKBatchCallsMeter() {
+ return reencryptEEKBatchCallsMeter;
+ }
+
public static Meter getUnauthorizedCallsMeter() {
return unauthorizedCallsMeter;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4ec5acc7/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KeyAuthorizationKeyProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KeyAuthorizationKeyProvider.java b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KeyAuthorizationKeyProvider.java
index d53cdea..868db76 100644
--- a/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KeyAuthorizationKeyProvider.java
+++ b/hadoop-common-project/hadoop-kms/src/main/java/org/apache/hadoop/crypto/key/kms/server/KeyAuthorizationKeyProvider.java
@@ -289,6 +289,25 @@ public class KeyAuthorizationKeyProvider extends KeyProviderCryptoExtension {
}
@Override
+ public void reencryptEncryptedKeys(List<EncryptedKeyVersion> ekvs)
+ throws IOException, GeneralSecurityException {
+ if (ekvs.isEmpty()) {
+ return;
+ }
+ readLock.lock();
+ try {
+ for (EncryptedKeyVersion ekv : ekvs) {
+ verifyKeyVersionBelongsToKey(ekv);
+ }
+ final String keyName = ekvs.get(0).getEncryptionKeyName();
+ doAccessCheck(keyName, KeyOpType.GENERATE_EEK);
+ provider.reencryptEncryptedKeys(ekvs);
+ } finally {
+ readLock.unlock();
+ }
+ }
+
+ @Override
public KeyVersion getKeyVersion(String versionName) throws IOException {
readLock.lock();
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4ec5acc7/hadoop-common-project/hadoop-kms/src/site/markdown/index.md.vm
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/site/markdown/index.md.vm b/hadoop-common-project/hadoop-kms/src/site/markdown/index.md.vm
index 4573b06..1dd89e9 100644
--- a/hadoop-common-project/hadoop-kms/src/site/markdown/index.md.vm
+++ b/hadoop-common-project/hadoop-kms/src/site/markdown/index.md.vm
@@ -506,7 +506,7 @@ $H5 Key ACLs
KMS supports access control for all non-read operations at the Key level. All Key Access operations are classified as :
* MANAGEMENT - createKey, deleteKey, rolloverNewVersion
-* GENERATE_EEK - generateEncryptedKey, reencryptEncryptedKey, warmUpEncryptedKeys
+* GENERATE_EEK - generateEncryptedKey, reencryptEncryptedKey, reencryptEncryptedKeys, warmUpEncryptedKeys
* DECRYPT_EEK - decryptEncryptedKey
* READ - getKeyVersion, getKeyVersions, getMetadata, getKeysMetadata, getCurrentKey
* ALL - all of the above
@@ -983,6 +983,64 @@ This is usually useful after a [Rollover](#Rollover_Key) of an encryption key. R
}
}
+$H4 Batch Re-encrypt Encrypted Keys With The Latest KeyVersion
+
+Batched version of the above re-encrypt Encrypted Key. This command takes a list of previously generated encrypted key, and re-encrypts them using the latest KeyVersion encryption key in the KeyProvider, and return the re-encrypted encrypted keys in the same sequence. For each encrypted key, if the latest KeyVersion is the same as the one used to generate the encrypted key, no action is taken and the same encrypted key is returned.
+
+This is usually useful after a [Rollover](#Rollover_Key) of an encryption key. Re-encrypting the encrypted key will allow it to be encrypted using the latest version of the encryption key, but still with the same key material and initialization vector.
+
+All Encrypted keys for a batch request must be under the same encryption key name, but could be potentially under different versions of the encryption key.
+
+*REQUEST:*
+
+ POST http://HOST:PORT/kms/v1/key/<key-name>/_reencryptbatch
+ Content-Type: application/json
+
+ [
+ {
+ "versionName" : "<encryptionVersionName>",
+ "iv" : "<iv>", //base64
+ "encryptedKeyVersion" : {
+ "versionName" : "EEK",
+ "material" : "<material>", //base64
+ }
+ },
+ {
+ "versionName" : "<encryptionVersionName>",
+ "iv" : "<iv>", //base64
+ "encryptedKeyVersion" : {
+ "versionName" : "EEK",
+ "material" : "<material>", //base64
+ }
+ },
+ ...
+ ]
+
+*RESPONSE:*
+
+ 200 OK
+ Content-Type: application/json
+
+ [
+ {
+ "versionName" : "<encryptionVersionName>",
+ "iv" : "<iv>", //base64
+ "encryptedKeyVersion" : {
+ "versionName" : "EEK",
+ "material" : "<material>", //base64
+ }
+ },
+ {
+ "versionName" : "<encryptionVersionName>",
+ "iv" : "<iv>", //base64
+ "encryptedKeyVersion" : {
+ "versionName" : "EEK",
+ "material" : "<material>", //base64
+ }
+ },
+ ...
+ ]
+
$H4 Get Key Version
*REQUEST:*
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4ec5acc7/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
index a45906a..d6f2c25 100644
--- a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
+++ b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMS.java
@@ -97,6 +97,7 @@ import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.fail;
import static org.mockito.Mockito.when;
public class TestKMS {
@@ -722,6 +723,22 @@ public class TestKMS {
assertArrayEquals(k1.getMaterial(), k1r.getMaterial());
assertEquals(kv.getMaterial().length, k1r.getMaterial().length);
+ // test re-encrypt batch
+ EncryptedKeyVersion ek3 = kpExt.generateEncryptedKey(kv.getName());
+ KeyVersion latest = kpExt.rollNewVersion(kv.getName());
+ List<EncryptedKeyVersion> ekvs = new ArrayList<>(3);
+ ekvs.add(ek1);
+ ekvs.add(ek2);
+ ekvs.add(ek3);
+ ekvs.add(ek1);
+ ekvs.add(ek2);
+ ekvs.add(ek3);
+ kpExt.reencryptEncryptedKeys(ekvs);
+ for (EncryptedKeyVersion ekv: ekvs) {
+ assertEquals(latest.getVersionName(),
+ ekv.getEncryptionKeyVersionName());
+ }
+
// deleteKey()
kp.deleteKey("k1");
@@ -1134,6 +1151,10 @@ public class TestKMS {
KeyProviderCryptoExtension.createKeyProviderCryptoExtension(kp);
EncryptedKeyVersion ekv = kpce.generateEncryptedKey("k1");
kpce.reencryptEncryptedKey(ekv);
+ List<EncryptedKeyVersion> ekvs = new ArrayList<>(2);
+ ekvs.add(ekv);
+ ekvs.add(ekv);
+ kpce.reencryptEncryptedKeys(ekvs);
return null;
}
});
@@ -1563,6 +1584,10 @@ public class TestKMS {
KeyProviderCryptoExtension kpCE = KeyProviderCryptoExtension.
createKeyProviderCryptoExtension(kp);
kpCE.reencryptEncryptedKey(encKv);
+ List<EncryptedKeyVersion> ekvs = new ArrayList<>(2);
+ ekvs.add(encKv);
+ ekvs.add(encKv);
+ kpCE.reencryptEncryptedKeys(ekvs);
return null;
}
});
@@ -1669,8 +1694,27 @@ public class TestKMS {
KeyProviderCryptoExtension kpCE = KeyProviderCryptoExtension.
createKeyProviderCryptoExtension(kp);
kpCE.reencryptEncryptedKey(encKv);
+ fail("Should not have been able to reencryptEncryptedKey");
+ } catch (AuthorizationException ex) {
+ LOG.info("reencryptEncryptedKey caught expected exception.", ex);
+ }
+ return null;
+ }
+ });
+ doAs("GENERATE_EEK", new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws Exception {
+ KeyProvider kp = createProvider(uri, conf);
+ try {
+ KeyProviderCryptoExtension kpCE = KeyProviderCryptoExtension.
+ createKeyProviderCryptoExtension(kp);
+ List<EncryptedKeyVersion> ekvs = new ArrayList<>(2);
+ ekvs.add(encKv);
+ ekvs.add(encKv);
+ kpCE.reencryptEncryptedKeys(ekvs);
+ fail("Should not have been able to reencryptEncryptedKeys");
} catch (AuthorizationException ex) {
- LOG.info("Caught expected exception.", ex);
+ LOG.info("reencryptEncryptedKeys caught expected exception.", ex);
}
return null;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4ec5acc7/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMSAudit.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMSAudit.java b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMSAudit.java
index bd4ddbd..2e01f98 100644
--- a/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMSAudit.java
+++ b/hadoop-common-project/hadoop-kms/src/test/java/org/apache/hadoop/crypto/key/kms/server/TestKMSAudit.java
@@ -115,6 +115,9 @@ public class TestKMSAudit {
kmsAudit.ok(luser, KMSOp.REENCRYPT_EEK, "k1", "testmsg");
kmsAudit.ok(luser, KMSOp.REENCRYPT_EEK, "k1", "testmsg");
kmsAudit.evictCacheForTesting();
+ kmsAudit.ok(luser, KMSOp.REENCRYPT_EEK_BATCH, "k1", "testmsg");
+ kmsAudit.ok(luser, KMSOp.REENCRYPT_EEK_BATCH, "k1", "testmsg");
+ kmsAudit.evictCacheForTesting();
String out = getAndResetLogOutput();
System.out.println(out);
Assert.assertTrue(
@@ -128,7 +131,9 @@ public class TestKMSAudit {
+ "OK\\[op=DECRYPT_EEK, key=k1, user=luser, accessCount=6, interval=[^m]{1,4}ms\\] testmsg"
+ "OK\\[op=DECRYPT_EEK, key=k1, user=luser, accessCount=1, interval=[^m]{1,4}ms\\] testmsg"
+ "OK\\[op=REENCRYPT_EEK, key=k1, user=luser, accessCount=1, interval=[^m]{1,4}ms\\] testmsg"
- + "OK\\[op=REENCRYPT_EEK, key=k1, user=luser, accessCount=3, interval=[^m]{1,4}ms\\] testmsg"));
+ + "OK\\[op=REENCRYPT_EEK, key=k1, user=luser, accessCount=3, interval=[^m]{1,4}ms\\] testmsg"
+ + "OK\\[op=REENCRYPT_EEK_BATCH, key=k1, user=luser\\] testmsg"
+ + "OK\\[op=REENCRYPT_EEK_BATCH, key=k1, user=luser\\] testmsg"));
}
@Test
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[39/50] [abbrv] hadoop git commit: HADOOP-14687. AuthenticatedURL
will reuse bad/expired session cookies. Contributed by Daryn Sharp
Posted by as...@apache.org.
HADOOP-14687. AuthenticatedURL will reuse bad/expired session cookies. Contributed by Daryn Sharp
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c3793102
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c3793102
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c3793102
Branch: refs/heads/YARN-5972
Commit: c3793102121767c46091805eae65ef3919a5f368
Parents: 657dd59
Author: Jason Lowe <jl...@apache.org>
Authored: Tue Aug 22 16:50:01 2017 -0500
Committer: Jason Lowe <jl...@apache.org>
Committed: Tue Aug 22 16:50:01 2017 -0500
----------------------------------------------------------------------
.../authentication/client/AuthenticatedURL.java | 184 +++++++++++---
.../client/KerberosAuthenticator.java | 30 +--
.../client/PseudoAuthenticator.java | 5 +-
.../crypto/key/kms/KMSClientProvider.java | 13 -
.../hadoop/http/TestHttpServerWithSpengo.java | 242 ++++++++++++++++++-
5 files changed, 403 insertions(+), 71 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c3793102/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/AuthenticatedURL.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/AuthenticatedURL.java b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/AuthenticatedURL.java
index 5696ba0..1093d8a 100644
--- a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/AuthenticatedURL.java
+++ b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/AuthenticatedURL.java
@@ -19,8 +19,14 @@ import org.slf4j.LoggerFactory;
import java.io.FileNotFoundException;
import java.io.IOException;
+import java.net.CookieHandler;
+import java.net.HttpCookie;
import java.net.HttpURLConnection;
+import java.net.URI;
import java.net.URL;
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -69,14 +75,99 @@ public class AuthenticatedURL {
*/
public static final String AUTH_COOKIE = "hadoop.auth";
- private static final String AUTH_COOKIE_EQ = AUTH_COOKIE + "=";
+ // a lightweight cookie handler that will be attached to url connections.
+ // client code is not required to extract or inject auth cookies.
+ private static class AuthCookieHandler extends CookieHandler {
+ private HttpCookie authCookie;
+ private Map<String, List<String>> cookieHeaders = Collections.emptyMap();
+
+ @Override
+ public synchronized Map<String, List<String>> get(URI uri,
+ Map<String, List<String>> requestHeaders) throws IOException {
+ // call getter so it will reset headers if token is expiring.
+ getAuthCookie();
+ return cookieHeaders;
+ }
+
+ @Override
+ public void put(URI uri, Map<String, List<String>> responseHeaders) {
+ List<String> headers = responseHeaders.get("Set-Cookie");
+ if (headers != null) {
+ for (String header : headers) {
+ List<HttpCookie> cookies;
+ try {
+ cookies = HttpCookie.parse(header);
+ } catch (IllegalArgumentException iae) {
+ // don't care. just skip malformed cookie headers.
+ LOG.debug("Cannot parse cookie header: " + header, iae);
+ continue;
+ }
+ for (HttpCookie cookie : cookies) {
+ if (AUTH_COOKIE.equals(cookie.getName())) {
+ setAuthCookie(cookie);
+ }
+ }
+ }
+ }
+ }
+
+ // return the auth cookie if still valid.
+ private synchronized HttpCookie getAuthCookie() {
+ if (authCookie != null && authCookie.hasExpired()) {
+ setAuthCookie(null);
+ }
+ return authCookie;
+ }
+
+ private synchronized void setAuthCookie(HttpCookie cookie) {
+ final HttpCookie oldCookie = authCookie;
+ // will redefine if new cookie is valid.
+ authCookie = null;
+ cookieHeaders = Collections.emptyMap();
+ boolean valid = cookie != null && !cookie.getValue().isEmpty() &&
+ !cookie.hasExpired();
+ if (valid) {
+ // decrease lifetime to avoid using a cookie soon to expire.
+ // allows authenticators to pre-emptively reauthenticate to
+ // prevent clients unnecessarily receiving a 401.
+ long maxAge = cookie.getMaxAge();
+ if (maxAge != -1) {
+ cookie.setMaxAge(maxAge * 9/10);
+ valid = !cookie.hasExpired();
+ }
+ }
+ if (valid) {
+ // v0 cookies value aren't quoted by default but tomcat demands
+ // quoting.
+ if (cookie.getVersion() == 0) {
+ String value = cookie.getValue();
+ if (!value.startsWith("\"")) {
+ value = "\"" + value + "\"";
+ cookie.setValue(value);
+ }
+ }
+ authCookie = cookie;
+ cookieHeaders = new HashMap<>();
+ cookieHeaders.put("Cookie", Arrays.asList(cookie.toString()));
+ }
+ LOG.trace("Setting token value to {} ({})", authCookie, oldCookie);
+ }
+
+ private void setAuthCookieValue(String value) {
+ HttpCookie c = null;
+ if (value != null) {
+ c = new HttpCookie(AUTH_COOKIE, value);
+ }
+ setAuthCookie(c);
+ }
+ }
/**
* Client side authentication token.
*/
public static class Token {
- private String token;
+ private final AuthCookieHandler cookieHandler = new AuthCookieHandler();
/**
* Creates a token.
@@ -102,7 +193,7 @@ public class AuthenticatedURL {
* @return if a token from the server has been set.
*/
public boolean isSet() {
- return token != null;
+ return cookieHandler.getAuthCookie() != null;
}
/**
@@ -111,7 +202,36 @@ public class AuthenticatedURL {
* @param tokenStr string representation of the tokenStr.
*/
void set(String tokenStr) {
- token = tokenStr;
+ cookieHandler.setAuthCookieValue(tokenStr);
+ }
+
+ /**
+ * Installs a cookie handler for the http request to manage session
+ * cookies.
+ * @param url
+ * @return HttpUrlConnection
+ * @throws IOException
+ */
+ HttpURLConnection openConnection(URL url,
+ ConnectionConfigurator connConfigurator) throws IOException {
+ // the cookie handler is unfortunately a global static. it's a
+ // synchronized class method so we can safely swap the handler while
+ // instantiating the connection object to prevent it leaking into
+ // other connections.
+ final HttpURLConnection conn;
+ synchronized(CookieHandler.class) {
+ CookieHandler current = CookieHandler.getDefault();
+ CookieHandler.setDefault(cookieHandler);
+ try {
+ conn = (HttpURLConnection)url.openConnection();
+ } finally {
+ CookieHandler.setDefault(current);
+ }
+ }
+ if (connConfigurator != null) {
+ connConfigurator.configure(conn);
+ }
+ return conn;
}
/**
@@ -121,7 +241,15 @@ public class AuthenticatedURL {
*/
@Override
public String toString() {
- return token;
+ String value = "";
+ HttpCookie authCookie = cookieHandler.getAuthCookie();
+ if (authCookie != null) {
+ value = authCookie.getValue();
+ if (value.startsWith("\"")) { // tests don't want the quotes.
+ value = value.substring(1, value.length()-1);
+ }
+ }
+ return value;
}
}
@@ -218,27 +346,25 @@ public class AuthenticatedURL {
throw new IllegalArgumentException("token cannot be NULL");
}
authenticator.authenticate(url, token);
- HttpURLConnection conn = (HttpURLConnection) url.openConnection();
- if (connConfigurator != null) {
- conn = connConfigurator.configure(conn);
- }
- injectToken(conn, token);
- return conn;
+
+ // allow the token to create the connection with a cookie handler for
+ // managing session cookies.
+ return token.openConnection(url, connConfigurator);
}
/**
- * Helper method that injects an authentication token to send with a connection.
+ * Helper method that injects an authentication token to send with a
+ * connection. Callers should prefer using
+ * {@link Token#openConnection(URL, ConnectionConfigurator)} which
+ * automatically manages authentication tokens.
*
* @param conn connection to inject the authentication token into.
* @param token authentication token to inject.
*/
public static void injectToken(HttpURLConnection conn, Token token) {
- String t = token.token;
- if (t != null) {
- if (!t.startsWith("\"")) {
- t = "\"" + t + "\"";
- }
- conn.addRequestProperty("Cookie", AUTH_COOKIE_EQ + t);
+ HttpCookie authCookie = token.cookieHandler.getAuthCookie();
+ if (authCookie != null) {
+ conn.addRequestProperty("Cookie", authCookie.toString());
}
}
@@ -258,24 +384,10 @@ public class AuthenticatedURL {
if (respCode == HttpURLConnection.HTTP_OK
|| respCode == HttpURLConnection.HTTP_CREATED
|| respCode == HttpURLConnection.HTTP_ACCEPTED) {
- Map<String, List<String>> headers = conn.getHeaderFields();
- List<String> cookies = headers.get("Set-Cookie");
- if (cookies != null) {
- for (String cookie : cookies) {
- if (cookie.startsWith(AUTH_COOKIE_EQ)) {
- String value = cookie.substring(AUTH_COOKIE_EQ.length());
- int separator = value.indexOf(";");
- if (separator > -1) {
- value = value.substring(0, separator);
- }
- if (value.length() > 0) {
- LOG.trace("Setting token value to {} ({}), resp={}", value,
- token, respCode);
- token.set(value);
- }
- }
- }
- }
+ // cookie handler should have already extracted the token. try again
+ // for backwards compatibility if this method is called on a connection
+ // not opened via this instance.
+ token.cookieHandler.put(null, conn.getHeaderFields());
} else if (respCode == HttpURLConnection.HTTP_NOT_FOUND) {
LOG.trace("Setting token value to null ({}), resp={}", token, respCode);
token.set(null);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c3793102/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/KerberosAuthenticator.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/KerberosAuthenticator.java b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/KerberosAuthenticator.java
index 9bcebc3..942d13c 100644
--- a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/KerberosAuthenticator.java
+++ b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/KerberosAuthenticator.java
@@ -147,7 +147,6 @@ public class KerberosAuthenticator implements Authenticator {
}
private URL url;
- private HttpURLConnection conn;
private Base64 base64;
private ConnectionConfigurator connConfigurator;
@@ -182,10 +181,7 @@ public class KerberosAuthenticator implements Authenticator {
if (!token.isSet()) {
this.url = url;
base64 = new Base64(0);
- conn = (HttpURLConnection) url.openConnection();
- if (connConfigurator != null) {
- conn = connConfigurator.configure(conn);
- }
+ HttpURLConnection conn = token.openConnection(url, connConfigurator);
conn.setRequestMethod(AUTH_HTTP_METHOD);
conn.connect();
@@ -200,7 +196,7 @@ public class KerberosAuthenticator implements Authenticator {
}
needFallback = true;
}
- if (!needFallback && isNegotiate()) {
+ if (!needFallback && isNegotiate(conn)) {
LOG.debug("Performing our own SPNEGO sequence.");
doSpnegoSequence(token);
} else {
@@ -249,7 +245,7 @@ public class KerberosAuthenticator implements Authenticator {
/*
* Indicates if the response is starting a SPNEGO negotiation.
*/
- private boolean isNegotiate() throws IOException {
+ private boolean isNegotiate(HttpURLConnection conn) throws IOException {
boolean negotiate = false;
if (conn.getResponseCode() == HttpURLConnection.HTTP_UNAUTHORIZED) {
String authHeader = conn.getHeaderField(WWW_AUTHENTICATE);
@@ -267,7 +263,8 @@ public class KerberosAuthenticator implements Authenticator {
* @throws IOException if an IO error occurred.
* @throws AuthenticationException if an authentication error occurred.
*/
- private void doSpnegoSequence(AuthenticatedURL.Token token) throws IOException, AuthenticationException {
+ private void doSpnegoSequence(final AuthenticatedURL.Token token)
+ throws IOException, AuthenticationException {
try {
AccessControlContext context = AccessController.getContext();
Subject subject = Subject.getSubject(context);
@@ -308,13 +305,15 @@ public class KerberosAuthenticator implements Authenticator {
// Loop while the context is still not established
while (!established) {
+ HttpURLConnection conn =
+ token.openConnection(url, connConfigurator);
outToken = gssContext.initSecContext(inToken, 0, inToken.length);
if (outToken != null) {
- sendToken(outToken);
+ sendToken(conn, outToken);
}
if (!gssContext.isEstablished()) {
- inToken = readToken();
+ inToken = readToken(conn);
} else {
established = true;
}
@@ -337,18 +336,14 @@ public class KerberosAuthenticator implements Authenticator {
} catch (LoginException ex) {
throw new AuthenticationException(ex);
}
- AuthenticatedURL.extractToken(conn, token);
}
/*
* Sends the Kerberos token to the server.
*/
- private void sendToken(byte[] outToken) throws IOException {
+ private void sendToken(HttpURLConnection conn, byte[] outToken)
+ throws IOException {
String token = base64.encodeToString(outToken);
- conn = (HttpURLConnection) url.openConnection();
- if (connConfigurator != null) {
- conn = connConfigurator.configure(conn);
- }
conn.setRequestMethod(AUTH_HTTP_METHOD);
conn.setRequestProperty(AUTHORIZATION, NEGOTIATE + " " + token);
conn.connect();
@@ -357,7 +352,8 @@ public class KerberosAuthenticator implements Authenticator {
/*
* Retrieves the Kerberos token returned by the server.
*/
- private byte[] readToken() throws IOException, AuthenticationException {
+ private byte[] readToken(HttpURLConnection conn)
+ throws IOException, AuthenticationException {
int status = conn.getResponseCode();
if (status == HttpURLConnection.HTTP_OK || status == HttpURLConnection.HTTP_UNAUTHORIZED) {
String authHeader = conn.getHeaderField(WWW_AUTHENTICATE);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c3793102/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/PseudoAuthenticator.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/PseudoAuthenticator.java b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/PseudoAuthenticator.java
index 29ca9cf..66c6252 100644
--- a/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/PseudoAuthenticator.java
+++ b/hadoop-common-project/hadoop-auth/src/main/java/org/apache/hadoop/security/authentication/client/PseudoAuthenticator.java
@@ -68,10 +68,7 @@ public class PseudoAuthenticator implements Authenticator {
String paramSeparator = (strUrl.contains("?")) ? "&" : "?";
strUrl += paramSeparator + USER_NAME_EQ + getUserName();
url = new URL(strUrl);
- HttpURLConnection conn = (HttpURLConnection) url.openConnection();
- if (connConfigurator != null) {
- conn = connConfigurator.configure(conn);
- }
+ HttpURLConnection conn = token.openConnection(url, connConfigurator);
conn.setRequestMethod("OPTIONS");
conn.connect();
AuthenticatedURL.extractToken(conn, token);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c3793102/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
index af0dd82..9bef32c 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/crypto/key/kms/KMSClientProvider.java
@@ -32,8 +32,6 @@ import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.ProviderUtils;
import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.authentication.client.AuthenticatedURL;
-import org.apache.hadoop.security.authentication.client.AuthenticationException;
import org.apache.hadoop.security.authentication.client.ConnectionConfigurator;
import org.apache.hadoop.security.ssl.SSLFactory;
import org.apache.hadoop.security.token.Token;
@@ -522,17 +520,6 @@ public class KMSClientProvider extends KeyProvider implements CryptoExtension,
authRetryCount - 1);
}
}
- try {
- AuthenticatedURL.extractToken(conn, authToken);
- if (LOG.isDebugEnabled()) {
- LOG.debug("Extracted token, authToken={}, its dt={}", authToken,
- authToken.getDelegationToken());
- }
- } catch (AuthenticationException e) {
- // Ignore the AuthExceptions.. since we are just using the method to
- // extract and set the authToken.. (Workaround till we actually fix
- // AuthenticatedURL properly to set authToken post initialization)
- }
HttpExceptionUtils.validateResponse(conn, expectedResponse);
if (conn.getContentType() != null
&& conn.getContentType().trim().toLowerCase()
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c3793102/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServerWithSpengo.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServerWithSpengo.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServerWithSpengo.java
index 5239ed6..8f5dd04 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServerWithSpengo.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServerWithSpengo.java
@@ -22,9 +22,12 @@ import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.minikdc.MiniKdc;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.AuthenticationFilterInitializer;
+import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
import org.apache.hadoop.security.authentication.KerberosTestUtils;
import org.apache.hadoop.security.authentication.client.AuthenticatedURL;
+import org.apache.hadoop.security.authentication.client.AuthenticationException;
import org.apache.hadoop.security.authentication.server.AuthenticationFilter;
import org.apache.hadoop.security.authentication.server.AuthenticationToken;
import org.apache.hadoop.security.authentication.util.Signer;
@@ -32,6 +35,7 @@ import org.apache.hadoop.security.authentication.util.SignerSecretProvider;
import org.apache.hadoop.security.authentication.util.StringSignerSecretProviderCreator;
import org.apache.hadoop.security.authorize.AccessControlList;
import org.apache.hadoop.security.authorize.ProxyUsers;
+import org.ietf.jgss.GSSException;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -45,7 +49,14 @@ import java.io.Writer;
import java.net.HttpURLConnection;
import java.net.URI;
import java.net.URL;
+import java.security.AccessController;
+import java.security.PrivilegedExceptionAction;
+import java.util.HashSet;
import java.util.Properties;
+import java.util.Set;
+import javax.security.auth.Subject;
+import javax.servlet.ServletContext;
+
import static org.junit.Assert.assertTrue;
/**
@@ -72,16 +83,25 @@ public class TestHttpServerWithSpengo {
private static MiniKdc testMiniKDC;
private static File secretFile = new File(testRootDir, SECRET_STR);
+ private static UserGroupInformation authUgi;
+
@BeforeClass
public static void setUp() throws Exception {
try {
testMiniKDC = new MiniKdc(MiniKdc.createConf(), testRootDir);
testMiniKDC.start();
testMiniKDC.createPrincipal(
- httpSpnegoKeytabFile, HTTP_USER + "/localhost");
+ httpSpnegoKeytabFile, HTTP_USER + "/localhost", "keytab-user");
} catch (Exception e) {
assertTrue("Couldn't setup MiniKDC", false);
}
+
+ System.setProperty("sun.security.krb5.debug", "true");
+ Configuration conf = new Configuration();
+ SecurityUtil.setAuthenticationMethod(AuthenticationMethod.KERBEROS, conf);
+ UserGroupInformation.setConfiguration(conf);
+ authUgi = UserGroupInformation.loginUserFromKeytabAndReturnUGI(
+ "keytab-user", httpSpnegoKeytabFile.toString());
Writer w = new FileWriter(secretFile);
w.write("secret");
w.close();
@@ -209,6 +229,226 @@ public class TestHttpServerWithSpengo {
}
}
+ @Test
+ public void testSessionCookie() throws Exception {
+ Configuration conf = new Configuration();
+ conf.set(HttpServer2.FILTER_INITIALIZER_PROPERTY,
+ AuthenticationFilterInitializer.class.getName());
+ conf.set(PREFIX + "type", "kerberos");
+ conf.setBoolean(PREFIX + "simple.anonymous.allowed", false);
+ conf.set(PREFIX + "signer.secret.provider",
+ TestSignerSecretProvider.class.getName());
+
+ conf.set(PREFIX + "kerberos.keytab",
+ httpSpnegoKeytabFile.getAbsolutePath());
+ conf.set(PREFIX + "kerberos.principal", httpSpnegoPrincipal);
+ conf.set(PREFIX + "cookie.domain", realm);
+ conf.setBoolean(CommonConfigurationKeys.HADOOP_SECURITY_AUTHORIZATION,
+ true);
+
+ //setup logs dir
+ System.setProperty("hadoop.log.dir", testRootDir.getAbsolutePath());
+
+ HttpServer2 httpServer = null;
+ // Create http server to test.
+ httpServer = getCommonBuilder()
+ .setConf(conf)
+ .build();
+ httpServer.start();
+
+ // Get signer to encrypt token
+ final Signer signer = new Signer(new TestSignerSecretProvider());
+ final AuthenticatedURL authUrl = new AuthenticatedURL();
+
+ final URL url = new URL("http://" + NetUtils.getHostPortString(
+ httpServer.getConnectorAddress(0)) + "/conf");
+
+ // this illustrates an inconsistency with AuthenticatedURL. the
+ // authenticator is only called when the token is not set. if the
+ // authenticator fails then it must throw an AuthenticationException to
+ // the caller, yet the caller may see 401 for subsequent requests
+ // that require re-authentication like token expiration.
+ final UserGroupInformation simpleUgi =
+ UserGroupInformation.createRemoteUser("simple-user");
+
+ authUgi.doAs(new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws Exception {
+ TestSignerSecretProvider.rollSecret();
+ HttpURLConnection conn = null;
+ AuthenticatedURL.Token token = new AuthenticatedURL.Token();
+
+ // initial request should trigger authentication and set the token.
+ conn = authUrl.openConnection(url, token);
+ Assert.assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode());
+ Assert.assertTrue(token.isSet());
+ String cookie = token.toString();
+
+ // token should not change.
+ conn = authUrl.openConnection(url, token);
+ Assert.assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode());
+ Assert.assertTrue(token.isSet());
+ Assert.assertEquals(cookie, token.toString());
+
+ // roll secret to invalidate token.
+ TestSignerSecretProvider.rollSecret();
+ conn = authUrl.openConnection(url, token);
+ // this may or may not happen. under normal circumstances the
+ // jdk will silently renegotiate and the client never sees a 401.
+ // however in some cases the jdk will give up doing spnego. since
+ // the token is already set, the authenticator isn't invoked (which
+ // would do the spnego if the jdk doesn't), which causes the client
+ // to see a 401.
+ if (conn.getResponseCode() == HttpURLConnection.HTTP_UNAUTHORIZED) {
+ // if this happens, the token should be cleared which means the
+ // next request should succeed and receive a new token.
+ Assert.assertFalse(token.isSet());
+ conn = authUrl.openConnection(url, token);
+ }
+
+ // token should change.
+ Assert.assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode());
+ Assert.assertTrue(token.isSet());
+ Assert.assertNotEquals(cookie, token.toString());
+ cookie = token.toString();
+
+ // token should not change.
+ for (int i=0; i < 3; i++) {
+ conn = authUrl.openConnection(url, token);
+ Assert.assertEquals("attempt"+i,
+ HttpURLConnection.HTTP_OK, conn.getResponseCode());
+ Assert.assertTrue(token.isSet());
+ Assert.assertEquals(cookie, token.toString());
+ }
+
+ // blow out the kerberos creds test only auth token is used.
+ Subject s = Subject.getSubject(AccessController.getContext());
+ Set<Object> oldCreds = new HashSet<>(s.getPrivateCredentials());
+ s.getPrivateCredentials().clear();
+
+ // token should not change.
+ for (int i=0; i < 3; i++) {
+ try {
+ conn = authUrl.openConnection(url, token);
+ Assert.assertEquals("attempt"+i,
+ HttpURLConnection.HTTP_OK, conn.getResponseCode());
+ } catch (AuthenticationException ae) {
+ Assert.fail("attempt"+i+" "+ae);
+ }
+ Assert.assertTrue(token.isSet());
+ Assert.assertEquals(cookie, token.toString());
+ }
+
+ // invalidate token. connections should fail now and token should be
+ // unset.
+ TestSignerSecretProvider.rollSecret();
+ conn = authUrl.openConnection(url, token);
+ Assert.assertEquals(
+ HttpURLConnection.HTTP_UNAUTHORIZED, conn.getResponseCode());
+ Assert.assertFalse(token.isSet());
+ Assert.assertEquals("", token.toString());
+
+ // restore the kerberos creds, should work again.
+ s.getPrivateCredentials().addAll(oldCreds);
+ conn = authUrl.openConnection(url, token);
+ Assert.assertEquals(
+ HttpURLConnection.HTTP_OK, conn.getResponseCode());
+ Assert.assertTrue(token.isSet());
+ cookie = token.toString();
+
+ // token should not change.
+ for (int i=0; i < 3; i++) {
+ conn = authUrl.openConnection(url, token);
+ Assert.assertEquals("attempt"+i,
+ HttpURLConnection.HTTP_OK, conn.getResponseCode());
+ Assert.assertTrue(token.isSet());
+ Assert.assertEquals(cookie, token.toString());
+ }
+ return null;
+ }
+ });
+
+ simpleUgi.doAs(new PrivilegedExceptionAction<Void>() {
+ @Override
+ public Void run() throws Exception {
+ TestSignerSecretProvider.rollSecret();
+ AuthenticatedURL authUrl = new AuthenticatedURL();
+ AuthenticatedURL.Token token = new AuthenticatedURL.Token();
+ HttpURLConnection conn = null;
+
+ // initial connect with unset token will trigger authenticator which
+ // should fail since we have no creds and leave token unset.
+ try {
+ authUrl.openConnection(url, token);
+ Assert.fail("should fail with no credentials");
+ } catch (AuthenticationException ae) {
+ Assert.assertNotNull(ae.getCause());
+ Assert.assertEquals(GSSException.class, ae.getCause().getClass());
+ GSSException gsse = (GSSException)ae.getCause();
+ Assert.assertEquals(GSSException.NO_CRED, gsse.getMajor());
+ } catch (Throwable t) {
+ Assert.fail("Unexpected exception" + t);
+ }
+ Assert.assertFalse(token.isSet());
+
+ // create a valid token and save its value.
+ token = getEncryptedAuthToken(signer, "valid");
+ String cookie = token.toString();
+
+ // server should accept token. after the request the token should
+ // be set to the same value (ie. server didn't reissue cookie)
+ conn = authUrl.openConnection(url, token);
+ Assert.assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode());
+ Assert.assertTrue(token.isSet());
+ Assert.assertEquals(cookie, token.toString());
+
+ conn = authUrl.openConnection(url, token);
+ Assert.assertEquals(HttpURLConnection.HTTP_OK, conn.getResponseCode());
+ Assert.assertTrue(token.isSet());
+ Assert.assertEquals(cookie, token.toString());
+
+ // change the secret to effectively invalidate the cookie. see above
+ // regarding inconsistency. the authenticator has no way to know the
+ // token is bad, so the client will encounter a 401 instead of
+ // AuthenticationException.
+ TestSignerSecretProvider.rollSecret();
+ conn = authUrl.openConnection(url, token);
+ Assert.assertEquals(
+ HttpURLConnection.HTTP_UNAUTHORIZED, conn.getResponseCode());
+ Assert.assertFalse(token.isSet());
+ Assert.assertEquals("", token.toString());
+ return null;
+ }
+ });
+ }
+
+ public static class TestSignerSecretProvider extends SignerSecretProvider {
+ static int n = 0;
+ static byte[] secret;
+
+ static void rollSecret() {
+ secret = ("secret[" + (n++) + "]").getBytes();
+ }
+
+ public TestSignerSecretProvider() {
+ }
+
+ @Override
+ public void init(Properties config, ServletContext servletContext,
+ long tokenValidity) throws Exception {
+ rollSecret();
+ }
+
+ @Override
+ public byte[] getCurrentSecret() {
+ return secret;
+ }
+
+ @Override
+ public byte[][] getAllSecrets() {
+ return new byte[][]{secret};
+ }
+ }
private AuthenticatedURL.Token getEncryptedAuthToken(Signer signer,
String user) throws Exception {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[49/50] [abbrv] hadoop git commit: YARN-5292. NM Container lifecycle
and state transitions to support for PAUSED container state. (Hitesh Sharma
via asuresh)
Posted by as...@apache.org.
YARN-5292. NM Container lifecycle and state transitions to support for PAUSED container state. (Hitesh Sharma via asuresh)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/96423b50
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/96423b50
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/96423b50
Branch: refs/heads/YARN-5972
Commit: 96423b50e3c9f16fff8f5377859df562b95247be
Parents: 652dd43
Author: Arun Suresh <as...@apache.org>
Authored: Fri Dec 9 07:51:03 2016 -0800
Committer: Arun Suresh <as...@apache.org>
Committed: Thu Aug 24 12:05:42 2017 -0700
----------------------------------------------------------------------
.../hadoop/yarn/api/records/ContainerState.java | 7 +-
.../src/main/proto/yarn_protos.proto | 1 +
.../server/nodemanager/ContainerExecutor.java | 22 +++
.../container/ContainerEventType.java | 6 +-
.../container/ContainerImpl.java | 170 ++++++++++++++++++-
.../container/ContainerPauseEvent.java | 40 +++++
.../container/ContainerResumeEvent.java | 39 +++++
.../container/ContainerState.java | 3 +-
.../launcher/ContainerLaunch.java | 90 +++++++++-
.../launcher/ContainersLauncher.java | 32 ++++
.../launcher/ContainersLauncherEventType.java | 3 +
.../scheduler/ContainerSchedulerEventType.java | 1 +
.../container/TestContainer.java | 51 ++++++
13 files changed, 454 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/96423b50/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerState.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerState.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerState.java
index 696fe06..45e5bd4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerState.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/ContainerState.java
@@ -33,11 +33,14 @@ public enum ContainerState {
/** Running container */
RUNNING,
-
+
/** Completed container */
COMPLETE,
/** Scheduled (awaiting resources) at the NM. */
@InterfaceStability.Unstable
- SCHEDULED
+ SCHEDULED,
+
+ /** Paused at the NM. */
+ PAUSED
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/96423b50/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
index 81ebd79..b299c23 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/proto/yarn_protos.proto
@@ -83,6 +83,7 @@ enum ContainerStateProto {
C_RUNNING = 2;
C_COMPLETE = 3;
C_SCHEDULED = 4;
+ C_PAUSED = 5;
}
message ContainerProto {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/96423b50/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
index 9767fb9..6e409be 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
@@ -695,6 +695,28 @@ public abstract class ContainerExecutor implements Configurable {
}
/**
+ * Pause the container. The default implementation is to raise a kill event.
+ * Specific executor implementations can override this behavior.
+ * @param container
+ * the Container
+ */
+ public void pauseContainer(Container container) {
+ LOG.warn(container.getContainerId() + " doesn't support pausing.");
+ throw new UnsupportedOperationException();
+ }
+
+ /**
+ * Resume the container from pause state. The default implementation ignores
+ * this event. Specific implementations can override this behavior.
+ * @param container
+ * the Container
+ */
+ public void resumeContainer(Container container) {
+ LOG.warn(container.getContainerId() + " doesn't support resume.");
+ throw new UnsupportedOperationException();
+ }
+
+ /**
* Get the process-identifier for the container.
*
* @param containerID the container ID
http://git-wip-us.apache.org/repos/asf/hadoop/blob/96423b50/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerEventType.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerEventType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerEventType.java
index afea0e6..1475435 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerEventType.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerEventType.java
@@ -27,6 +27,8 @@ public enum ContainerEventType {
CONTAINER_DONE,
REINITIALIZE_CONTAINER,
ROLLBACK_REINIT,
+ PAUSE_CONTAINER,
+ RESUME_CONTAINER,
// DownloadManager
CONTAINER_INITED,
@@ -38,5 +40,7 @@ public enum ContainerEventType {
CONTAINER_LAUNCHED,
CONTAINER_EXITED_WITH_SUCCESS,
CONTAINER_EXITED_WITH_FAILURE,
- CONTAINER_KILLED_ON_REQUEST
+ CONTAINER_KILLED_ON_REQUEST,
+ CONTAINER_PAUSED,
+ CONTAINER_RESUMED
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/96423b50/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
index 8e42133..031ce54 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
@@ -294,6 +294,8 @@ public class ContainerImpl implements Container {
UPDATE_DIAGNOSTICS_TRANSITION)
.addTransition(ContainerState.NEW, ContainerState.DONE,
ContainerEventType.KILL_CONTAINER, new KillOnNewTransition())
+ .addTransition(ContainerState.NEW, ContainerState.DONE,
+ ContainerEventType.PAUSE_CONTAINER, new KillOnPauseTransition())
// From LOCALIZING State
.addTransition(ContainerState.LOCALIZING,
@@ -309,6 +311,8 @@ public class ContainerImpl implements Container {
.addTransition(ContainerState.LOCALIZING, ContainerState.KILLING,
ContainerEventType.KILL_CONTAINER,
new KillBeforeRunningTransition())
+ .addTransition(ContainerState.LOCALIZING, ContainerState.KILLING,
+ ContainerEventType.PAUSE_CONTAINER, new KillOnPauseTransition())
// From LOCALIZATION_FAILED State
.addTransition(ContainerState.LOCALIZATION_FAILED,
@@ -322,7 +326,8 @@ public class ContainerImpl implements Container {
// container not launched so kill is a no-op
.addTransition(ContainerState.LOCALIZATION_FAILED,
ContainerState.LOCALIZATION_FAILED,
- ContainerEventType.KILL_CONTAINER)
+ EnumSet.of(ContainerEventType.KILL_CONTAINER,
+ ContainerEventType.PAUSE_CONTAINER))
// container cleanup triggers a release of all resources
// regardless of whether they were localized or not
// LocalizedResource handles release event in all states
@@ -378,6 +383,76 @@ public class ContainerImpl implements Container {
ContainerState.EXITED_WITH_FAILURE,
ContainerEventType.CONTAINER_KILLED_ON_REQUEST,
new KilledExternallyTransition())
+ .addTransition(ContainerState.RUNNING, ContainerState.PAUSING,
+ ContainerEventType.PAUSE_CONTAINER, new PauseContainerTransition())
+
+ // From PAUSING State
+ .addTransition(ContainerState.PAUSING, ContainerState.KILLING,
+ ContainerEventType.KILL_CONTAINER, new KillTransition())
+ .addTransition(ContainerState.PAUSING, ContainerState.PAUSING,
+ ContainerEventType.UPDATE_DIAGNOSTICS_MSG,
+ UPDATE_DIAGNOSTICS_TRANSITION)
+ .addTransition(ContainerState.PAUSING, ContainerState.PAUSED,
+ ContainerEventType.CONTAINER_PAUSED, new PausedContainerTransition())
+ // In case something goes wrong then container will exit from the
+ // PAUSING state
+ .addTransition(ContainerState.PAUSING,
+ ContainerState.EXITED_WITH_SUCCESS,
+ ContainerEventType.CONTAINER_EXITED_WITH_SUCCESS)
+ .addTransition(ContainerState.PAUSING,
+ ContainerState.EXITED_WITH_FAILURE,
+ ContainerEventType.CONTAINER_EXITED_WITH_FAILURE,
+ new ExitedWithFailureTransition(true))
+ .addTransition(ContainerState.PAUSING, ContainerState.EXITED_WITH_FAILURE,
+ ContainerEventType.CONTAINER_KILLED_ON_REQUEST,
+ new KilledExternallyTransition())
+
+ // From PAUSED State
+ .addTransition(ContainerState.PAUSED, ContainerState.KILLING,
+ ContainerEventType.KILL_CONTAINER, new KillTransition())
+ .addTransition(ContainerState.PAUSED, ContainerState.PAUSED,
+ ContainerEventType.UPDATE_DIAGNOSTICS_MSG,
+ UPDATE_DIAGNOSTICS_TRANSITION)
+ .addTransition(ContainerState.PAUSED, ContainerState.PAUSED,
+ ContainerEventType.PAUSE_CONTAINER)
+ .addTransition(ContainerState.PAUSED, ContainerState.RESUMING,
+ ContainerEventType.RESUME_CONTAINER, new ResumeContainerTransition())
+ // In case something goes wrong then container will exit from the
+ // PAUSED state
+ .addTransition(ContainerState.PAUSED,
+ ContainerState.EXITED_WITH_FAILURE,
+ ContainerEventType.CONTAINER_EXITED_WITH_FAILURE,
+ new ExitedWithFailureTransition(true))
+ .addTransition(ContainerState.PAUSED, ContainerState.EXITED_WITH_FAILURE,
+ ContainerEventType.CONTAINER_KILLED_ON_REQUEST,
+ new KilledExternallyTransition())
+ .addTransition(ContainerState.PAUSED,
+ ContainerState.EXITED_WITH_SUCCESS,
+ ContainerEventType.CONTAINER_EXITED_WITH_SUCCESS,
+ new ExitedWithSuccessTransition(true))
+
+ // From RESUMING State
+ .addTransition(ContainerState.RESUMING, ContainerState.KILLING,
+ ContainerEventType.KILL_CONTAINER, new KillTransition())
+ .addTransition(ContainerState.RESUMING, ContainerState.RUNNING,
+ ContainerEventType.CONTAINER_RESUMED)
+ .addTransition(ContainerState.RESUMING, ContainerState.RESUMING,
+ ContainerEventType.UPDATE_DIAGNOSTICS_MSG,
+ UPDATE_DIAGNOSTICS_TRANSITION)
+ // In case something goes wrong then container will exit from the
+ // RESUMING state
+ .addTransition(ContainerState.RESUMING,
+ ContainerState.EXITED_WITH_FAILURE,
+ ContainerEventType.CONTAINER_EXITED_WITH_FAILURE,
+ new ExitedWithFailureTransition(true))
+ .addTransition(ContainerState.RESUMING,
+ ContainerState.EXITED_WITH_FAILURE,
+ ContainerEventType.CONTAINER_KILLED_ON_REQUEST,
+ new KilledExternallyTransition())
+ .addTransition(ContainerState.RESUMING,
+ ContainerState.EXITED_WITH_SUCCESS,
+ ContainerEventType.CONTAINER_EXITED_WITH_SUCCESS,
+ new ExitedWithSuccessTransition(true))
// From REINITIALIZING State
.addTransition(ContainerState.REINITIALIZING,
@@ -401,6 +476,8 @@ public class ContainerImpl implements Container {
UPDATE_DIAGNOSTICS_TRANSITION)
.addTransition(ContainerState.REINITIALIZING, ContainerState.KILLING,
ContainerEventType.KILL_CONTAINER, new KillTransition())
+ .addTransition(ContainerState.REINITIALIZING, ContainerState.KILLING,
+ ContainerEventType.PAUSE_CONTAINER, new KillOnPauseTransition())
.addTransition(ContainerState.REINITIALIZING,
ContainerState.SCHEDULED,
ContainerEventType.CONTAINER_KILLED_ON_REQUEST,
@@ -418,6 +495,8 @@ public class ContainerImpl implements Container {
UPDATE_DIAGNOSTICS_TRANSITION)
.addTransition(ContainerState.RELAUNCHING, ContainerState.KILLING,
ContainerEventType.KILL_CONTAINER, new KillTransition())
+ .addTransition(ContainerState.RELAUNCHING, ContainerState.KILLING,
+ ContainerEventType.PAUSE_CONTAINER, new KillOnPauseTransition())
// From CONTAINER_EXITED_WITH_SUCCESS State
.addTransition(ContainerState.EXITED_WITH_SUCCESS, ContainerState.DONE,
@@ -429,7 +508,8 @@ public class ContainerImpl implements Container {
UPDATE_DIAGNOSTICS_TRANSITION)
.addTransition(ContainerState.EXITED_WITH_SUCCESS,
ContainerState.EXITED_WITH_SUCCESS,
- ContainerEventType.KILL_CONTAINER)
+ EnumSet.of(ContainerEventType.KILL_CONTAINER,
+ ContainerEventType.PAUSE_CONTAINER))
// From EXITED_WITH_FAILURE State
.addTransition(ContainerState.EXITED_WITH_FAILURE, ContainerState.DONE,
@@ -441,7 +521,8 @@ public class ContainerImpl implements Container {
UPDATE_DIAGNOSTICS_TRANSITION)
.addTransition(ContainerState.EXITED_WITH_FAILURE,
ContainerState.EXITED_WITH_FAILURE,
- ContainerEventType.KILL_CONTAINER)
+ EnumSet.of(ContainerEventType.KILL_CONTAINER,
+ ContainerEventType.PAUSE_CONTAINER))
// From KILLING State.
.addTransition(ContainerState.KILLING,
@@ -475,7 +556,8 @@ public class ContainerImpl implements Container {
// in the container launcher
.addTransition(ContainerState.KILLING,
ContainerState.KILLING,
- ContainerEventType.CONTAINER_LAUNCHED)
+ EnumSet.of(ContainerEventType.CONTAINER_LAUNCHED,
+ ContainerEventType.PAUSE_CONTAINER))
// From CONTAINER_CLEANEDUP_AFTER_KILL State.
.addTransition(ContainerState.CONTAINER_CLEANEDUP_AFTER_KILL,
@@ -491,11 +573,13 @@ public class ContainerImpl implements Container {
EnumSet.of(ContainerEventType.KILL_CONTAINER,
ContainerEventType.RESOURCE_FAILED,
ContainerEventType.CONTAINER_EXITED_WITH_SUCCESS,
- ContainerEventType.CONTAINER_EXITED_WITH_FAILURE))
+ ContainerEventType.CONTAINER_EXITED_WITH_FAILURE,
+ ContainerEventType.PAUSE_CONTAINER))
// From DONE
.addTransition(ContainerState.DONE, ContainerState.DONE,
- ContainerEventType.KILL_CONTAINER)
+ EnumSet.of(ContainerEventType.KILL_CONTAINER,
+ ContainerEventType.PAUSE_CONTAINER))
.addTransition(ContainerState.DONE, ContainerState.DONE,
ContainerEventType.INIT_CONTAINER)
.addTransition(ContainerState.DONE, ContainerState.DONE,
@@ -521,6 +605,8 @@ public class ContainerImpl implements Container {
case LOCALIZING:
case LOCALIZATION_FAILED:
case SCHEDULED:
+ case PAUSED:
+ case RESUMING:
return org.apache.hadoop.yarn.api.records.ContainerState.SCHEDULED;
case RUNNING:
case RELAUNCHING:
@@ -530,6 +616,7 @@ public class ContainerImpl implements Container {
case KILLING:
case CONTAINER_CLEANEDUP_AFTER_KILL:
case CONTAINER_RESOURCES_CLEANINGUP:
+ case PAUSING:
return org.apache.hadoop.yarn.api.records.ContainerState.RUNNING;
case DONE:
default:
@@ -1481,6 +1568,26 @@ public class ContainerImpl implements Container {
}
/**
+ * Transitions upon receiving PAUSE_CONTAINER.
+ * - LOCALIZED -> KILLING.
+ * - REINITIALIZING -> KILLING.
+ */
+ @SuppressWarnings("unchecked") // dispatcher not typed
+ static class KillOnPauseTransition implements
+ SingleArcTransition<ContainerImpl, ContainerEvent> {
+
+ @SuppressWarnings("unchecked")
+ @Override
+ public void transition(ContainerImpl container, ContainerEvent event) {
+ // Kill the process/process-grp
+ container.setIsReInitializing(false);
+ container.dispatcher.getEventHandler().handle(
+ new ContainersLauncherEvent(container,
+ ContainersLauncherEventType.CLEANUP_CONTAINER));
+ }
+ }
+
+ /**
* Transition from KILLING to CONTAINER_CLEANEDUP_AFTER_KILL
* upon receiving CONTAINER_KILLED_ON_REQUEST.
*/
@@ -1670,6 +1777,57 @@ public class ContainerImpl implements Container {
}
}
+ /**
+ * Transitions upon receiving PAUSE_CONTAINER.
+ * - RUNNING -> PAUSED
+ */
+ @SuppressWarnings("unchecked") // dispatcher not typed
+ static class PauseContainerTransition implements
+ SingleArcTransition<ContainerImpl, ContainerEvent> {
+ @Override
+ public void transition(ContainerImpl container, ContainerEvent event) {
+ // Pause the process/process-grp if it is supported by the container
+ container.dispatcher.getEventHandler().handle(
+ new ContainersLauncherEvent(container,
+ ContainersLauncherEventType.PAUSE_CONTAINER));
+ ContainerPauseEvent pauseEvent = (ContainerPauseEvent) event;
+ container.addDiagnostics(pauseEvent.getDiagnostic(), "\n");
+ }
+ }
+
+ /**
+ * Transitions upon receiving PAUSED_CONTAINER.
+ */
+ @SuppressWarnings("unchecked") // dispatcher not typed
+ static class PausedContainerTransition implements
+ SingleArcTransition<ContainerImpl, ContainerEvent> {
+ @Override
+ public void transition(ContainerImpl container, ContainerEvent event) {
+ // Container was PAUSED so tell the scheduler
+ container.dispatcher.getEventHandler().handle(
+ new ContainerSchedulerEvent(container,
+ ContainerSchedulerEventType.CONTAINER_PAUSED));
+ }
+ }
+
+ /**
+ * Transitions upon receiving RESUME_CONTAINER.
+ * - PAUSED -> RUNNING
+ */
+ @SuppressWarnings("unchecked") // dispatcher not typed
+ static class ResumeContainerTransition implements
+ SingleArcTransition<ContainerImpl, ContainerEvent> {
+ @Override
+ public void transition(ContainerImpl container, ContainerEvent event) {
+ // Pause the process/process-grp if it is supported by the container
+ container.dispatcher.getEventHandler().handle(
+ new ContainersLauncherEvent(container,
+ ContainersLauncherEventType.RESUME_CONTAINER));
+ ContainerResumeEvent resumeEvent = (ContainerResumeEvent) event;
+ container.addDiagnostics(resumeEvent.getDiagnostic(), "\n");
+ }
+ }
+
@Override
public void handle(ContainerEvent event) {
try {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/96423b50/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerPauseEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerPauseEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerPauseEvent.java
new file mode 100644
index 0000000..898304e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerPauseEvent.java
@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.container;
+
+import org.apache.hadoop.yarn.api.records.ContainerId;
+
+/**
+ * ContainerEvent for ContainerEventType.PAUSE_CONTAINER.
+ */
+public class ContainerPauseEvent extends ContainerEvent {
+
+ private final String diagnostic;
+
+ public ContainerPauseEvent(ContainerId cId,
+ String diagnostic) {
+ super(cId, ContainerEventType.PAUSE_CONTAINER);
+ this.diagnostic = diagnostic;
+ }
+
+ public String getDiagnostic() {
+ return this.diagnostic;
+ }
+}
+
http://git-wip-us.apache.org/repos/asf/hadoop/blob/96423b50/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerResumeEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerResumeEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerResumeEvent.java
new file mode 100644
index 0000000..d7c9e9a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerResumeEvent.java
@@ -0,0 +1,39 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.nodemanager.containermanager.container;
+
+import org.apache.hadoop.yarn.api.records.ContainerId;
+
+/**
+ * ContainerEvent for ContainerEventType.RESUME_CONTAINER.
+ */
+public class ContainerResumeEvent extends ContainerEvent {
+
+ private final String diagnostic;
+
+ public ContainerResumeEvent(ContainerId cId,
+ String diagnostic) {
+ super(cId, ContainerEventType.RESUME_CONTAINER);
+ this.diagnostic = diagnostic;
+ }
+
+ public String getDiagnostic() {
+ return this.diagnostic;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/96423b50/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerState.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerState.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerState.java
index 91d1356..7c3fea8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerState.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerState.java
@@ -21,5 +21,6 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager.container;
public enum ContainerState {
NEW, LOCALIZING, LOCALIZATION_FAILED, SCHEDULED, RUNNING, RELAUNCHING,
REINITIALIZING, EXITED_WITH_SUCCESS, EXITED_WITH_FAILURE, KILLING,
- CONTAINER_CLEANEDUP_AFTER_KILL, CONTAINER_RESOURCES_CLEANINGUP, DONE
+ CONTAINER_CLEANEDUP_AFTER_KILL, CONTAINER_RESOURCES_CLEANINGUP, DONE,
+ PAUSING, PAUSED, RESUMING
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/96423b50/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
index a0055c5..182214f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
@@ -75,6 +75,7 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Cont
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerEventType;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerExitEvent;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerKillEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ContainerState;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceLocalizationService;
@@ -86,6 +87,7 @@ import org.apache.hadoop.yarn.util.Apps;
import org.apache.hadoop.yarn.util.AuxiliaryServiceHelper;
import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.yarn.util.ConverterUtils;
public class ContainerLaunch implements Callable<Integer> {
@@ -106,8 +108,10 @@ public class ContainerLaunch implements Callable<Integer> {
private final Configuration conf;
private final Context context;
private final ContainerManagerImpl containerManager;
-
+
protected AtomicBoolean containerAlreadyLaunched = new AtomicBoolean(false);
+ protected AtomicBoolean shouldPauseContainer = new AtomicBoolean(false);
+
protected AtomicBoolean completed = new AtomicBoolean(false);
private volatile boolean killedBeforeStart = false;
@@ -778,6 +782,90 @@ public class ContainerLaunch implements Callable<Integer> {
}
/**
+ * Pause the container.
+ * Cancels the launch if the container isn't launched yet. Otherwise asks the
+ * executor to pause the container.
+ * @throws IOException in case of errors.
+ */
+ @SuppressWarnings("unchecked") // dispatcher not typed
+ public void pauseContainer() throws IOException {
+ ContainerId containerId = container.getContainerId();
+ String containerIdStr = containerId.toString();
+ LOG.info("Pausing the container " + containerIdStr);
+
+ // The pause event is only handled if the container is in the running state
+ // (the container state machine), so we don't check for
+ // shouldLaunchContainer over here
+
+ if (!shouldPauseContainer.compareAndSet(false, true)) {
+ LOG.info("Container " + containerId + " not paused as "
+ + "resume already called");
+ return;
+ }
+
+ try {
+ // Pause the container
+ exec.pauseContainer(container);
+
+ // PauseContainer is a blocking call. We are here almost means the
+ // container is paused, so send out the event.
+ dispatcher.getEventHandler().handle(new ContainerEvent(
+ containerId,
+ ContainerEventType.CONTAINER_PAUSED));
+ } catch (Exception e) {
+ String message =
+ "Exception when trying to pause container " + containerIdStr
+ + ": " + StringUtils.stringifyException(e);
+ LOG.info(message);
+ container.handle(new ContainerKillEvent(container.getContainerId(),
+ ContainerExitStatus.PREEMPTED, "Container preempted as there was "
+ + " an exception in pausing it."));
+ }
+ }
+
+ /**
+ * Resume the container.
+ * Cancels the launch if the container isn't launched yet. Otherwise asks the
+ * executor to pause the container.
+ * @throws IOException in case of error.
+ */
+ @SuppressWarnings("unchecked") // dispatcher not typed
+ public void resumeContainer() throws IOException {
+ ContainerId containerId = container.getContainerId();
+ String containerIdStr = containerId.toString();
+ LOG.info("Resuming the container " + containerIdStr);
+
+ // The resume event is only handled if the container is in a paused state
+ // so we don't check for the launched flag here.
+
+ // paused flag will be set to true if process already paused
+ boolean alreadyPaused = !shouldPauseContainer.compareAndSet(false, true);
+ if (!alreadyPaused) {
+ LOG.info("Container " + containerIdStr + " not paused."
+ + " No resume necessary");
+ return;
+ }
+
+ // If the container has already started
+ try {
+ exec.resumeContainer(container);
+ // ResumeContainer is a blocking call. We are here almost means the
+ // container is resumed, so send out the event.
+ dispatcher.getEventHandler().handle(new ContainerEvent(
+ containerId,
+ ContainerEventType.CONTAINER_RESUMED));
+ } catch (Exception e) {
+ String message =
+ "Exception when trying to resume container " + containerIdStr
+ + ": " + StringUtils.stringifyException(e);
+ LOG.info(message);
+ container.handle(new ContainerKillEvent(container.getContainerId(),
+ ContainerExitStatus.PREEMPTED, "Container preempted as there was "
+ + " an exception in pausing it."));
+ }
+ }
+
+ /**
* Loop through for a time-bounded interval waiting to
* read the process id from a file generated by a running process.
* @param pidFilePath File from which to read the process id
http://git-wip-us.apache.org/repos/asf/hadoop/blob/96423b50/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java
index 25909b9..ca69712 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java
@@ -30,6 +30,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.UnsupportedFileSystemException;
import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.util.concurrent.HadoopExecutors;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.event.Dispatcher;
@@ -41,6 +42,7 @@ import org.apache.hadoop.yarn.server.nodemanager.LocalDirsHandlerService;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManagerImpl;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceLocalizationService;
import com.google.common.annotations.VisibleForTesting;
@@ -171,6 +173,36 @@ public class ContainersLauncher extends AbstractService
+ " with command " + signalEvent.getCommand());
}
break;
+ case PAUSE_CONTAINER:
+ ContainerLaunch launchedContainer = running.get(containerId);
+ if (launchedContainer == null) {
+ // Container not launched. So nothing needs to be done.
+ return;
+ }
+
+ // Pause the container
+ try {
+ launchedContainer.pauseContainer();
+ } catch (Exception e) {
+ LOG.info("Got exception while pausing container: " +
+ StringUtils.stringifyException(e));
+ }
+ break;
+ case RESUME_CONTAINER:
+ ContainerLaunch launchCont = running.get(containerId);
+ if (launchCont == null) {
+ // Container not launched. So nothing needs to be done.
+ return;
+ }
+
+ // Resume the container.
+ try {
+ launchCont.resumeContainer();
+ } catch (Exception e) {
+ LOG.info("Got exception while resuming container: " +
+ StringUtils.stringifyException(e));
+ }
+ break;
}
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/96423b50/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncherEventType.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncherEventType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncherEventType.java
index 380a032..1054e06 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncherEventType.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncherEventType.java
@@ -25,4 +25,7 @@ public enum ContainersLauncherEventType {
CLEANUP_CONTAINER, // The process(grp) itself.
CLEANUP_CONTAINER_FOR_REINIT, // The process(grp) itself.
SIGNAL_CONTAINER,
+ PAUSE_CONTAINER,
+ RESUME_CONTAINER
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/96423b50/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerSchedulerEventType.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerSchedulerEventType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerSchedulerEventType.java
index 917eda0..a9cbf74 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerSchedulerEventType.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/ContainerSchedulerEventType.java
@@ -27,4 +27,5 @@ public enum ContainerSchedulerEventType {
UPDATE_CONTAINER,
// Producer: Node HB response - RM has asked to shed the queue
SHED_QUEUED_CONTAINERS,
+ CONTAINER_PAUSED
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/96423b50/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java
index 33f4609..8909088 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java
@@ -103,6 +103,7 @@ import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.junit.Assert;
import org.junit.Test;
import org.mockito.ArgumentMatcher;
+import org.mockito.Mockito;
public class TestContainer {
@@ -207,6 +208,42 @@ public class TestContainer {
@Test
@SuppressWarnings("unchecked") // mocked generic
+ public void testContainerPauseAndResume() throws Exception {
+ WrappedContainer wc = null;
+ try {
+ wc = new WrappedContainer(13, 314159265358979L, 4344, "yak");
+ wc.initContainer();
+ wc.localizeResources();
+ int running = metrics.getRunningContainers();
+ wc.launchContainer();
+ assertEquals(running + 1, metrics.getRunningContainers());
+ reset(wc.localizerBus);
+ wc.pauseContainer();
+ assertEquals(ContainerState.PAUSED,
+ wc.c.getContainerState());
+ wc.resumeContainer();
+ assertEquals(ContainerState.RUNNING,
+ wc.c.getContainerState());
+ wc.containerKilledOnRequest();
+ assertEquals(ContainerState.EXITED_WITH_FAILURE,
+ wc.c.getContainerState());
+ assertNull(wc.c.getLocalizedResources());
+ verifyCleanupCall(wc);
+ int failed = metrics.getFailedContainers();
+ wc.containerResourcesCleanup();
+ assertEquals(ContainerState.DONE, wc.c.getContainerState());
+ assertEquals(failed + 1, metrics.getFailedContainers());
+ assertEquals(running, metrics.getRunningContainers());
+ }
+ finally {
+ if (wc != null) {
+ wc.finished();
+ }
+ }
+ }
+
+ @Test
+ @SuppressWarnings("unchecked") // mocked generic
public void testCleanupOnFailure() throws Exception {
WrappedContainer wc = null;
try {
@@ -955,6 +992,8 @@ public class TestContainer {
NodeStatusUpdater nodeStatusUpdater = mock(NodeStatusUpdater.class);
when(context.getNodeStatusUpdater()).thenReturn(nodeStatusUpdater);
ContainerExecutor executor = mock(ContainerExecutor.class);
+ Mockito.doNothing().when(executor).pauseContainer(any(Container.class));
+ Mockito.doNothing().when(executor).resumeContainer(any(Container.class));
launcher =
new ContainersLauncher(context, dispatcher, executor, null, null);
// create a mock ExecutorService, which will not really launch
@@ -1143,6 +1182,18 @@ public class TestContainer {
drainDispatcherEvents();
}
+ public void pauseContainer() {
+ c.handle(new ContainerPauseEvent(cId,
+ "PauseRequest"));
+ drainDispatcherEvents();
+ }
+
+ public void resumeContainer() {
+ c.handle(new ContainerResumeEvent(cId,
+ "ResumeRequest"));
+ drainDispatcherEvents();
+ }
+
public void containerKilledOnRequest() {
int exitCode = ContainerExitStatus.KILLED_BY_RESOURCEMANAGER;
String diagnosticMsg = "Container completed with exit code " + exitCode;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[42/50] [abbrv] hadoop git commit: HADOOP-14251. Credential provider
should handle property key deprecation. Contributed by John Zhuge.
Posted by as...@apache.org.
HADOOP-14251. Credential provider should handle property key deprecation. Contributed by John Zhuge.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/7e6463d2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/7e6463d2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/7e6463d2
Branch: refs/heads/YARN-5972
Commit: 7e6463d2fb5f9383d88baec290461868cf476e4c
Parents: f49843a
Author: John Zhuge <jz...@cloudera.com>
Authored: Sat Aug 12 22:56:49 2017 -0700
Committer: John Zhuge <jz...@apache.org>
Committed: Wed Aug 23 11:37:18 2017 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/conf/Configuration.java | 71 ++++++++++++++++++--
.../apache/hadoop/conf/TestConfiguration.java | 60 +++++++++++++++++
2 files changed, 127 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7e6463d2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
index edaee68..a1a40b9 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/conf/Configuration.java
@@ -308,18 +308,25 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
this.customMessage = customMessage;
}
+ private final String getWarningMessage(String key) {
+ return getWarningMessage(key, null);
+ }
+
/**
* Method to provide the warning message. It gives the custom message if
* non-null, and default message otherwise.
* @param key the associated deprecated key.
+ * @param source the property source.
* @return message that is to be logged when a deprecated key is used.
*/
- private final String getWarningMessage(String key) {
+ private String getWarningMessage(String key, String source) {
String warningMessage;
if(customMessage == null) {
StringBuilder message = new StringBuilder(key);
- String deprecatedKeySuffix = " is deprecated. Instead, use ";
- message.append(deprecatedKeySuffix);
+ if (source != null) {
+ message.append(" in " + source);
+ }
+ message.append(" is deprecated. Instead, use ");
for (int i = 0; i < newKeys.length; i++) {
message.append(newKeys[i]);
if(i != newKeys.length-1) {
@@ -593,6 +600,14 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
return deprecationContext.get().getDeprecatedKeyMap().containsKey(key);
}
+ private static String getDeprecatedKey(String key) {
+ return deprecationContext.get().getReverseDeprecatedKeyMap().get(key);
+ }
+
+ private static DeprecatedKeyInfo getDeprecatedKeyInfo(String key) {
+ return deprecationContext.get().getDeprecatedKeyMap().get(key);
+ }
+
/**
* Sets all deprecated properties that are not currently set but have a
* corresponding new property that is set. Useful for iterating the
@@ -1270,6 +1285,13 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
LOG_DEPRECATION.info(message);
}
+ void logDeprecationOnce(String name, String source) {
+ DeprecatedKeyInfo keyInfo = getDeprecatedKeyInfo(name);
+ if (keyInfo != null && !keyInfo.getAndSetAccessed()) {
+ LOG_DEPRECATION.info(keyInfo.getWarningMessage(name, source));
+ }
+ }
+
/**
* Unset a previously set property.
*/
@@ -2080,6 +2102,47 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
}
/**
+ * Get the credential entry by name from a credential provider.
+ *
+ * Handle key deprecation.
+ *
+ * @param provider a credential provider
+ * @param name alias of the credential
+ * @return the credential entry or null if not found
+ */
+ private CredentialEntry getCredentialEntry(CredentialProvider provider,
+ String name) throws IOException {
+ CredentialEntry entry = provider.getCredentialEntry(name);
+ if (entry != null) {
+ return entry;
+ }
+
+ // The old name is stored in the credential provider.
+ String oldName = getDeprecatedKey(name);
+ if (oldName != null) {
+ entry = provider.getCredentialEntry(oldName);
+ if (entry != null) {
+ logDeprecationOnce(oldName, provider.toString());
+ return entry;
+ }
+ }
+
+ // The name is deprecated.
+ DeprecatedKeyInfo keyInfo = getDeprecatedKeyInfo(name);
+ if (keyInfo != null && keyInfo.newKeys != null) {
+ for (String newName : keyInfo.newKeys) {
+ entry = provider.getCredentialEntry(newName);
+ if (entry != null) {
+ logDeprecationOnce(name, null);
+ return entry;
+ }
+ }
+ }
+
+ return null;
+ }
+
+ /**
* Try and resolve the provided element name as a credential provider
* alias.
* @param name alias of the provisioned credential
@@ -2096,7 +2159,7 @@ public class Configuration implements Iterable<Map.Entry<String,String>>,
if (providers != null) {
for (CredentialProvider provider : providers) {
try {
- CredentialEntry entry = provider.getCredentialEntry(name);
+ CredentialEntry entry = getCredentialEntry(provider, name);
if (entry != null) {
pass = entry.getCredential();
break;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/7e6463d2/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
index 91f25fa..8fe88bc 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/conf/TestConfiguration.java
@@ -50,9 +50,13 @@ import static org.junit.Assert.assertArrayEquals;
import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.conf.Configuration.IntegerRanges;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.io.IOUtils;
import org.apache.hadoop.net.NetUtils;
+import org.apache.hadoop.security.alias.CredentialProvider;
+import org.apache.hadoop.security.alias.CredentialProviderFactory;
+import org.apache.hadoop.security.alias.LocalJavaKeyStoreProvider;
import org.apache.hadoop.test.GenericTestUtils;
import static org.apache.hadoop.util.PlatformName.IBM_JAVA;
@@ -60,6 +64,8 @@ import static org.apache.hadoop.util.PlatformName.IBM_JAVA;
import org.apache.log4j.AppenderSkeleton;
import org.apache.log4j.Logger;
import org.apache.log4j.spi.LoggingEvent;
+import org.hamcrest.CoreMatchers;
+import org.junit.Assert;
import org.mockito.Mockito;
public class TestConfiguration extends TestCase {
@@ -2068,6 +2074,60 @@ public class TestConfiguration extends TestCase {
assertEquals("value", conf.get("attr"));
}
+ public void testGetPasswordDeprecatedKeyStored() throws Exception {
+ final String oldKey = "test.password.old.key";
+ final String newKey = "test.password.new.key";
+ final String password = "MyPasswordForDeprecatedKey";
+
+ final File tmpDir = GenericTestUtils.getRandomizedTestDir();
+ tmpDir.mkdirs();
+ final String ourUrl = new URI(LocalJavaKeyStoreProvider.SCHEME_NAME,
+ "file", new File(tmpDir, "test.jks").toString(), null).toString();
+
+ conf = new Configuration(false);
+ conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, ourUrl);
+ CredentialProvider provider =
+ CredentialProviderFactory.getProviders(conf).get(0);
+ provider.createCredentialEntry(oldKey, password.toCharArray());
+ provider.flush();
+
+ Configuration.addDeprecation(oldKey, newKey);
+
+ Assert.assertThat(conf.getPassword(newKey),
+ CoreMatchers.is(password.toCharArray()));
+ Assert.assertThat(conf.getPassword(oldKey),
+ CoreMatchers.is(password.toCharArray()));
+
+ FileUtil.fullyDelete(tmpDir);
+ }
+
+ public void testGetPasswordByDeprecatedKey() throws Exception {
+ final String oldKey = "test.password.old.key";
+ final String newKey = "test.password.new.key";
+ final String password = "MyPasswordForDeprecatedKey";
+
+ final File tmpDir = GenericTestUtils.getRandomizedTestDir();
+ tmpDir.mkdirs();
+ final String ourUrl = new URI(LocalJavaKeyStoreProvider.SCHEME_NAME,
+ "file", new File(tmpDir, "test.jks").toString(), null).toString();
+
+ conf = new Configuration(false);
+ conf.set(CredentialProviderFactory.CREDENTIAL_PROVIDER_PATH, ourUrl);
+ CredentialProvider provider =
+ CredentialProviderFactory.getProviders(conf).get(0);
+ provider.createCredentialEntry(newKey, password.toCharArray());
+ provider.flush();
+
+ Configuration.addDeprecation(oldKey, newKey);
+
+ Assert.assertThat(conf.getPassword(newKey),
+ CoreMatchers.is(password.toCharArray()));
+ Assert.assertThat(conf.getPassword(oldKey),
+ CoreMatchers.is(password.toCharArray()));
+
+ FileUtil.fullyDelete(tmpDir);
+ }
+
public static void main(String[] argv) throws Exception {
junit.textui.TestRunner.main(new String[]{
TestConfiguration.class.getName()
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[34/50] [abbrv] hadoop git commit: YARN-7047. Moving logging APIs
over to slf4j in hadoop-yarn-server-nodemanager. Contributed by Yeliang Cang.
Posted by as...@apache.org.
YARN-7047. Moving logging APIs over to slf4j in hadoop-yarn-server-nodemanager. Contributed by Yeliang Cang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d5ff57a0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d5ff57a0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d5ff57a0
Branch: refs/heads/YARN-5972
Commit: d5ff57a08fac983f8b5d201064ce07945f0f216e
Parents: ae8fb13
Author: Akira Ajisaka <aa...@apache.org>
Authored: Tue Aug 22 17:14:12 2017 +0900
Committer: Akira Ajisaka <aa...@apache.org>
Committed: Tue Aug 22 17:14:12 2017 +0900
----------------------------------------------------------------------
.../yarn/server/nodemanager/ContainerExecutor.java | 7 ++++---
.../server/nodemanager/DefaultContainerExecutor.java | 12 ++++++------
.../yarn/server/nodemanager/DeletionService.java | 7 ++++---
.../yarn/server/nodemanager/DirectoryCollection.java | 7 ++++---
.../server/nodemanager/LinuxContainerExecutor.java | 8 ++++----
.../server/nodemanager/LocalDirsHandlerService.java | 7 ++++---
.../hadoop/yarn/server/nodemanager/NMAuditLogger.java | 7 ++++---
.../hadoop/yarn/server/nodemanager/NodeManager.java | 13 +++++++------
.../server/nodemanager/NodeResourceMonitorImpl.java | 8 ++++----
.../server/nodemanager/NodeStatusUpdaterImpl.java | 9 +++++----
.../amrmproxy/AMRMProxyTokenSecretManager.java | 8 ++++----
.../server/nodemanager/api/impl/pb/NMProtoUtils.java | 7 ++++---
.../collectormanager/NMCollectorService.java | 7 ++++---
.../nodemanager/containermanager/AuxServices.java | 11 ++++++-----
.../containermanager/ContainerManagerImpl.java | 7 ++++---
.../containermanager/application/ApplicationImpl.java | 7 ++++---
.../containermanager/container/ContainerImpl.java | 7 ++++---
.../containermanager/deletion/task/DeletionTask.java | 7 ++++---
.../containermanager/launcher/ContainerLaunch.java | 14 ++++++++------
.../containermanager/launcher/ContainerRelaunch.java | 7 ++++---
.../containermanager/launcher/ContainersLauncher.java | 7 ++++---
.../launcher/RecoveredContainerLaunch.java | 8 ++++----
.../linux/privileged/PrivilegedOperationExecutor.java | 7 ++++---
.../resources/CGroupsBlkioResourceHandlerImpl.java | 8 ++++----
.../resources/CGroupsCpuResourceHandlerImpl.java | 7 ++++---
.../linux/resources/CGroupsHandlerImpl.java | 9 +++++----
.../resources/CGroupsMemoryResourceHandlerImpl.java | 8 ++++----
.../linux/resources/ResourceHandlerModule.java | 7 ++++---
.../resources/TrafficControlBandwidthHandlerImpl.java | 10 +++++-----
.../linux/resources/TrafficController.java | 7 ++++---
.../linux/runtime/DefaultLinuxContainerRuntime.java | 8 ++++----
.../runtime/DelegatingLinuxContainerRuntime.java | 8 ++++----
.../linux/runtime/DockerLinuxContainerRuntime.java | 8 ++++----
.../runtime/JavaSandboxLinuxContainerRuntime.java | 13 ++++++-------
.../linux/runtime/docker/DockerClient.java | 7 ++++---
.../linux/runtime/docker/DockerCommandExecutor.java | 7 ++++---
.../localizer/ContainerLocalizer.java | 7 ++++---
.../localizer/LocalResourcesTrackerImpl.java | 7 ++++---
.../containermanager/localizer/LocalizedResource.java | 7 ++++---
.../localizer/ResourceLocalizationService.java | 9 +++++----
.../containermanager/localizer/ResourceSet.java | 7 ++++---
.../localizer/security/LocalizerSecurityInfo.java | 7 ++++---
.../localizer/security/LocalizerTokenSelector.java | 8 ++++----
.../sharedcache/SharedCacheUploadService.java | 8 ++++----
.../localizer/sharedcache/SharedCacheUploader.java | 7 ++++---
.../logaggregation/AppLogAggregatorImpl.java | 8 ++++----
.../logaggregation/LogAggregationService.java | 7 ++++---
.../SampleContainerLogAggregationPolicy.java | 8 ++++----
.../loghandler/NonAggregatingLogHandler.java | 8 ++++----
.../monitor/ContainersMonitorImpl.java | 8 ++++----
.../nodelabels/ConfigurationNodeLabelsProvider.java | 8 ++++----
.../nodelabels/ScriptBasedNodeLabelsProvider.java | 7 ++++---
.../recovery/NMLeveldbStateStoreService.java | 13 ++++++-------
.../security/NMContainerTokenSecretManager.java | 8 ++++----
.../security/NMTokenSecretManagerInNM.java | 8 ++++----
.../timelineservice/NMTimelinePublisher.java | 7 ++++---
.../nodemanager/util/CgroupsLCEResourcesHandler.java | 10 +++++-----
.../nodemanager/util/DefaultLCEResourcesHandler.java | 8 ++++----
.../nodemanager/util/NodeManagerHardwareUtils.java | 8 ++++----
.../server/nodemanager/util/ProcessIdFileReader.java | 7 ++++---
.../yarn/server/nodemanager/webapp/NMWebServices.java | 7 ++++---
.../yarn/server/nodemanager/webapp/NavBlock.java | 8 +++-----
.../yarn/server/nodemanager/webapp/WebServer.java | 7 ++++---
.../server/nodemanager/DummyContainerManager.java | 8 ++++----
.../server/nodemanager/MockNodeStatusUpdater.java | 7 ++++---
.../nodemanager/TestContainerManagerWithLCE.java | 8 ++++----
.../nodemanager/TestLinuxContainerExecutor.java | 8 ++++----
.../TestLinuxContainerExecutorWithMocks.java | 8 ++++----
.../server/nodemanager/TestNodeHealthService.java | 8 ++++----
.../server/nodemanager/TestNodeManagerReboot.java | 7 ++++---
.../server/nodemanager/TestNodeManagerResync.java | 8 ++++----
.../server/nodemanager/TestNodeStatusUpdater.java | 7 ++++---
.../nodemanager/amrmproxy/BaseAMRMProxyTest.java | 8 ++++----
.../nodemanager/amrmproxy/TestAMRMProxyService.java | 8 ++++----
.../containermanager/BaseContainerManagerTest.java | 8 ++++----
.../nodemanager/containermanager/TestAuxServices.java | 7 ++++---
.../containermanager/TestContainerManager.java | 4 ++--
.../privileged/TestPrivilegedOperationExecutor.java | 8 ++++----
.../linux/resources/TestCGroupsHandlerImpl.java | 8 ++++----
.../linux/resources/TestResourceHandlerModule.java | 8 ++++----
.../TestTrafficControlBandwidthHandlerImpl.java | 8 ++++----
.../linux/resources/TestTrafficController.java | 7 ++++---
.../linux/runtime/TestDockerContainerRuntime.java | 8 ++++----
.../localizer/TestContainerLocalizer.java | 11 ++++++-----
.../logaggregation/TestLogAggregationService.java | 4 ++--
.../monitor/TestContainersMonitor.java | 4 ++--
.../scheduler/TestContainerSchedulerQueuing.java | 4 ++--
87 files changed, 361 insertions(+), 321 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5ff57a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
index 0581878..9767fb9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
@@ -35,11 +35,11 @@ import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.io.FileUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configurable;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
@@ -68,7 +68,8 @@ import org.apache.hadoop.util.StringUtils;
* underlying OS. All executor implementations must extend ContainerExecutor.
*/
public abstract class ContainerExecutor implements Configurable {
- private static final Log LOG = LogFactory.getLog(ContainerExecutor.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ContainerExecutor.class);
protected static final String WILDCARD = "*";
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5ff57a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
index 18604df..8c58056 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
@@ -20,6 +20,8 @@ package org.apache.hadoop.yarn.server.nodemanager;
import static org.apache.hadoop.fs.CreateFlag.CREATE;
import static org.apache.hadoop.fs.CreateFlag.OVERWRITE;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.io.DataOutputStream;
import java.io.File;
@@ -34,8 +36,6 @@ import java.util.List;
import java.util.Map;
import org.apache.commons.lang.math.RandomUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.FileUtil;
@@ -73,8 +73,8 @@ import com.google.common.base.Optional;
*/
public class DefaultContainerExecutor extends ContainerExecutor {
- private static final Log LOG = LogFactory
- .getLog(DefaultContainerExecutor.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(DefaultContainerExecutor.class);
private static final int WIN_MAX_PATH = 260;
@@ -423,7 +423,7 @@ public class DefaultContainerExecutor extends ContainerExecutor {
pout = new PrintStream(out, false, "UTF-8");
writeLocalWrapperScript(launchDst, pidFile, pout);
} finally {
- IOUtils.cleanup(LOG, pout, out);
+ IOUtils.cleanupWithLogger(LOG, pout, out);
}
}
@@ -505,7 +505,7 @@ public class DefaultContainerExecutor extends ContainerExecutor {
String exec = Shell.isSetsidAvailable? "exec setsid" : "exec";
pout.printf("%s /bin/bash \"%s\"", exec, launchDst.toUri().getPath());
} finally {
- IOUtils.cleanup(LOG, pout, out);
+ IOUtils.cleanupWithLogger(LOG, pout, out);
}
lfs.setPermission(sessionScriptPath,
ContainerExecutor.TASK_LAUNCH_SCRIPT_PERMISSION);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5ff57a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DeletionService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DeletionService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DeletionService.java
index 38d69a3..ae81dc1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DeletionService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DeletionService.java
@@ -19,6 +19,8 @@
package org.apache.hadoop.yarn.server.nodemanager;
import static java.util.concurrent.TimeUnit.SECONDS;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.HashMap;
@@ -31,8 +33,6 @@ import java.util.concurrent.ThreadFactory;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicInteger;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.service.AbstractService;
@@ -49,7 +49,8 @@ import com.google.common.util.concurrent.ThreadFactoryBuilder;
public class DeletionService extends AbstractService {
- private static final Log LOG = LogFactory.getLog(DeletionService.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(DeletionService.class);
private int debugDelay;
private final ContainerExecutor containerExecutor;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5ff57a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DirectoryCollection.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DirectoryCollection.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DirectoryCollection.java
index 502485f..6aec0ff 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DirectoryCollection.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DirectoryCollection.java
@@ -33,11 +33,11 @@ import java.util.concurrent.CopyOnWriteArrayList;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.RandomStringUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.fs.FileAlreadyExistsException;
import org.apache.hadoop.fs.FileContext;
@@ -55,7 +55,8 @@ import com.google.common.annotations.VisibleForTesting;
* Manages a list of local storage directories.
*/
public class DirectoryCollection {
- private static final Log LOG = LogFactory.getLog(DirectoryCollection.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(DirectoryCollection.class);
private final Configuration conf;
private final DiskValidator diskValidator;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5ff57a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
index b3e13b4..dc68680 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
@@ -20,8 +20,8 @@ package org.apache.hadoop.yarn.server.nodemanager;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Optional;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.security.UserGroupInformation;
@@ -98,8 +98,8 @@ import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.r
*/
public class LinuxContainerExecutor extends ContainerExecutor {
- private static final Log LOG = LogFactory
- .getLog(LinuxContainerExecutor.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(LinuxContainerExecutor.class);
private String nonsecureLocalUser;
private Pattern nonsecureLocalUserPattern;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5ff57a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java
index 6e00808..c0630b2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java
@@ -27,9 +27,9 @@ import java.util.List;
import java.util.Set;
import java.util.Timer;
import java.util.TimerTask;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileContext;
@@ -51,7 +51,8 @@ import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
*/
public class LocalDirsHandlerService extends AbstractService {
- private static Log LOG = LogFactory.getLog(LocalDirsHandlerService.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(LocalDirsHandlerService.class);
private static final String diskCapacityExceededErrorMsg = "usable space is below configured utilization percentage/no more usable space";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5ff57a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NMAuditLogger.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NMAuditLogger.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NMAuditLogger.java
index 279e299..3083064 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NMAuditLogger.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NMAuditLogger.java
@@ -18,9 +18,9 @@
package org.apache.hadoop.yarn.server.nodemanager;
import java.net.InetAddress;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.ipc.Server;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
@@ -31,7 +31,8 @@ import org.apache.hadoop.yarn.api.records.ContainerId;
* Audit log format is written as key=value pairs. Tab separated.
*/
public class NMAuditLogger {
- private static final Log LOG = LogFactory.getLog(NMAuditLogger.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(NMAuditLogger.class);
enum Keys {USER, OPERATION, TARGET, RESULT, IP,
DESCRIPTION, APPID, CONTAINERID}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5ff57a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
index 3c0e498..bf4b43c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
@@ -26,9 +26,9 @@ import java.util.concurrent.ConcurrentLinkedQueue;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ConcurrentSkipListMap;
import java.util.concurrent.atomic.AtomicBoolean;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
@@ -105,7 +105,8 @@ public class NodeManager extends CompositeService
*/
public static final int SHUTDOWN_HOOK_PRIORITY = 30;
- private static final Log LOG = LogFactory.getLog(NodeManager.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(NodeManager.class);
private static long nmStartupTime = System.currentTimeMillis();
protected final NodeManagerMetrics metrics = NodeManagerMetrics.create();
private JvmPauseMonitor pauseMonitor;
@@ -469,7 +470,7 @@ public class NodeManager extends CompositeService
((NodeStatusUpdaterImpl) nodeStatusUpdater)
.rebootNodeStatusUpdaterAndRegisterWithRM();
} catch (YarnRuntimeException e) {
- LOG.fatal("Error while rebooting NodeStatusUpdater.", e);
+ LOG.error("Error while rebooting NodeStatusUpdater.", e);
shutDown(NodeManagerStatus.EXCEPTION.getExitCode());
}
}
@@ -729,7 +730,7 @@ public class NodeManager extends CompositeService
String message =
"Failing NodeManager start since we're on a "
+ "Unix-based system but bash doesn't seem to be available.";
- LOG.fatal(message);
+ LOG.error(message);
throw new YarnRuntimeException(message);
}
}
@@ -748,7 +749,7 @@ public class NodeManager extends CompositeService
this.init(conf);
this.start();
} catch (Throwable t) {
- LOG.fatal("Error starting NodeManager", t);
+ LOG.error("Error starting NodeManager", t);
System.exit(-1);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5ff57a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeResourceMonitorImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeResourceMonitorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeResourceMonitorImpl.java
index e1116da..8b96ba5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeResourceMonitorImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeResourceMonitorImpl.java
@@ -18,13 +18,13 @@
package org.apache.hadoop.yarn.server.nodemanager;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.service.AbstractService;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.api.records.ResourceUtilization;
import org.apache.hadoop.yarn.util.ResourceCalculatorPlugin;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Implementation of the node resource monitor. It periodically tracks the
@@ -34,8 +34,8 @@ public class NodeResourceMonitorImpl extends AbstractService implements
NodeResourceMonitor {
/** Logging infrastructure. */
- final static Log LOG = LogFactory
- .getLog(NodeResourceMonitorImpl.class);
+ final static Logger LOG =
+ LoggerFactory.getLogger(NodeResourceMonitorImpl.class);
/** Interval to monitor the node resource utilization. */
private long monitoringInterval;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5ff57a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
index ade42e3..3b46545 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
@@ -33,9 +33,9 @@ import java.util.Map.Entry;
import java.util.Random;
import java.util.Set;
import java.util.concurrent.ConcurrentLinkedQueue;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.DataInputByteBuffer;
@@ -99,7 +99,8 @@ public class NodeStatusUpdaterImpl extends AbstractService implements
public static final String YARN_NODEMANAGER_DURATION_TO_TRACK_STOPPED_CONTAINERS =
YarnConfiguration.NM_PREFIX + "duration-to-track-stopped-containers";
- private static final Log LOG = LogFactory.getLog(NodeStatusUpdaterImpl.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(NodeStatusUpdaterImpl.class);
private final Object heartbeatMonitor = new Object();
private final Object shutdownMonitor = new Object();
@@ -427,7 +428,7 @@ public class NodeStatusUpdaterImpl extends AbstractService implements
successfullRegistrationMsg.append(nodeLabelsHandler
.verifyRMRegistrationResponseForNodeLabels(regNMResponse));
- LOG.info(successfullRegistrationMsg);
+ LOG.info(successfullRegistrationMsg.toString());
}
private List<ApplicationId> createKeepAliveApplicationList() {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5ff57a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyTokenSecretManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyTokenSecretManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyTokenSecretManager.java
index aa3c70f..f36d4da 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyTokenSecretManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/AMRMProxyTokenSecretManager.java
@@ -27,9 +27,9 @@ import java.util.TimerTask;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.Text;
@@ -52,8 +52,8 @@ import com.google.common.annotations.VisibleForTesting;
public class AMRMProxyTokenSecretManager extends
SecretManager<AMRMTokenIdentifier> {
- private static final Log LOG = LogFactory
- .getLog(AMRMProxyTokenSecretManager.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(AMRMProxyTokenSecretManager.class);
private int serialNo = new SecureRandom().nextInt();
private MasterKeyData nextMasterKey;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5ff57a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/impl/pb/NMProtoUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/impl/pb/NMProtoUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/impl/pb/NMProtoUtils.java
index e47b3ee..f9b762a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/impl/pb/NMProtoUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/impl/pb/NMProtoUtils.java
@@ -16,8 +16,6 @@
*/
package org.apache.hadoop.yarn.server.nodemanager.api.impl.pb;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.proto.YarnServerNodemanagerRecoveryProtos.DeletionServiceDeleteTaskProto;
import org.apache.hadoop.yarn.server.nodemanager.DeletionService;
@@ -25,6 +23,8 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.deletion.recov
import org.apache.hadoop.yarn.server.nodemanager.containermanager.deletion.task.DeletionTask;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.deletion.task.DeletionTaskType;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.deletion.task.FileDeletionTask;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.List;
@@ -34,7 +34,8 @@ import java.util.List;
*/
public final class NMProtoUtils {
- private static final Log LOG = LogFactory.getLog(NMProtoUtils.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(NMProtoUtils.class);
private NMProtoUtils() { }
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5ff57a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/collectormanager/NMCollectorService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/collectormanager/NMCollectorService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/collectormanager/NMCollectorService.java
index d667c0e..e52e1ec 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/collectormanager/NMCollectorService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/collectormanager/NMCollectorService.java
@@ -22,9 +22,9 @@ import java.net.InetSocketAddress;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.ipc.Server;
import org.apache.hadoop.service.CompositeService;
@@ -50,7 +50,8 @@ import org.apache.hadoop.yarn.server.nodemanager.timelineservice.NMTimelinePubli
public class NMCollectorService extends CompositeService implements
CollectorNodemanagerProtocol {
- private static final Log LOG = LogFactory.getLog(NMCollectorService.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(NMCollectorService.class);
private final Context context;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5ff57a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java
index c0e1f5a..2efc932 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/AuxServices.java
@@ -25,9 +25,9 @@ import java.util.HashMap;
import java.util.Map;
import java.util.Map.Entry;
import java.util.regex.Pattern;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -51,7 +51,8 @@ public class AuxServices extends AbstractService
static final String STATE_STORE_ROOT_NAME = "nm-aux-services";
- private static final Log LOG = LogFactory.getLog(AuxServices.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(AuxServices.class);
protected final Map<String,AuxiliaryService> serviceMap;
protected final Map<String,ByteBuffer> serviceMetaData;
@@ -161,7 +162,7 @@ public class AuxServices extends AbstractService
}
s.init(conf);
} catch (RuntimeException e) {
- LOG.fatal("Failed to initialize " + sName, e);
+ LOG.error("Failed to initialize " + sName, e);
throw e;
}
}
@@ -205,7 +206,7 @@ public class AuxServices extends AbstractService
@Override
public void stateChanged(Service service) {
- LOG.fatal("Service " + service.getName() + " changed state: " +
+ LOG.error("Service " + service.getName() + " changed state: " +
service.getServiceState());
stop();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5ff57a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
index 22484b7..ef36ba6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
@@ -20,8 +20,8 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager;
import com.google.common.annotations.VisibleForTesting;
import com.google.protobuf.ByteString;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
@@ -189,7 +189,8 @@ public class ContainerManagerImpl extends CompositeService implements
*/
private static final int SHUTDOWN_CLEANUP_SLOP_MS = 1000;
- private static final Log LOG = LogFactory.getLog(ContainerManagerImpl.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ContainerManagerImpl.class);
public static final String INVALID_NMTOKEN_MSG = "Invalid NMToken";
static final String INVALID_CONTAINERTOKEN_MSG =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5ff57a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java
index 80863a1..dd20071 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java
@@ -25,10 +25,10 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock.ReadLock;
import java.util.concurrent.locks.ReentrantReadWriteLock.WriteLock;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import com.google.protobuf.ByteString;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.io.DataOutputBuffer;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.Credentials;
@@ -85,7 +85,8 @@ public class ApplicationImpl implements Application {
private final WriteLock writeLock;
private final Context context;
- private static final Log LOG = LogFactory.getLog(ApplicationImpl.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ApplicationImpl.class);
private LogAggregationContext logAggregationContext;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5ff57a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
index 734a27b..8e42133 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
@@ -33,10 +33,10 @@ import java.util.Set;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import com.google.common.annotations.VisibleForTesting;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.security.Credentials;
@@ -174,7 +174,8 @@ public class ContainerImpl implements Container {
/** The NM-wide configuration - not specific to this container */
private final Configuration daemonConf;
- private static final Log LOG = LogFactory.getLog(ContainerImpl.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ContainerImpl.class);
// whether container has been recovered after a restart
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5ff57a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/deletion/task/DeletionTask.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/deletion/task/DeletionTask.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/deletion/task/DeletionTask.java
index 635d7a9..f4353ac 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/deletion/task/DeletionTask.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/deletion/task/DeletionTask.java
@@ -16,11 +16,11 @@
*/
package org.apache.hadoop.yarn.server.nodemanager.containermanager.deletion.task;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.yarn.proto.YarnServerNodemanagerRecoveryProtos.DeletionServiceDeleteTaskProto;
import org.apache.hadoop.yarn.server.nodemanager.DeletionService;
import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.HashSet;
@@ -34,7 +34,8 @@ import java.util.concurrent.atomic.AtomicInteger;
*/
public abstract class DeletionTask implements Runnable {
- static final Log LOG = LogFactory.getLog(DeletionTask.class);
+ static final Logger LOG =
+ LoggerFactory.getLogger(DeletionTask.class);
public static final int INVALID_TASK_ID = -1;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5ff57a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
index 0b599a8..a0055c5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
@@ -20,6 +20,8 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher;
import static org.apache.hadoop.fs.CreateFlag.CREATE;
import static org.apache.hadoop.fs.CreateFlag.OVERWRITE;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.io.DataOutputStream;
import java.io.File;
@@ -36,8 +38,6 @@ import java.util.Map.Entry;
import java.util.concurrent.Callable;
import java.util.concurrent.atomic.AtomicBoolean;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileContext;
@@ -89,7 +89,8 @@ import com.google.common.annotations.VisibleForTesting;
public class ContainerLaunch implements Callable<Integer> {
- private static final Log LOG = LogFactory.getLog(ContainerLaunch.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ContainerLaunch.class);
public static final String CONTAINER_SCRIPT =
Shell.appendScriptExtension("launch_container");
@@ -269,7 +270,8 @@ public class ContainerLaunch implements Callable<Integer> {
creds.writeTokenStorageToStream(tokensOutStream);
// /////////// End of writing out container-tokens
} finally {
- IOUtils.cleanup(LOG, containerScriptOutStream, tokensOutStream);
+ IOUtils.cleanupWithLogger(LOG, containerScriptOutStream,
+ tokensOutStream);
}
ret = launchContainer(new ContainerStartContext.Builder()
@@ -518,7 +520,7 @@ public class ContainerLaunch implements Callable<Integer> {
@SuppressWarnings("unchecked")
protected void handleContainerExitWithFailure(ContainerId containerID,
int ret, Path containerLogDir, StringBuilder diagnosticInfo) {
- LOG.warn(diagnosticInfo);
+ LOG.warn(diagnosticInfo.toString());
String errorFileNamePattern =
conf.get(YarnConfiguration.NM_CONTAINER_STDERR_PATTERN,
@@ -569,7 +571,7 @@ public class ContainerLaunch implements Callable<Integer> {
} catch (IOException e) {
LOG.error("Failed to get tail of the container's error log file", e);
} finally {
- IOUtils.cleanup(LOG, errorFileIS);
+ IOUtils.cleanupWithLogger(LOG, errorFileIS);
}
this.dispatcher.getEventHandler()
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5ff57a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerRelaunch.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerRelaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerRelaunch.java
index ac20fa8..2b032e9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerRelaunch.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerRelaunch.java
@@ -18,8 +18,6 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
@@ -38,6 +36,8 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.Cont
import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerStartContext;
import org.apache.hadoop.yarn.server.nodemanager.executor.DeletionAsUserContext;
import org.apache.hadoop.yarn.util.ConverterUtils;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.List;
@@ -48,7 +48,8 @@ import java.util.Map;
*/
public class ContainerRelaunch extends ContainerLaunch {
- private static final Log LOG = LogFactory.getLog(ContainerRelaunch.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ContainerRelaunch.class);
public ContainerRelaunch(Context context, Configuration configuration,
Dispatcher dispatcher, ContainerExecutor exec, Application app,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5ff57a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java
index d4a7bfd..25909b9 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java
@@ -23,9 +23,9 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.ExecutorService;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.UnsupportedFileSystemException;
@@ -55,7 +55,8 @@ import com.google.common.util.concurrent.ThreadFactoryBuilder;
public class ContainersLauncher extends AbstractService
implements EventHandler<ContainersLauncherEvent> {
- private static final Log LOG = LogFactory.getLog(ContainersLauncher.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ContainersLauncher.class);
private final Context context;
private final ContainerExecutor exec;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5ff57a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/RecoveredContainerLaunch.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/RecoveredContainerLaunch.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/RecoveredContainerLaunch.java
index a04a23f..2eba0df 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/RecoveredContainerLaunch.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/RecoveredContainerLaunch.java
@@ -21,9 +21,9 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher;
import java.io.File;
import java.io.InterruptedIOException;
import java.io.IOException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.api.records.ContainerId;
@@ -47,8 +47,8 @@ import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerReacquisition
*/
public class RecoveredContainerLaunch extends ContainerLaunch {
- private static final Log LOG = LogFactory.getLog(
- RecoveredContainerLaunch.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(RecoveredContainerLaunch.class);
public RecoveredContainerLaunch(Context context, Configuration configuration,
Dispatcher dispatcher, ContainerExecutor exec, Application app,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5ff57a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/privileged/PrivilegedOperationExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/privileged/PrivilegedOperationExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/privileged/PrivilegedOperationExecutor.java
index 1d874a7..5a3ce74 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/privileged/PrivilegedOperationExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/privileged/PrivilegedOperationExecutor.java
@@ -22,8 +22,8 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privile
import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.lang.StringUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
@@ -46,7 +46,8 @@ import java.util.Map;
@InterfaceAudience.Private
@InterfaceStability.Unstable
public class PrivilegedOperationExecutor {
- private static final Log LOG = LogFactory.getLog(PrivilegedOperationExecutor
+ private static final Logger LOG =
+ LoggerFactory.getLogger(PrivilegedOperationExecutor
.class);
private volatile static PrivilegedOperationExecutor instance;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5ff57a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsBlkioResourceHandlerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsBlkioResourceHandlerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsBlkioResourceHandlerImpl.java
index e0b43d3..42fc634 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsBlkioResourceHandlerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsBlkioResourceHandlerImpl.java
@@ -19,8 +19,8 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources;
import com.google.common.annotations.VisibleForTesting;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
@@ -45,8 +45,8 @@ import java.util.List;
@InterfaceStability.Unstable
public class CGroupsBlkioResourceHandlerImpl implements DiskResourceHandler {
- static final Log LOG = LogFactory
- .getLog(CGroupsBlkioResourceHandlerImpl.class);
+ static final Logger LOG =
+ LoggerFactory.getLogger(CGroupsBlkioResourceHandlerImpl.class);
private CGroupsHandler cGroupsHandler;
// Arbitrarily choose a weight - all that matters is that all containers
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5ff57a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsCpuResourceHandlerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsCpuResourceHandlerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsCpuResourceHandlerImpl.java
index 830782d..7ea7be2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsCpuResourceHandlerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsCpuResourceHandlerImpl.java
@@ -20,8 +20,8 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resourc
import com.google.common.annotations.VisibleForTesting;
import org.apache.commons.io.FileUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
@@ -59,7 +59,8 @@ import java.util.List;
@InterfaceAudience.Private
public class CGroupsCpuResourceHandlerImpl implements CpuResourceHandler {
- static final Log LOG = LogFactory.getLog(CGroupsCpuResourceHandlerImpl.class);
+ static final Logger LOG =
+ LoggerFactory.getLogger(CGroupsCpuResourceHandlerImpl.class);
private CGroupsHandler cGroupsHandler;
private boolean strictResourceUsageMode = false;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5ff57a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandlerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandlerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandlerImpl.java
index 9fd20eb..44ae12b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandlerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsHandlerImpl.java
@@ -22,8 +22,8 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resourc
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Joiner;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
@@ -59,7 +59,8 @@ import java.util.regex.Pattern;
@InterfaceStability.Unstable
class CGroupsHandlerImpl implements CGroupsHandler {
- private static final Log LOG = LogFactory.getLog(CGroupsHandlerImpl.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(CGroupsHandlerImpl.class);
private static final String MTAB_FILE = "/proc/mounts";
private static final String CGROUPS_FSTYPE = "cgroup";
@@ -243,7 +244,7 @@ class CGroupsHandlerImpl implements CGroupsHandler {
LOG.warn("Error while reading " + mtab, e);
}
} finally {
- IOUtils.cleanup(LOG, in);
+ IOUtils.cleanupWithLogger(LOG, in);
}
return ret;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5ff57a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsMemoryResourceHandlerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsMemoryResourceHandlerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsMemoryResourceHandlerImpl.java
index d159aad..d3e787e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsMemoryResourceHandlerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/CGroupsMemoryResourceHandlerImpl.java
@@ -19,8 +19,8 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources;
import com.google.common.annotations.VisibleForTesting;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
@@ -44,8 +44,8 @@ import java.util.List;
@InterfaceStability.Unstable
public class CGroupsMemoryResourceHandlerImpl implements MemoryResourceHandler {
- static final Log LOG = LogFactory.getLog(
- CGroupsMemoryResourceHandlerImpl.class);
+ static final Logger LOG =
+ LoggerFactory.getLogger(CGroupsMemoryResourceHandlerImpl.class);
private static final CGroupsHandler.CGroupController MEMORY =
CGroupsHandler.CGroupController.MEMORY;
private static final int OPPORTUNISTIC_SWAPPINESS = 100;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5ff57a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/ResourceHandlerModule.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/ResourceHandlerModule.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/ResourceHandlerModule.java
index 4d137f0..a81a77b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/ResourceHandlerModule.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/ResourceHandlerModule.java
@@ -21,8 +21,8 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources;
import com.google.common.annotations.VisibleForTesting;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
@@ -49,7 +49,8 @@ import java.util.List;
@InterfaceAudience.Private
@InterfaceStability.Unstable
public class ResourceHandlerModule {
- static final Log LOG = LogFactory.getLog(ResourceHandlerModule.class);
+ static final Logger LOG =
+ LoggerFactory.getLogger(ResourceHandlerModule.class);
private static volatile ResourceHandlerChain resourceHandlerChain;
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5ff57a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TrafficControlBandwidthHandlerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TrafficControlBandwidthHandlerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TrafficControlBandwidthHandlerImpl.java
index 3bb8035..126685f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TrafficControlBandwidthHandlerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TrafficControlBandwidthHandlerImpl.java
@@ -20,8 +20,6 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
@@ -31,6 +29,8 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Cont
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperation;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationException;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationExecutor;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.util.ArrayList;
import java.util.HashMap;
@@ -43,8 +43,8 @@ import java.util.concurrent.ConcurrentHashMap;
public class TrafficControlBandwidthHandlerImpl
implements OutboundBandwidthResourceHandler {
- private static final Log LOG = LogFactory
- .getLog(TrafficControlBandwidthHandlerImpl.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TrafficControlBandwidthHandlerImpl.class);
//In the absence of 'scheduling' support, we'll 'infer' the guaranteed
//outbound bandwidth for each container based on this number. This will
//likely go away once we add support on the RM for this resource type.
@@ -117,7 +117,7 @@ public class TrafficControlBandwidthHandlerImpl
.append("containerBandwidthMbit soft limit (in mbit/sec) is set to : ")
.append(containerBandwidthMbit);
- LOG.info(logLine);
+ LOG.info(logLine.toString());
trafficController.bootstrap(device, rootBandwidthMbit, yarnBandwidthMbit);
return null;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5ff57a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TrafficController.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TrafficController.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TrafficController.java
index 4698eb3..83db5fc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TrafficController.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TrafficController.java
@@ -20,8 +20,6 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
@@ -29,6 +27,8 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperation;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationException;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationExecutor;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.io.*;
import java.util.ArrayList;
@@ -46,7 +46,8 @@ import java.util.regex.Pattern;
@InterfaceAudience.Private
@InterfaceStability.Unstable class TrafficController {
- private static final Log LOG = LogFactory.getLog(TrafficController.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TrafficController.class);
private static final int ROOT_QDISC_HANDLE = 42;
private static final int ZERO_CLASS_ID = 0;
private static final int ROOT_CLASS_ID = 1;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5ff57a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java
index ee94aaa..d09e4a1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DefaultLinuxContainerRuntime.java
@@ -20,8 +20,6 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
@@ -34,6 +32,8 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileg
import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerExecutionException;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerRuntime;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerRuntimeContext;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.util.List;
@@ -48,8 +48,8 @@ import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.r
@InterfaceAudience.Private
@InterfaceStability.Unstable
public class DefaultLinuxContainerRuntime implements LinuxContainerRuntime {
- private static final Log LOG =
- LogFactory.getLog(DefaultLinuxContainerRuntime.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(DefaultLinuxContainerRuntime.class);
private final PrivilegedOperationExecutor privilegedOperationExecutor;
private Configuration conf;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5ff57a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DelegatingLinuxContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DelegatingLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DelegatingLinuxContainerRuntime.java
index 90b13a2..5273334 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DelegatingLinuxContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DelegatingLinuxContainerRuntime.java
@@ -20,8 +20,6 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
@@ -30,6 +28,8 @@ import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileg
import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerExecutionException;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerRuntime;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerRuntimeContext;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.util.Map;
@@ -45,8 +45,8 @@ import java.util.Map;
@InterfaceAudience.Private
@InterfaceStability.Unstable
public class DelegatingLinuxContainerRuntime implements LinuxContainerRuntime {
- private static final Log LOG = LogFactory
- .getLog(DelegatingLinuxContainerRuntime.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(DelegatingLinuxContainerRuntime.class);
private DefaultLinuxContainerRuntime defaultLinuxContainerRuntime;
private DockerLinuxContainerRuntime dockerLinuxContainerRuntime;
private JavaSandboxLinuxContainerRuntime javaSandboxLinuxContainerRuntime;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5ff57a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
index e058d6e..8217564 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/DockerLinuxContainerRuntime.java
@@ -21,8 +21,8 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime;
import com.google.common.annotations.VisibleForTesting;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
@@ -132,8 +132,8 @@ import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.r
@InterfaceAudience.Private
@InterfaceStability.Unstable
public class DockerLinuxContainerRuntime implements LinuxContainerRuntime {
- private static final Log LOG = LogFactory.getLog(
- DockerLinuxContainerRuntime.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(DockerLinuxContainerRuntime.class);
// This validates that the image is a proper docker image
public static final String DOCKER_IMAGE_PATTERN =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5ff57a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/JavaSandboxLinuxContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/JavaSandboxLinuxContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/JavaSandboxLinuxContainerRuntime.java
index 0b858bc..cfafcde 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/JavaSandboxLinuxContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/JavaSandboxLinuxContainerRuntime.java
@@ -19,8 +19,6 @@
package org.apache.hadoop.yarn.server.nodemanager.
containermanager.linux.runtime;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
@@ -30,7 +28,8 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationExecutor;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerExecutionException;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerRuntimeContext;
-import org.apache.log4j.Logger;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.io.FilePermission;
import java.io.IOException;
@@ -117,8 +116,8 @@ import static org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.r
@InterfaceStability.Unstable
public class JavaSandboxLinuxContainerRuntime
extends DefaultLinuxContainerRuntime {
- private static final Log LOG =
- LogFactory.getLog(DefaultLinuxContainerRuntime.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(DefaultLinuxContainerRuntime.class);
private Configuration configuration;
private SandboxMode sandboxMode;
@@ -254,7 +253,7 @@ public class JavaSandboxLinuxContainerRuntime
} catch (IOException e) {
throw new ContainerExecutionException(e);
} finally {
- IOUtils.cleanup(LOG, policyOutputStream);
+ IOUtils.cleanupWithLogger(LOG, policyOutputStream);
}
}
}
@@ -417,7 +416,7 @@ public class JavaSandboxLinuxContainerRuntime
+ SEPARATOR + "-\" {%n" +
" permission " + AllPermission.class.getCanonicalName() + ";%n};%n";
static final Logger LOG =
- Logger.getLogger(NMContainerPolicyUtils.class);
+ LoggerFactory.getLogger(NMContainerPolicyUtils.class);
/**
* Write new policy file to policyOutStream which will include read access
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5ff57a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerClient.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerClient.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerClient.java
index faf955f..536a22d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerClient.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerClient.java
@@ -20,13 +20,13 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.ResourceHandlerException;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerExecutionException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.FileOutputStream;
@@ -38,7 +38,8 @@ import java.io.Writer;
@InterfaceAudience.Private
@InterfaceStability.Unstable
public final class DockerClient {
- private static final Log LOG = LogFactory.getLog(DockerClient.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(DockerClient.class);
private static final String TMP_FILE_PREFIX = "docker.";
private static final String TMP_FILE_SUFFIX = ".cmd";
private final String tmpDirPath;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[28/50] [abbrv] hadoop git commit: HDFS-12325. SFTPFileSystem
operations should restore cwd. Contributed by Chen Liang.
Posted by as...@apache.org.
HDFS-12325. SFTPFileSystem operations should restore cwd. Contributed by Chen Liang.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/736ceab2
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/736ceab2
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/736ceab2
Branch: refs/heads/YARN-5972
Commit: 736ceab2f58fb9ab5907c5b5110bd44384038e6b
Parents: 913760c
Author: Arpit Agarwal <ar...@apache.org>
Authored: Sun Aug 20 23:41:06 2017 -0700
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Mon Aug 21 11:48:51 2017 -0700
----------------------------------------------------------------------
.../main/java/org/apache/hadoop/fs/sftp/SFTPFileSystem.java | 6 ++++++
1 file changed, 6 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/736ceab2/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/sftp/SFTPFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/sftp/SFTPFileSystem.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/sftp/SFTPFileSystem.java
index 421769d..43eb783 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/sftp/SFTPFileSystem.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/sftp/SFTPFileSystem.java
@@ -326,8 +326,10 @@ public class SFTPFileSystem extends FileSystem {
String parentDir = parent.toUri().getPath();
boolean succeeded = true;
try {
+ final String previousCwd = client.pwd();
client.cd(parentDir);
client.mkdir(pathName);
+ client.cd(previousCwd);
} catch (SftpException e) {
throw new IOException(String.format(E_MAKE_DIR_FORPATH, pathName,
parentDir));
@@ -474,8 +476,10 @@ public class SFTPFileSystem extends FileSystem {
}
boolean renamed = true;
try {
+ final String previousCwd = channel.pwd();
channel.cd("/");
channel.rename(src.toUri().getPath(), dst.toUri().getPath());
+ channel.cd(previousCwd);
} catch (SftpException e) {
renamed = false;
}
@@ -558,8 +562,10 @@ public class SFTPFileSystem extends FileSystem {
}
OutputStream os;
try {
+ final String previousCwd = client.pwd();
client.cd(parent.toUri().getPath());
os = client.put(f.getName());
+ client.cd(previousCwd);
} catch (SftpException e) {
throw new IOException(e);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[22/50] [abbrv] hadoop git commit: YARN-6969. Clean up unused code in
class FairSchedulerQueueInfo. (Larry Lo via Yufei Gu)
Posted by as...@apache.org.
YARN-6969. Clean up unused code in class FairSchedulerQueueInfo. (Larry Lo via Yufei Gu)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/8991f0ba
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/8991f0ba
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/8991f0ba
Branch: refs/heads/YARN-5972
Commit: 8991f0baec62625c45144e2544066195800ab95b
Parents: 2d105a2
Author: Yufei Gu <yu...@apache.org>
Authored: Fri Aug 18 14:38:44 2017 -0700
Committer: Yufei Gu <yu...@apache.org>
Committed: Fri Aug 18 14:38:44 2017 -0700
----------------------------------------------------------------------
.../webapp/dao/FairSchedulerQueueInfo.java | 17 -----------------
1 file changed, 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/8991f0ba/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java
index 79339c7..913513c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/dao/FairSchedulerQueueInfo.java
@@ -48,8 +48,6 @@ public class FairSchedulerQueueInfo {
@XmlTransient
private float fractionMemFairShare;
@XmlTransient
- private float fractionMemMinShare;
- @XmlTransient
private float fractionMemMaxShare;
private ResourceInfo minResources;
@@ -63,7 +61,6 @@ public class FairSchedulerQueueInfo {
private ResourceInfo clusterResources;
private ResourceInfo reservedResources;
- private long pendingContainers;
private long allocatedContainers;
private long reservedContainers;
@@ -108,12 +105,10 @@ public class FairSchedulerQueueInfo {
(float)steadyFairResources.getMemorySize() / clusterResources.getMemorySize();
fractionMemFairShare = (float) fairResources.getMemorySize()
/ clusterResources.getMemorySize();
- fractionMemMinShare = (float)minResources.getMemorySize() / clusterResources.getMemorySize();
fractionMemMaxShare = (float)maxResources.getMemorySize() / clusterResources.getMemorySize();
maxApps = queue.getMaxRunningApps();
- pendingContainers = queue.getMetrics().getPendingContainers();
allocatedContainers = queue.getMetrics().getAllocatedContainers();
reservedContainers = queue.getMetrics().getReservedContainers();
@@ -126,10 +121,6 @@ public class FairSchedulerQueueInfo {
childQueues = getChildQueues(queue, scheduler);
}
- public long getPendingContainers() {
- return pendingContainers;
- }
-
public long getAllocatedContainers() {
return allocatedContainers;
}
@@ -234,14 +225,6 @@ public class FairSchedulerQueueInfo {
}
/**
- * Returns the queue's min share in as a fraction of the entire
- * cluster capacity.
- */
- public float getMinShareMemoryFraction() {
- return fractionMemMinShare;
- }
-
- /**
* Returns the memory used by this queue as a fraction of the entire
* cluster capacity.
*/
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[27/50] [abbrv] hadoop git commit: HDFS-11988. Verify HDFS Snapshots
with open files captured are consistent across truncates and appends to
current version file.
Posted by as...@apache.org.
HDFS-11988. Verify HDFS Snapshots with open files captured are consistent across truncates and appends to current version file.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/913760cb
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/913760cb
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/913760cb
Branch: refs/heads/YARN-5972
Commit: 913760cb4fe7123e55004800f75dc00540a79f69
Parents: 267e19a
Author: Manoj Govindassamy <ma...@apache.org>
Authored: Mon Aug 21 11:08:38 2017 -0700
Committer: Manoj Govindassamy <ma...@apache.org>
Committed: Mon Aug 21 11:08:38 2017 -0700
----------------------------------------------------------------------
.../snapshot/TestOpenFilesWithSnapshot.java | 112 +++++++++++++++++++
1 file changed, 112 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/913760cb/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestOpenFilesWithSnapshot.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestOpenFilesWithSnapshot.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestOpenFilesWithSnapshot.java
index bf27f2c..537612c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestOpenFilesWithSnapshot.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestOpenFilesWithSnapshot.java
@@ -30,6 +30,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileChecksum;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSOutputStream;
@@ -731,6 +732,117 @@ public class TestOpenFilesWithSnapshot {
cluster.waitActive();
}
+ /**
+ * Verify snapshots with open files captured are safe even when the
+ * 'current' version of the file is truncated and appended later.
+ */
+ @Test (timeout = 120000)
+ public void testOpenFilesSnapChecksumWithTrunkAndAppend() throws Exception {
+ conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_SNAPSHOT_CAPTURE_OPENFILES,
+ true);
+ // Construct the directory tree
+ final Path dir = new Path("/A/B/C");
+ fs.mkdirs(dir);
+
+ // String constants
+ final Path hbaseSnapRootDir = dir;
+ final String hbaseFileName = "hbase.wal";
+ final String hbaseSnap1Name = "hbase_snap_s1";
+ final String hbaseSnap2Name = "hbase_snap_s2";
+ final String hbaseSnap3Name = "hbase_snap_s3";
+ final String hbaseSnap4Name = "hbase_snap_s4";
+
+ // Create files and open a stream
+ final Path hbaseFile = new Path(dir, hbaseFileName);
+ createFile(hbaseFile);
+ final FileChecksum hbaseWALFileCksum0 =
+ fs.getFileChecksum(hbaseFile);
+ FSDataOutputStream hbaseOutputStream = fs.append(hbaseFile);
+
+ // Create Snapshot S1
+ final Path hbaseS1Dir = SnapshotTestHelper.createSnapshot(
+ fs, hbaseSnapRootDir, hbaseSnap1Name);
+ final Path hbaseS1Path = new Path(hbaseS1Dir, hbaseFileName);
+ final FileChecksum hbaseFileCksumS1 = fs.getFileChecksum(hbaseS1Path);
+
+ // Verify if Snap S1 checksum is same as the current version one
+ Assert.assertEquals("Live and snap1 file checksum doesn't match!",
+ hbaseWALFileCksum0, fs.getFileChecksum(hbaseS1Path));
+
+ int newWriteLength = (int) (BLOCKSIZE * 1.5);
+ byte[] buf = new byte[newWriteLength];
+ Random random = new Random();
+ random.nextBytes(buf);
+ writeToStream(hbaseOutputStream, buf);
+
+ // Create Snapshot S2
+ final Path hbaseS2Dir = SnapshotTestHelper.createSnapshot(
+ fs, hbaseSnapRootDir, hbaseSnap2Name);
+ final Path hbaseS2Path = new Path(hbaseS2Dir, hbaseFileName);
+ final FileChecksum hbaseFileCksumS2 = fs.getFileChecksum(hbaseS2Path);
+
+ // Verify if the s1 checksum is still the same
+ Assert.assertEquals("Snap file checksum has changed!",
+ hbaseFileCksumS1, fs.getFileChecksum(hbaseS1Path));
+ // Verify if the s2 checksum is different from the s1 checksum
+ Assert.assertNotEquals("Snap1 and snap2 file checksum should differ!",
+ hbaseFileCksumS1, hbaseFileCksumS2);
+
+ newWriteLength = (int) (BLOCKSIZE * 2.5);
+ buf = new byte[newWriteLength];
+ random.nextBytes(buf);
+ writeToStream(hbaseOutputStream, buf);
+
+ // Create Snapshot S3
+ final Path hbaseS3Dir = SnapshotTestHelper.createSnapshot(
+ fs, hbaseSnapRootDir, hbaseSnap3Name);
+ final Path hbaseS3Path = new Path(hbaseS3Dir, hbaseFileName);
+ FileChecksum hbaseFileCksumS3 = fs.getFileChecksum(hbaseS3Path);
+
+ // Record the checksum for the before truncate current file
+ hbaseOutputStream.close();
+ final FileChecksum hbaseFileCksumBeforeTruncate =
+ fs.getFileChecksum(hbaseFile);
+ Assert.assertEquals("Snap3 and before truncate file checksum should match!",
+ hbaseFileCksumBeforeTruncate, hbaseFileCksumS3);
+
+ // Truncate the current file and record the after truncate checksum
+ long currentFileLen = fs.getFileStatus(hbaseFile).getLen();
+ boolean fileTruncated = fs.truncate(hbaseFile, currentFileLen / 2);
+ Assert.assertTrue("File truncation failed!", fileTruncated);
+ final FileChecksum hbaseFileCksumAfterTruncate =
+ fs.getFileChecksum(hbaseFile);
+
+ Assert.assertNotEquals("Snap3 and after truncate checksum shouldn't match!",
+ hbaseFileCksumS3, hbaseFileCksumAfterTruncate);
+
+ // Append more data to the current file
+ hbaseOutputStream = fs.append(hbaseFile);
+ newWriteLength = (int) (BLOCKSIZE * 5.5);
+ buf = new byte[newWriteLength];
+ random.nextBytes(buf);
+ writeToStream(hbaseOutputStream, buf);
+
+ // Create Snapshot S4
+ final Path hbaseS4Dir = SnapshotTestHelper.createSnapshot(
+ fs, hbaseSnapRootDir, hbaseSnap4Name);
+ final Path hbaseS4Path = new Path(hbaseS4Dir, hbaseFileName);
+ final FileChecksum hbaseFileCksumS4 = fs.getFileChecksum(hbaseS4Path);
+
+ // Record the checksum for the current file after append
+ hbaseOutputStream.close();
+ final FileChecksum hbaseFileCksumAfterAppend =
+ fs.getFileChecksum(hbaseFile);
+
+ Assert.assertEquals("Snap4 and after append file checksum should match!",
+ hbaseFileCksumAfterAppend, hbaseFileCksumS4);
+
+ // Recompute checksum for S3 path and verify it has not changed
+ hbaseFileCksumS3 = fs.getFileChecksum(hbaseS3Path);
+ Assert.assertEquals("Snap3 and before truncate file checksum should match!",
+ hbaseFileCksumBeforeTruncate, hbaseFileCksumS3);
+ }
+
private void restartNameNode() throws Exception {
cluster.triggerBlockReports();
NameNode nameNode = cluster.getNameNode();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[23/50] [abbrv] hadoop git commit: YARN-6852. Native code changes to
support isolate GPU devices by using CGroups. (wangda)
Posted by as...@apache.org.
YARN-6852. Native code changes to support isolate GPU devices by using CGroups. (wangda)
Change-Id: I4869cc4d8ad539539ccba4bea5a178cacdb741ab
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/436c2638
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/436c2638
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/436c2638
Branch: refs/heads/YARN-5972
Commit: 436c2638f9ca1fb8de6a630cb5e91d956ac75216
Parents: 8991f0b
Author: Wangda Tan <wa...@apache.org>
Authored: Fri Aug 18 18:26:36 2017 -0700
Committer: Wangda Tan <wa...@apache.org>
Committed: Fri Aug 18 18:26:36 2017 -0700
----------------------------------------------------------------------
.../src/CMakeLists.txt | 12 +-
.../impl/container-executor.c | 10 +-
.../impl/container-executor.h | 2 +
.../main/native/container-executor/impl/main.c | 13 +-
.../impl/modules/cgroups/cgroups-operations.c | 161 +++++++++++++
.../impl/modules/cgroups/cgroups-operations.h | 55 +++++
.../impl/modules/common/constants.h | 29 +++
.../impl/modules/common/module-configs.c | 41 ++++
.../impl/modules/common/module-configs.h | 33 +++
.../impl/modules/gpu/gpu-module.c | 229 +++++++++++++++++++
.../impl/modules/gpu/gpu-module.h | 45 ++++
.../container-executor/impl/utils/path-utils.c | 52 +++++
.../container-executor/impl/utils/path-utils.h | 35 +++
.../impl/utils/string-utils.c | 106 +++++++--
.../impl/utils/string-utils.h | 7 +-
.../test/modules/cgroups/test-cgroups-module.cc | 121 ++++++++++
.../test/modules/gpu/test-gpu-module.cc | 203 ++++++++++++++++
.../test/test-container-executor-common.h | 36 +++
.../test/test-container-executor.c | 23 +-
.../native/container-executor/test/test_main.cc | 11 +-
.../test/utils/test-path-utils.cc | 67 ++++++
.../test/utils/test-string-utils.cc | 93 ++++++++
22 files changed, 1338 insertions(+), 46 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/436c2638/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt
index 100d7ca..07c29bf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/CMakeLists.txt
@@ -101,6 +101,10 @@ add_library(container
main/native/container-executor/impl/container-executor.c
main/native/container-executor/impl/get_executable.c
main/native/container-executor/impl/utils/string-utils.c
+ main/native/container-executor/impl/utils/path-utils.c
+ main/native/container-executor/impl/modules/cgroups/cgroups-operations.c
+ main/native/container-executor/impl/modules/common/module-configs.c
+ main/native/container-executor/impl/modules/gpu/gpu-module.c
)
add_executable(container-executor
@@ -113,12 +117,14 @@ target_link_libraries(container-executor
output_directory(container-executor target/usr/local/bin)
+# Test cases
add_executable(test-container-executor
main/native/container-executor/test/test-container-executor.c
)
target_link_libraries(test-container-executor
container ${EXTRA_LIBS}
)
+
output_directory(test-container-executor target/usr/local/bin)
# unit tests for container executor
@@ -126,6 +132,10 @@ add_executable(cetest
main/native/container-executor/impl/util.c
main/native/container-executor/test/test_configuration.cc
main/native/container-executor/test/test_main.cc
+ main/native/container-executor/test/utils/test-string-utils.cc
+ main/native/container-executor/test/utils/test-path-utils.cc
+ main/native/container-executor/test/modules/cgroups/test-cgroups-module.cc
+ main/native/container-executor/test/modules/gpu/test-gpu-module.cc
main/native/container-executor/test/test_util.cc)
-target_link_libraries(cetest gtest)
+target_link_libraries(cetest gtest container)
output_directory(cetest test)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/436c2638/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
index 7361808..560ec18 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.c
@@ -1384,7 +1384,6 @@ char* sanitize_docker_command(const char *line) {
}
char* parse_docker_command_file(const char* command_file) {
-
size_t len = 0;
char *line = NULL;
ssize_t read;
@@ -2443,3 +2442,12 @@ int traffic_control_read_state(char *command_file) {
int traffic_control_read_stats(char *command_file) {
return run_traffic_control(TC_READ_STATS_OPTS, command_file);
}
+
+/**
+ * FIXME: (wangda) it's better to move executor_cfg out of container-executor.c
+ * Now initialize of executor_cfg and data structures are stored inside
+ * container-executor which is not a good design.
+ */
+struct configuration* get_cfg() {
+ return &CFG;
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/436c2638/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
index ea8b5e3..aa38abf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/container-executor.h
@@ -274,3 +274,5 @@ int execute_regex_match(const char *regex_str, const char *input);
* Return 0 on success.
*/
int validate_docker_image_name(const char *image_name);
+
+struct configuration* get_cfg();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/436c2638/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c
index b2187c9..a05dc78 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/main.c
@@ -20,6 +20,8 @@
#include "configuration.h"
#include "container-executor.h"
#include "util.h"
+#include "modules/gpu/gpu-module.h"
+#include "modules/cgroups/cgroups-operations.h"
#include <errno.h>
#include <grp.h>
@@ -253,6 +255,14 @@ static int validate_arguments(int argc, char **argv , int *operation) {
return INVALID_ARGUMENT_NUMBER;
}
+ /*
+ * Check if it is a known module, if yes, redirect to module
+ */
+ if (strcmp("--module-gpu", argv[1]) == 0) {
+ return handle_gpu_request(&update_cgroups_parameters, "gpu", argc - 1,
+ &argv[1]);
+ }
+
if (strcmp("--checksetup", argv[1]) == 0) {
*operation = CHECK_SETUP;
return 0;
@@ -332,6 +342,7 @@ static int validate_arguments(int argc, char **argv , int *operation) {
return FEATURE_DISABLED;
}
}
+
/* Now we have to validate 'run as user' operations that don't use
a 'long option' - we should fix this at some point. The validation/argument
parsing here is extensive enough that it done in a separate function */
@@ -522,7 +533,7 @@ int main(int argc, char **argv) {
open_log_files();
assert_valid_setup(argv[0]);
- int operation;
+ int operation = -1;
int ret = validate_arguments(argc, argv, &operation);
if (ret != 0) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/436c2638/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/cgroups/cgroups-operations.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/cgroups/cgroups-operations.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/cgroups/cgroups-operations.c
new file mode 100644
index 0000000..b234109
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/cgroups/cgroups-operations.c
@@ -0,0 +1,161 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "configuration.h"
+#include "container-executor.h"
+#include "utils/string-utils.h"
+#include "utils/path-utils.h"
+#include "modules/common/module-configs.h"
+#include "modules/common/constants.h"
+#include "modules/cgroups/cgroups-operations.h"
+#include "util.h"
+
+#include <string.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <sys/stat.h>
+
+#define MAX_PATH_LEN 4096
+
+static const struct section* cgroup_cfg_section = NULL;
+
+void reload_cgroups_configuration() {
+ cgroup_cfg_section = get_configuration_section(CGROUPS_SECTION_NAME, get_cfg());
+}
+
+char* get_cgroups_path_to_write(
+ const char* hierarchy_name,
+ const char* param_name,
+ const char* group_id) {
+ int failed = 0;
+ char* buffer = NULL;
+ const char* cgroups_root = get_section_value(CGROUPS_ROOT_KEY,
+ cgroup_cfg_section);
+ const char* yarn_hierarchy_name = get_section_value(
+ CGROUPS_YARN_HIERARCHY_KEY, cgroup_cfg_section);
+
+ // Make sure it is defined.
+ if (!cgroups_root || cgroups_root[0] == 0) {
+ fprintf(ERRORFILE, "%s is not defined in container-executor.cfg\n",
+ CGROUPS_ROOT_KEY);
+ failed = 1;
+ goto cleanup;
+ }
+
+ // Make sure it is defined.
+ if (!yarn_hierarchy_name || yarn_hierarchy_name[0] == 0) {
+ fprintf(ERRORFILE, "%s is not defined in container-executor.cfg\n",
+ CGROUPS_YARN_HIERARCHY_KEY);
+ failed = 1;
+ goto cleanup;
+ }
+
+ buffer = malloc(MAX_PATH_LEN + 1);
+ if (!buffer) {
+ fprintf(ERRORFILE, "Failed to allocate memory for output path.\n");
+ failed = 1;
+ goto cleanup;
+ }
+
+ // Make a path.
+ // CGroups path should not be too long.
+ if (snprintf(buffer, MAX_PATH_LEN, "%s/%s/%s/%s/%s.%s",
+ cgroups_root, hierarchy_name, yarn_hierarchy_name,
+ group_id, hierarchy_name, param_name) < 0) {
+ fprintf(ERRORFILE, "Failed to print output path.\n");
+ failed = 1;
+ goto cleanup;
+ }
+
+cleanup:
+ if (failed) {
+ if (buffer) {
+ free(buffer);
+ }
+ return NULL;
+ }
+ return buffer;
+}
+
+int update_cgroups_parameters(
+ const char* hierarchy_name,
+ const char* param_name,
+ const char* group_id,
+ const char* value) {
+#ifndef __linux
+ fprintf(ERRORFILE, "Failed to update cgroups parameters, not supported\n");
+ return -1;
+#endif
+ int failure = 0;
+
+ if (!cgroup_cfg_section) {
+ reload_cgroups_configuration();
+ }
+
+ char* full_path = get_cgroups_path_to_write(hierarchy_name, param_name,
+ group_id);
+
+ if (!full_path) {
+ fprintf(ERRORFILE,
+ "Failed to get cgroups path to write, it should be a configuration issue");
+ failure = 1;
+ goto cleanup;
+ }
+
+ if (!verify_path_safety(full_path)) {
+ failure = 1;
+ goto cleanup;
+ }
+
+ // Make sure file exists
+ struct stat sb;
+ if (stat(full_path, &sb) != 0) {
+ fprintf(ERRORFILE, "CGroups: Could not find file to write, %s", full_path);
+ failure = 1;
+ goto cleanup;
+ }
+
+ fprintf(ERRORFILE, "CGroups: Updating cgroups, path=%s, value=%s",
+ full_path, value);
+
+ // Write values to file
+ FILE *f;
+ f = fopen(full_path, "a");
+ if (!f) {
+ fprintf(ERRORFILE, "CGroups: Failed to open cgroups file, %s", full_path);
+ failure = 1;
+ goto cleanup;
+ }
+ if (fprintf(f, "%s", value) < 0) {
+ fprintf(ERRORFILE, "CGroups: Failed to write cgroups file, %s", full_path);
+ fclose(f);
+ failure = 1;
+ goto cleanup;
+ }
+ if (fclose(f) != 0) {
+ fprintf(ERRORFILE, "CGroups: Failed to close cgroups file, %s", full_path);
+ failure = 1;
+ goto cleanup;
+ }
+
+cleanup:
+ if (full_path) {
+ free(full_path);
+ }
+ return -failure;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/436c2638/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/cgroups/cgroups-operations.h
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/cgroups/cgroups-operations.h b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/cgroups/cgroups-operations.h
new file mode 100644
index 0000000..cf80bcf
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/cgroups/cgroups-operations.h
@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifndef _CGROUPS_OPERATIONS_H_
+#define _CGROUPS_OPERATIONS_H_
+
+#define CGROUPS_SECTION_NAME "cgroups"
+#define CGROUPS_ROOT_KEY "root"
+#define CGROUPS_YARN_HIERARCHY_KEY "yarn-hierarchy"
+
+/**
+ * Handle update CGroups parameter update requests:
+ * - hierarchy_name: e.g. devices / cpu,cpuacct
+ * - param_name: e.g. deny
+ * - group_id: e.g. container_x_y
+ * - value: e.g. "a *:* rwm"
+ *
+ * return 0 if succeeded
+ */
+int update_cgroups_parameters(
+ const char* hierarchy_name,
+ const char* param_name,
+ const char* group_id,
+ const char* value);
+
+ /**
+ * Get CGroups path to update. Visible for testing.
+ * Return 0 if succeeded
+ */
+ char* get_cgroups_path_to_write(
+ const char* hierarchy_name,
+ const char* param_name,
+ const char* group_id);
+
+ /**
+ * Reload config from filesystem, visible for testing.
+ */
+ void reload_cgroups_configuration();
+
+#endif
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/436c2638/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/common/constants.h
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/common/constants.h b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/common/constants.h
new file mode 100644
index 0000000..5c8c4e9
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/common/constants.h
@@ -0,0 +1,29 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* FreeBSD protects the getline() prototype. See getline(3) for more */
+#ifdef __FreeBSD__
+#define _WITH_GETLINE
+#endif
+
+#ifndef _MODULES_COMMON_CONSTANTS_H_
+#define _MODULES_COMMON_CONSTANTS_H_
+
+#define CONFIGS_MODULES_PREFIX "yarn.container-executor.modules."
+
+#endif
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/436c2638/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/common/module-configs.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/common/module-configs.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/common/module-configs.c
new file mode 100644
index 0000000..f0c6d16
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/common/module-configs.c
@@ -0,0 +1,41 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "util.h"
+#include "configuration.h"
+#include "container-executor.h"
+#include "modules/common/constants.h"
+
+#include <string.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+#define ENABLED_CONFIG_KEY "module.enabled"
+
+int module_enabled(const struct section* section_cfg, const char* module_name) {
+ char* enabled_str = get_section_value(ENABLED_CONFIG_KEY, section_cfg);
+ int enabled = 0;
+ if (enabled_str && 0 == strcmp(enabled_str, "true")) {
+ enabled = 1;
+ } else {
+ fprintf(LOGFILE, "Module %s is disabled\n", module_name);
+ }
+
+ free(enabled_str);
+ return enabled;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/436c2638/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/common/module-configs.h
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/common/module-configs.h b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/common/module-configs.h
new file mode 100644
index 0000000..d58c618
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/common/module-configs.h
@@ -0,0 +1,33 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifdef __FreeBSD__
+#define _WITH_GETLINE
+#endif
+
+#ifndef _MODULES_COMMON_MODULE_CONFIGS_H_
+#define _MODULES_COMMON_MODULE_CONFIGS_H_
+
+
+/**
+ * check if module enabled given name of module.
+ * return 0 if disabled
+ */
+int module_enabled(const struct section* section_cfg, const char* module_name);
+
+#endif
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/436c2638/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/gpu/gpu-module.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/gpu/gpu-module.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/gpu/gpu-module.c
new file mode 100644
index 0000000..f96645d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/gpu/gpu-module.c
@@ -0,0 +1,229 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "configuration.h"
+#include "container-executor.h"
+#include "utils/string-utils.h"
+#include "modules/gpu/gpu-module.h"
+#include "modules/cgroups/cgroups-operations.h"
+#include "modules/common/module-configs.h"
+#include "modules/common/constants.h"
+#include "util.h"
+
+#include <stdio.h>
+#include <string.h>
+#include <stdlib.h>
+#include <getopt.h>
+#include <unistd.h>
+
+#define EXCLUDED_GPUS_OPTION "excluded_gpus"
+#define CONTAINER_ID_OPTION "container_id"
+#define DEFAULT_NVIDIA_MAJOR_NUMBER 195
+#define MAX_CONTAINER_ID_LEN 128
+
+static const struct section* cfg_section;
+
+static int internal_handle_gpu_request(
+ update_cgroups_parameters_func update_cgroups_parameters_func_p,
+ size_t n_minor_devices_to_block, int minor_devices[],
+ const char* container_id) {
+ char* allowed_minor_numbers_str = NULL;
+ int* allowed_minor_numbers = NULL;
+ size_t n_allowed_minor_numbers = 0;
+ int return_code = 0;
+
+ if (n_minor_devices_to_block == 0) {
+ // no device to block, just return;
+ return 0;
+ }
+
+ // Get major device number from cfg, if not set, major number of (Nvidia)
+ // will be the default value.
+ int major_device_number;
+ char* major_number_str = get_section_value(GPU_MAJOR_NUMBER_CONFIG_KEY,
+ cfg_section);
+ if (!major_number_str || 0 == major_number_str[0]) {
+ // Default major number of Nvidia devices
+ major_device_number = DEFAULT_NVIDIA_MAJOR_NUMBER;
+ } else {
+ major_device_number = strtol(major_number_str, NULL, 0);
+ }
+
+ // Get allowed minor device numbers from cfg, if not set, means all minor
+ // devices can be used by YARN
+ allowed_minor_numbers_str = get_section_value(
+ GPU_ALLOWED_DEVICES_MINOR_NUMBERS,
+ cfg_section);
+ if (!allowed_minor_numbers_str || 0 == allowed_minor_numbers_str[0]) {
+ allowed_minor_numbers = NULL;
+ } else {
+ int rc = get_numbers_split_by_comma(allowed_minor_numbers_str,
+ &allowed_minor_numbers,
+ &n_allowed_minor_numbers);
+ if (0 != rc) {
+ fprintf(ERRORFILE,
+ "Failed to get allowed minor device numbers from cfg, value=%s\n",
+ allowed_minor_numbers_str);
+ return_code = -1;
+ goto cleanup;
+ }
+
+ // Make sure we're trying to black devices allowed in config
+ for (int i = 0; i < n_minor_devices_to_block; i++) {
+ int found = 0;
+ for (int j = 0; j < n_allowed_minor_numbers; j++) {
+ if (minor_devices[i] == allowed_minor_numbers[j]) {
+ found = 1;
+ break;
+ }
+ }
+
+ if (!found) {
+ fprintf(ERRORFILE,
+ "Trying to blacklist device with minor-number=%d which is not on allowed list\n",
+ minor_devices[i]);
+ return_code = -1;
+ goto cleanup;
+ }
+ }
+ }
+
+ // Use cgroup helpers to blacklist devices
+ for (int i = 0; i < n_minor_devices_to_block; i++) {
+ char param_value[128];
+ memset(param_value, 0, sizeof(param_value));
+ snprintf(param_value, sizeof(param_value), "c %d:%d rwm",
+ major_device_number, i);
+
+ int rc = update_cgroups_parameters_func_p("devices", "deny",
+ container_id, param_value);
+
+ if (0 != rc) {
+ fprintf(ERRORFILE, "CGroups: Failed to update cgroups\n");
+ return_code = -1;
+ goto cleanup;
+ }
+ }
+
+cleanup:
+ if (major_number_str) {
+ free(major_number_str);
+ }
+ if (allowed_minor_numbers) {
+ free(allowed_minor_numbers);
+ }
+ if (allowed_minor_numbers_str) {
+ free(allowed_minor_numbers_str);
+ }
+
+ return return_code;
+}
+
+void reload_gpu_configuration() {
+ cfg_section = get_configuration_section(GPU_MODULE_SECTION_NAME, get_cfg());
+}
+
+/*
+ * Format of GPU request commandline:
+ *
+ * c-e gpu --excluded_gpus 0,1,3 --container_id container_x_y
+ */
+int handle_gpu_request(update_cgroups_parameters_func func,
+ const char* module_name, int module_argc, char** module_argv) {
+ if (!cfg_section) {
+ reload_gpu_configuration();
+ }
+
+ if (!module_enabled(cfg_section, GPU_MODULE_SECTION_NAME)) {
+ fprintf(ERRORFILE,
+ "Please make sure gpu module is enabled before using it.\n");
+ return -1;
+ }
+
+ static struct option long_options[] = {
+ {EXCLUDED_GPUS_OPTION, required_argument, 0, 'e' },
+ {CONTAINER_ID_OPTION, required_argument, 0, 'c' },
+ {0, 0, 0, 0}
+ };
+
+ int rc = 0;
+ int c = 0;
+ int option_index = 0;
+
+ int* minor_devices = NULL;
+ char container_id[MAX_CONTAINER_ID_LEN];
+ memset(container_id, 0, sizeof(container_id));
+ size_t n_minor_devices_to_block = 0;
+ int failed = 0;
+
+ optind = 1;
+ while((c = getopt_long(module_argc, module_argv, "e:c:",
+ long_options, &option_index)) != -1) {
+ switch(c) {
+ case 'e':
+ rc = get_numbers_split_by_comma(optarg, &minor_devices,
+ &n_minor_devices_to_block);
+ if (0 != rc) {
+ fprintf(ERRORFILE,
+ "Failed to get minor devices number from command line, value=%s\n",
+ optarg);
+ failed = 1;
+ goto cleanup;
+ }
+ break;
+ case 'c':
+ if (!validate_container_id(optarg)) {
+ fprintf(ERRORFILE,
+ "Specified container_id=%s is invalid\n", optarg);
+ failed = 1;
+ goto cleanup;
+ }
+ strncpy(container_id, optarg, MAX_CONTAINER_ID_LEN);
+ break;
+ default:
+ fprintf(ERRORFILE,
+ "Unknown option in gpu command character %d %c, optionindex = %d\n",
+ c, c, optind);
+ failed = 1;
+ goto cleanup;
+ }
+ }
+
+ if (0 == container_id[0]) {
+ fprintf(ERRORFILE,
+ "[%s] --container_id must be specified.\n", __func__);
+ failed = 1;
+ goto cleanup;
+ }
+
+ if (!minor_devices) {
+ // Minor devices is null, skip following call.
+ fprintf(ERRORFILE, "is not specified, skip cgroups call.\n");
+ goto cleanup;
+ }
+
+ failed = internal_handle_gpu_request(func, n_minor_devices_to_block,
+ minor_devices,
+ container_id);
+
+cleanup:
+ if (minor_devices) {
+ free(minor_devices);
+ }
+ return failed;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/436c2638/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/gpu/gpu-module.h
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/gpu/gpu-module.h b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/gpu/gpu-module.h
new file mode 100644
index 0000000..59d4c7e
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/modules/gpu/gpu-module.h
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifdef __FreeBSD__
+#define _WITH_GETLINE
+#endif
+
+#ifndef _MODULES_GPU_GPU_MUDULE_H_
+#define _MODULES_GPU_GPU_MUDULE_H_
+
+#define GPU_MAJOR_NUMBER_CONFIG_KEY "gpu.major-device-number"
+#define GPU_ALLOWED_DEVICES_MINOR_NUMBERS "gpu.allowed-device-minor-numbers"
+#define GPU_MODULE_SECTION_NAME "gpu"
+
+// For unit test stubbing
+typedef int (*update_cgroups_parameters_func)(const char*, const char*,
+ const char*, const char*);
+
+/**
+ * Handle gpu requests
+ */
+int handle_gpu_request(update_cgroups_parameters_func func,
+ const char* module_name, int module_argc, char** module_argv);
+
+/**
+ * Reload config from filesystem, visible for testing.
+ */
+void reload_gpu_configuration();
+
+#endif
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/436c2638/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/path-utils.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/path-utils.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/path-utils.c
new file mode 100644
index 0000000..dea656b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/path-utils.c
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include "util.h"
+
+#include <strings.h>
+#include <string.h>
+#include <stdio.h>
+#include <stdlib.h>
+
+int verify_path_safety(const char* path) {
+ if (!path || path[0] == 0) {
+ return 1;
+ }
+
+ char* dup = strdup(path);
+ if (!dup) {
+ fprintf(ERRORFILE, "%s: Failed to allocate memory for path.\n", __func__);
+ return 0;
+ }
+
+ char* p = strtok(dup, "/");
+ int succeeded = 1;
+
+ while (p != NULL) {
+ if (0 == strcmp(p, "..")) {
+ fprintf(ERRORFILE, "%s: Path included \"..\", path=%s.\n", __func__, path);
+ succeeded = 0;
+ break;
+ }
+
+ p = strtok(NULL, "/");
+ }
+ free(dup);
+
+ return succeeded;
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/436c2638/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/path-utils.h
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/path-utils.h b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/path-utils.h
new file mode 100644
index 0000000..a42f936
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/path-utils.h
@@ -0,0 +1,35 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#ifdef __FreeBSD__
+#define _WITH_GETLINE
+#endif
+
+#ifndef _UTILS_PATH_UTILS_H_
+#define _UTILS_PATH_UTILS_H_
+
+/*
+ * Verify if a given path is safe or not. For example, we don't want a path
+ * include ".." which can do things like:
+ * - "/cgroups/cpu,cpuacct/container/../../../etc/passwd"
+ *
+ * return false/true
+ */
+int verify_path_safety(const char* path);
+
+#endif
http://git-wip-us.apache.org/repos/asf/hadoop/blob/436c2638/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/string-utils.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/string-utils.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/string-utils.c
index 063df7e..d19c084 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/string-utils.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/string-utils.c
@@ -15,7 +15,10 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+#include "util.h"
+#include <limits.h>
+#include <errno.h>
#include <strings.h>
#include <string.h>
#include <stdio.h>
@@ -26,60 +29,131 @@
* return true/false
*/
static int all_numbers(char* input) {
- if (0 == strlen(input)) {
- return 0;
- }
-
- for (int i = 0; i < strlen(input); i++) {
- if (input[i] < '0' || input[i] > '9') {
+ for (; input[0] != 0; input++) {
+ if (input[0] < '0' || input[0] > '9') {
return 0;
}
}
return 1;
}
+int get_numbers_split_by_comma(const char* input, int** numbers,
+ size_t* ret_n_numbers) {
+ size_t allocation_size = 1;
+ int i = 0;
+ while (input[i] != 0) {
+ if (input[i] == ',') {
+ allocation_size++;
+ }
+ i++;
+ }
+
+ (*numbers) = malloc(sizeof(int) * allocation_size);
+ if (!(*numbers)) {
+ fprintf(ERRORFILE, "Failed to allocating memory for *numbers: %s\n",
+ __func__);
+ exit(OUT_OF_MEMORY);
+ }
+ memset(*numbers, 0, sizeof(int) * allocation_size);
+
+ char* input_cpy = strdup(input);
+ if (!input_cpy) {
+ fprintf(ERRORFILE, "Failed to allocating memory for input_cpy: %s\n",
+ __func__);
+ exit(OUT_OF_MEMORY);
+ }
+
+ char* p = strtok(input_cpy, ",");
+ int idx = 0;
+ size_t n_numbers = 0;
+ while (p != NULL) {
+ char *temp;
+ long n = strtol(p, &temp, 0);
+ // According to answer:
+ // https://stackoverflow.com/questions/14176123/correct-usage-of-strtol
+ // We need to properly check errno and overflows
+ if (temp == p || *temp != '\0' ||
+ ((n == LONG_MIN || n == LONG_MAX) && errno == ERANGE)) {
+ fprintf(stderr,
+ "Could not convert '%s' to long and leftover string is: '%s'\n",
+ p, temp);
+ free(input_cpy);
+ return -1;
+ }
+
+ n_numbers++;
+ (*numbers)[idx] = n;
+ p = strtok(NULL, ",");
+ idx++;
+ }
+
+ free(input_cpy);
+ *ret_n_numbers = n_numbers;
+
+ return 0;
+}
+
int validate_container_id(const char* input) {
+ int is_container_id = 1;
+
/*
* Two different forms of container_id
* container_e17_1410901177871_0001_01_000005
* container_1410901177871_0001_01_000005
*/
+ if (!input) {
+ return 0;
+ }
+
char* input_cpy = strdup(input);
+ if (!input_cpy) {
+ return 0;
+ }
+
char* p = strtok(input_cpy, "_");
int idx = 0;
while (p != NULL) {
if (0 == idx) {
if (0 != strcmp("container", p)) {
- return 0;
+ is_container_id = 0;
+ goto cleanup;
}
} else if (1 == idx) {
// this could be e[n][n], or [n][n]...
if (!all_numbers(p)) {
- if (strlen(p) == 0) {
- return 0;
+ if (p[0] == 0) {
+ is_container_id = 0;
+ goto cleanup;
}
if (p[0] != 'e') {
- return 0;
+ is_container_id = 0;
+ goto cleanup;
}
if (!all_numbers(p + 1)) {
- return 0;
+ is_container_id = 0;
+ goto cleanup;
}
}
} else {
// otherwise, should be all numbers
if (!all_numbers(p)) {
- return 0;
+ is_container_id = 0;
+ goto cleanup;
}
}
p = strtok(NULL, "_");
idx++;
}
- free(input_cpy);
+
+cleanup:
+ if (input_cpy) {
+ free(input_cpy);
+ }
// We should have [5,6] elements split by '_'
if (idx > 6 || idx < 5) {
- return 0;
+ is_container_id = 0;
}
- return 1;
-}
\ No newline at end of file
+ return is_container_id;
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/436c2638/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/string-utils.h
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/string-utils.h b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/string-utils.h
index 0a41ad1..c095eb6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/string-utils.h
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/string-utils.h
@@ -29,4 +29,9 @@
*/
int validate_container_id(const char* input);
-#endif
\ No newline at end of file
+/*
+ * return 0 if succeeded
+ */
+int get_numbers_split_by_comma(const char* input, int** numbers, size_t* n_numbers);
+
+#endif
http://git-wip-us.apache.org/repos/asf/hadoop/blob/436c2638/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/modules/cgroups/test-cgroups-module.cc
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/modules/cgroups/test-cgroups-module.cc b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/modules/cgroups/test-cgroups-module.cc
new file mode 100644
index 0000000..8ffbe88
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/modules/cgroups/test-cgroups-module.cc
@@ -0,0 +1,121 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <errno.h>
+#include <fcntl.h>
+#include <inttypes.h>
+#include <signal.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/stat.h>
+#include <sys/wait.h>
+#include <unistd.h>
+
+#include <gtest/gtest.h>
+#include <sstream>
+
+extern "C" {
+#include "configuration.h"
+#include "container-executor.h"
+#include "modules/cgroups/cgroups-operations.h"
+#include "test/test-container-executor-common.h"
+#include "util.h"
+}
+
+namespace ContainerExecutor {
+
+class TestCGroupsModule : public ::testing::Test {
+protected:
+ virtual void SetUp() {
+ if (mkdirs(TEST_ROOT, 0755) != 0) {
+ fprintf(ERRORFILE, "Failed to mkdir TEST_ROOT: %s\n", TEST_ROOT);
+ exit(1);
+ }
+ LOGFILE = stdout;
+ ERRORFILE = stderr;
+ }
+
+ virtual void TearDown() {}
+};
+
+TEST_F(TestCGroupsModule, test_cgroups_get_path_without_define_root) {
+ // Write config file.
+ const char *filename = TEST_ROOT "/test_cgroups_get_path_without_root.cfg";
+ FILE *file = fopen(filename, "w");
+ if (file == NULL) {
+ printf("FAIL: Could not open configuration file: %s\n", filename);
+ exit(1);
+ }
+ fprintf(file, "[cgroups]\n");
+ fprintf(file, "yarn-hierarchy=yarn\n");
+ fclose(file);
+
+ // Read config file
+ read_executor_config(filename);
+ reload_cgroups_configuration();
+
+ char* path = get_cgroups_path_to_write("devices", "deny", "container_1");
+
+ ASSERT_TRUE(NULL == path) << "Should fail.\n";
+}
+
+TEST_F(TestCGroupsModule, test_cgroups_get_path_without_define_yarn_hierarchy) {
+ // Write config file.
+ const char *filename = TEST_ROOT "/test_cgroups_get_path_without_root.cfg";
+ FILE *file = fopen(filename, "w");
+
+ ASSERT_TRUE(file) << "FAIL: Could not open configuration file: " << filename
+ << "\n";
+ fprintf(file, "[cgroups]\n");
+ fprintf(file, "root=/sys/fs/cgroups\n");
+ fclose(file);
+
+ // Read config file
+ read_executor_config(filename);
+ reload_cgroups_configuration();
+ char* path = get_cgroups_path_to_write("devices", "deny", "container_1");
+
+ ASSERT_TRUE(NULL == path) << "Should fail.\n";
+}
+
+TEST_F(TestCGroupsModule, test_cgroups_get_path_succeeded) {
+ // Write config file.
+ const char *filename = TEST_ROOT "/test_cgroups_get_path.cfg";
+ FILE *file = fopen(filename, "w");
+
+ ASSERT_TRUE(file) << "FAIL: Could not open configuration file\n";
+ fprintf(file, "[cgroups]\n");
+ fprintf(file, "root=/sys/fs/cgroups \n");
+ fprintf(file, "yarn-hierarchy=yarn \n");
+ fclose(file);
+
+ // Read config file
+ read_executor_config(filename);
+ reload_cgroups_configuration();
+
+ char* path = get_cgroups_path_to_write("devices", "deny", "container_1");
+ ASSERT_TRUE(NULL != path) << "Should success.\n";
+
+ const char *EXPECTED =
+ "/sys/fs/cgroups/devices/yarn/container_1/devices.deny";
+
+ ASSERT_STREQ(EXPECTED, path)
+ << "Return cgroup-path-to-write is not expected\n";
+}
+} // namespace ContainerExecutor
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/436c2638/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/modules/gpu/test-gpu-module.cc
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/modules/gpu/test-gpu-module.cc b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/modules/gpu/test-gpu-module.cc
new file mode 100644
index 0000000..7e41fb4
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/modules/gpu/test-gpu-module.cc
@@ -0,0 +1,203 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+#include <vector>
+
+#include <errno.h>
+#include <fcntl.h>
+#include <inttypes.h>
+#include <signal.h>
+#include <stdio.h>
+#include <stdlib.h>
+#include <string.h>
+#include <sys/stat.h>
+#include <sys/wait.h>
+#include <unistd.h>
+
+#include <gtest/gtest.h>
+#include <sstream>
+
+extern "C" {
+#include "configuration.h"
+#include "container-executor.h"
+#include "modules/cgroups/cgroups-operations.h"
+#include "modules/gpu/gpu-module.h"
+#include "test/test-container-executor-common.h"
+#include "util.h"
+}
+
+namespace ContainerExecutor {
+
+class TestGpuModule : public ::testing::Test {
+protected:
+ virtual void SetUp() {
+ if (mkdirs(TEST_ROOT, 0755) != 0) {
+ fprintf(ERRORFILE, "Failed to mkdir TEST_ROOT: %s\n", TEST_ROOT);
+ exit(1);
+ }
+ LOGFILE = stdout;
+ ERRORFILE = stderr;
+ }
+
+ virtual void TearDown() {
+
+ }
+};
+
+static std::vector<const char*> cgroups_parameters_invoked;
+
+static int mock_update_cgroups_parameters(
+ const char* controller_name,
+ const char* param_name,
+ const char* group_id,
+ const char* value) {
+ char* buf = (char*) malloc(128);
+ strcpy(buf, controller_name);
+ cgroups_parameters_invoked.push_back(buf);
+
+ buf = (char*) malloc(128);
+ strcpy(buf, param_name);
+ cgroups_parameters_invoked.push_back(buf);
+
+ buf = (char*) malloc(128);
+ strcpy(buf, group_id);
+ cgroups_parameters_invoked.push_back(buf);
+
+ buf = (char*) malloc(128);
+ strcpy(buf, value);
+ cgroups_parameters_invoked.push_back(buf);
+ return 0;
+}
+
+static void verify_param_updated_to_cgroups(
+ int argc, const char** argv) {
+ ASSERT_EQ(argc, cgroups_parameters_invoked.size());
+
+ int offset = 0;
+ while (offset < argc) {
+ ASSERT_STREQ(argv[offset], cgroups_parameters_invoked[offset]);
+ offset++;
+ }
+}
+
+static void write_and_load_gpu_module_to_cfg(const char* cfg_filepath, int enabled) {
+ FILE *file = fopen(cfg_filepath, "w");
+ if (file == NULL) {
+ printf("FAIL: Could not open configuration file: %s\n", cfg_filepath);
+ exit(1);
+ }
+ fprintf(file, "[gpu]\n");
+ if (enabled) {
+ fprintf(file, "module.enabled=true\n");
+ } else {
+ fprintf(file, "module.enabled=false\n");
+ }
+ fclose(file);
+
+ // Read config file
+ read_executor_config(cfg_filepath);
+ reload_gpu_configuration();
+}
+
+static void test_gpu_module_enabled_disabled(int enabled) {
+ // Write config file.
+ const char *filename = TEST_ROOT "/test_cgroups_module_enabled_disabled.cfg";
+ write_and_load_gpu_module_to_cfg(filename, enabled);
+
+ char* argv[] = { (char*) "--module-gpu", (char*) "--excluded_gpus", (char*) "0,1",
+ (char*) "--container_id",
+ (char*) "container_1498064906505_0001_01_000001" };
+
+ int rc = handle_gpu_request(&mock_update_cgroups_parameters,
+ "gpu", 5, argv);
+
+ int EXPECTED_RC;
+ if (enabled) {
+ EXPECTED_RC = 0;
+ } else {
+ EXPECTED_RC = -1;
+ }
+ ASSERT_EQ(EXPECTED_RC, rc);
+}
+
+TEST_F(TestGpuModule, test_verify_gpu_module_calls_cgroup_parameter) {
+ // Write config file.
+ const char *filename = TEST_ROOT "/test_verify_gpu_module_calls_cgroup_parameter.cfg";
+ write_and_load_gpu_module_to_cfg(filename, 1);
+
+ char* container_id = (char*) "container_1498064906505_0001_01_000001";
+ char* argv[] = { (char*) "--module-gpu", (char*) "--excluded_gpus", (char*) "0,1",
+ (char*) "--container_id",
+ container_id };
+
+ /* Test case 1: block 2 devices */
+ cgroups_parameters_invoked.clear();
+ int rc = handle_gpu_request(&mock_update_cgroups_parameters,
+ "gpu", 5, argv);
+ ASSERT_EQ(0, rc) << "Should success.\n";
+
+ // Verify cgroups parameters
+ const char* expected_cgroups_argv[] = { "devices", "deny", container_id, "c 195:0 rwm",
+ "devices", "deny", container_id, "c 195:1 rwm"};
+ verify_param_updated_to_cgroups(8, expected_cgroups_argv);
+
+ /* Test case 2: block 0 devices */
+ cgroups_parameters_invoked.clear();
+ char* argv_1[] = { (char*) "--module-gpu", (char*) "--container_id", container_id };
+ rc = handle_gpu_request(&mock_update_cgroups_parameters,
+ "gpu", 3, argv_1);
+ ASSERT_EQ(0, rc) << "Should success.\n";
+
+ // Verify cgroups parameters
+ verify_param_updated_to_cgroups(0, NULL);
+}
+
+TEST_F(TestGpuModule, test_illegal_cli_parameters) {
+ // Write config file.
+ const char *filename = TEST_ROOT "/test_illegal_cli_parameters.cfg";
+ write_and_load_gpu_module_to_cfg(filename, 1);
+
+ // Illegal container id - 1
+ char* argv[] = { (char*) "--module-gpu", (char*) "--excluded_gpus", (char*) "0,1",
+ (char*) "--container_id", (char*) "xxxx" };
+ int rc = handle_gpu_request(&mock_update_cgroups_parameters,
+ "gpu", 5, argv);
+ ASSERT_NE(0, rc) << "Should fail.\n";
+
+ // Illegal container id - 2
+ char* argv_1[] = { (char*) "--module-gpu", (char*) "--excluded_gpus", (char*) "0,1",
+ (char*) "--container_id", (char*) "container_1" };
+ rc = handle_gpu_request(&mock_update_cgroups_parameters,
+ "gpu", 5, argv_1);
+ ASSERT_NE(0, rc) << "Should fail.\n";
+
+ // Illegal container id - 3
+ char* argv_2[] = { (char*) "--module-gpu", (char*) "--excluded_gpus", (char*) "0,1" };
+ rc = handle_gpu_request(&mock_update_cgroups_parameters,
+ "gpu", 3, argv_2);
+ ASSERT_NE(0, rc) << "Should fail.\n";
+}
+
+TEST_F(TestGpuModule, test_gpu_module_disabled) {
+ test_gpu_module_enabled_disabled(0);
+}
+
+TEST_F(TestGpuModule, test_gpu_module_enabled) {
+ test_gpu_module_enabled_disabled(1);
+}
+} // namespace ContainerExecutor
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/436c2638/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor-common.h
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor-common.h b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor-common.h
new file mode 100644
index 0000000..d353625
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor-common.h
@@ -0,0 +1,36 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ #ifdef __APPLE__
+ #include <CoreFoundation/CFString.h>
+ #include <CoreFoundation/CFPreferences.h>
+
+ #define TMPDIR "/private/tmp"
+ #define RELTMPDIR "../.."
+ #else
+ #define RELTMPDIR ".."
+ #define TMPDIR "/tmp"
+ #endif
+
+ #define TEST_ROOT TMPDIR "/test-container-executor"
+
+ #define DONT_TOUCH_FILE "dont-touch-me"
+ #define NM_LOCAL_DIRS TEST_ROOT "/local-1%" TEST_ROOT "/local-2%" \
+ TEST_ROOT "/local-3%" TEST_ROOT "/local-4%" TEST_ROOT "/local-5"
+ #define NM_LOG_DIRS TEST_ROOT "/logs/userlogs"
+ #define ARRAY_SIZE 1000
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/436c2638/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
index 3cfefa0..64ee717 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test-container-executor.c
@@ -19,6 +19,7 @@
#include "container-executor.h"
#include "utils/string-utils.h"
#include "util.h"
+#include "test/test-container-executor-common.h"
#include <inttypes.h>
#include <errno.h>
@@ -31,25 +32,6 @@
#include <sys/stat.h>
#include <sys/wait.h>
-#ifdef __APPLE__
-#include <CoreFoundation/CFString.h>
-#include <CoreFoundation/CFPreferences.h>
-
-#define TMPDIR "/private/tmp"
-#define RELTMPDIR "../.."
-#else
-#define RELTMPDIR ".."
-#define TMPDIR "/tmp"
-#endif
-
-#define TEST_ROOT TMPDIR "/test-container-executor"
-
-#define DONT_TOUCH_FILE "dont-touch-me"
-#define NM_LOCAL_DIRS TEST_ROOT "/local-1%" TEST_ROOT "/local-2%" \
- TEST_ROOT "/local-3%" TEST_ROOT "/local-4%" TEST_ROOT "/local-5"
-#define NM_LOG_DIRS TEST_ROOT "/logs/userlogs"
-#define ARRAY_SIZE 1000
-
static char* username = NULL;
static char* yarn_username = NULL;
static char** local_dirs = NULL;
@@ -1486,10 +1468,7 @@ int main(int argc, char **argv) {
test_check_user(1);
#endif
- run("rm -fr " TEST_ROOT);
-
test_trim_function();
-
printf("\nFinished tests\n");
free(current_username);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/436c2638/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test_main.cc
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test_main.cc b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test_main.cc
index d59a3f2..44c9b1b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test_main.cc
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/test_main.cc
@@ -20,10 +20,13 @@
#include <main/native/container-executor/impl/util.h>
#include <cstdio>
-FILE* ERRORFILE = stderr;
-FILE* LOGFILE = stdout;
+extern "C" {
+#include "util.h"
+}
int main(int argc, char **argv) {
- testing::InitGoogleTest(&argc, argv);
- return RUN_ALL_TESTS();
+ ERRORFILE = stderr;
+ LOGFILE = stdout;
+ testing::InitGoogleTest(&argc, argv);
+ return RUN_ALL_TESTS();
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/436c2638/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test-path-utils.cc
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test-path-utils.cc b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test-path-utils.cc
new file mode 100644
index 0000000..a24c0c7
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test-path-utils.cc
@@ -0,0 +1,67 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ #include <errno.h>
+ #include <fcntl.h>
+ #include <inttypes.h>
+ #include <signal.h>
+ #include <stdio.h>
+ #include <stdlib.h>
+ #include <string.h>
+ #include <sys/stat.h>
+ #include <sys/wait.h>
+ #include <unistd.h>
+
+ #include <gtest/gtest.h>
+ #include <sstream>
+
+ extern "C" {
+ #include "utils/path-utils.h"
+ }
+
+ namespace ContainerExecutor {
+
+ class TestPathUtils : public ::testing::Test {
+ protected:
+ virtual void SetUp() {
+
+ }
+
+ virtual void TearDown() {
+
+ }
+ };
+
+ TEST_F(TestPathUtils, test_path_safety) {
+ const char* input = "./../abc/";
+ int flag = verify_path_safety(input);
+ std::cout << "Testing input=" << input << "\n";
+ ASSERT_FALSE(flag) << "Should failed\n";
+
+ input = "abc/./cde";
+ flag = verify_path_safety(input);
+ std::cout << "Testing input=" << input << "\n";
+ ASSERT_TRUE(flag) << "Should succeeded\n";
+
+ input = "/etc/abc/cde/./x/./y";
+ flag = verify_path_safety(input);
+ std::cout << "Testing input=" << input << "\n";
+ ASSERT_TRUE(flag) << "Should succeeded\n";
+}
+
+} // namespace ContainerExecutor
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/436c2638/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test-string-utils.cc
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test-string-utils.cc b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test-string-utils.cc
new file mode 100644
index 0000000..037816a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/test/utils/test-string-utils.cc
@@ -0,0 +1,93 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+ #include <errno.h>
+ #include <fcntl.h>
+ #include <inttypes.h>
+ #include <signal.h>
+ #include <stdio.h>
+ #include <stdlib.h>
+ #include <string.h>
+ #include <sys/stat.h>
+ #include <sys/wait.h>
+ #include <unistd.h>
+
+ #include <gtest/gtest.h>
+ #include <sstream>
+
+ extern "C" {
+ #include "utils/string-utils.h"
+ }
+
+ namespace ContainerExecutor {
+
+ class TestStringUtils : public ::testing::Test {
+ protected:
+ virtual void SetUp() {
+
+ }
+
+ virtual void TearDown() {
+
+ }
+ };
+
+ TEST_F(TestStringUtils, test_get_numbers_split_by_comma) {
+ const char* input = ",1,2,3,-1,,1,,0,";
+ int* numbers;
+ size_t n_numbers;
+ int rc = get_numbers_split_by_comma(input, &numbers, &n_numbers);
+
+ std::cout << "Testing input=" << input << "\n";
+ ASSERT_EQ(0, rc) << "Should succeeded\n";
+ ASSERT_EQ(6, n_numbers);
+ ASSERT_EQ(1, numbers[0]);
+ ASSERT_EQ(-1, numbers[3]);
+ ASSERT_EQ(0, numbers[5]);
+
+ input = "3";
+ rc = get_numbers_split_by_comma(input, &numbers, &n_numbers);
+ std::cout << "Testing input=" << input << "\n";
+ ASSERT_EQ(0, rc) << "Should succeeded\n";
+ ASSERT_EQ(1, n_numbers);
+ ASSERT_EQ(3, numbers[0]);
+
+ input = "";
+ rc = get_numbers_split_by_comma(input, &numbers, &n_numbers);
+ std::cout << "Testing input=" << input << "\n";
+ ASSERT_EQ(0, rc) << "Should succeeded\n";
+ ASSERT_EQ(0, n_numbers);
+
+ input = ",,";
+ rc = get_numbers_split_by_comma(input, &numbers, &n_numbers);
+ std::cout << "Testing input=" << input << "\n";
+ ASSERT_EQ(0, rc) << "Should succeeded\n";
+ ASSERT_EQ(0, n_numbers);
+
+ input = "1,2,aa,bb";
+ rc = get_numbers_split_by_comma(input, &numbers, &n_numbers);
+ std::cout << "Testing input=" << input << "\n";
+ ASSERT_TRUE(0 != rc) << "Should failed\n";
+
+ input = "1,2,3,-12312312312312312312321311231231231";
+ rc = get_numbers_split_by_comma(input, &numbers, &n_numbers);
+ std::cout << "Testing input=" << input << "\n";
+ ASSERT_TRUE(0 != rc) << "Should failed\n";
+}
+
+} // namespace ContainerExecutor
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[17/50] [abbrv] hadoop git commit: HDFS-12316. Verify HDFS snapshot
deletion doesn't crash the ongoing file writes.
Posted by as...@apache.org.
HDFS-12316. Verify HDFS snapshot deletion doesn't crash the ongoing file writes.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/4230872d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/4230872d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/4230872d
Branch: refs/heads/YARN-5972
Commit: 4230872dd66d748172903b1522885b03f34bbf9b
Parents: b298948
Author: Manoj Govindassamy <ma...@apache.org>
Authored: Thu Aug 17 16:23:48 2017 -0700
Committer: Manoj Govindassamy <ma...@apache.org>
Committed: Thu Aug 17 16:23:48 2017 -0700
----------------------------------------------------------------------
.../snapshot/TestOpenFilesWithSnapshot.java | 109 +++++++++++++++++++
1 file changed, 109 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/4230872d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestOpenFilesWithSnapshot.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestOpenFilesWithSnapshot.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestOpenFilesWithSnapshot.java
index fb83a3e..bf27f2c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestOpenFilesWithSnapshot.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/snapshot/TestOpenFilesWithSnapshot.java
@@ -23,7 +23,11 @@ import java.util.EnumSet;
import java.util.HashSet;
import java.util.Random;
import java.util.Set;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.atomic.AtomicBoolean;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.Path;
@@ -38,12 +42,15 @@ import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
import org.apache.hadoop.hdfs.server.protocol.NamenodeProtocols;
+import org.apache.hadoop.util.Time;
import org.junit.After;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
public class TestOpenFilesWithSnapshot {
+ private static final Log LOG =
+ LogFactory.getLog(TestOpenFilesWithSnapshot.class.getName());
private final Configuration conf = new Configuration();
MiniDFSCluster cluster = null;
DistributedFileSystem fs = null;
@@ -622,6 +629,108 @@ public class TestOpenFilesWithSnapshot {
hbaseOutputStream.close();
}
+ /**
+ * Test client writing to open files are not interrupted when snapshots
+ * that captured open files get deleted.
+ */
+ @Test (timeout = 240000)
+ public void testOpenFileWritingAcrossSnapDeletion() throws Exception {
+ final Path snapRootDir = new Path("/level_0_A");
+ final String flumeFileName = "flume.log";
+ final String hbaseFileName = "hbase.log";
+ final String snap1Name = "snap_1";
+ final String snap2Name = "snap_2";
+ final String snap3Name = "snap_3";
+
+ // Create files and open streams
+ final Path flumeFile = new Path(snapRootDir, flumeFileName);
+ FSDataOutputStream flumeOut = fs.create(flumeFile, false,
+ 8000, (short)3, 1048576);
+ flumeOut.close();
+ final Path hbaseFile = new Path(snapRootDir, hbaseFileName);
+ FSDataOutputStream hbaseOut = fs.create(hbaseFile, false,
+ 8000, (short)3, 1048576);
+ hbaseOut.close();
+
+ final AtomicBoolean writerError = new AtomicBoolean(false);
+ final CountDownLatch startLatch = new CountDownLatch(1);
+ final CountDownLatch deleteLatch = new CountDownLatch(1);
+ Thread t = new Thread(new Runnable() {
+ @Override
+ public void run() {
+ try {
+ FSDataOutputStream flumeOutputStream = fs.append(flumeFile, 8000);
+ FSDataOutputStream hbaseOutputStream = fs.append(hbaseFile, 8000);
+ byte[] bytes = new byte[(int) (1024 * 0.2)];
+ Random r = new Random(Time.now());
+
+ for (int i = 0; i < 200000; i++) {
+ r.nextBytes(bytes);
+ flumeOutputStream.write(bytes);
+ if (hbaseOutputStream != null) {
+ hbaseOutputStream.write(bytes);
+ }
+ if (i == 50000) {
+ startLatch.countDown();
+ } else if (i == 100000) {
+ deleteLatch.countDown();
+ } else if (i == 150000) {
+ hbaseOutputStream.hsync();
+ fs.delete(hbaseFile, true);
+ try {
+ hbaseOutputStream.close();
+ } catch (Exception e) {
+ // since the file is deleted before the open stream close,
+ // it might throw FileNotFoundException. Ignore the
+ // expected exception.
+ }
+ hbaseOutputStream = null;
+ } else if (i % 5000 == 0) {
+ LOG.info("Write pos: " + flumeOutputStream.getPos()
+ + ", size: " + fs.getFileStatus(flumeFile).getLen()
+ + ", loop: " + (i + 1));
+ }
+ }
+ } catch (Exception e) {
+ LOG.warn("Writer error: " + e);
+ writerError.set(true);
+ }
+ }
+ });
+ t.start();
+
+ startLatch.await();
+ final Path snap1Dir = SnapshotTestHelper.createSnapshot(
+ fs, snapRootDir, snap1Name);
+ final Path flumeS1Path = new Path(snap1Dir, flumeFileName);
+ LOG.info("Snap1 file status: " + fs.getFileStatus(flumeS1Path));
+ LOG.info("Current file status: " + fs.getFileStatus(flumeFile));
+
+ deleteLatch.await();
+ LOG.info("Snap1 file status: " + fs.getFileStatus(flumeS1Path));
+ LOG.info("Current file status: " + fs.getFileStatus(flumeFile));
+
+ // Verify deletion of snapshot which had the under construction file
+ // captured is not truncating the under construction file and the thread
+ // writing to the same file not crashing on newer block allocations.
+ LOG.info("Deleting " + snap1Name);
+ fs.deleteSnapshot(snapRootDir, snap1Name);
+
+ // Verify creation and deletion of snapshot newer than the oldest
+ // snapshot is not crashing the thread writing to under construction file.
+ SnapshotTestHelper.createSnapshot(fs, snapRootDir, snap2Name);
+ SnapshotTestHelper.createSnapshot(fs, snapRootDir, snap3Name);
+ fs.deleteSnapshot(snapRootDir, snap3Name);
+ fs.deleteSnapshot(snapRootDir, snap2Name);
+ SnapshotTestHelper.createSnapshot(fs, snapRootDir, "test");
+
+ t.join();
+ Assert.assertFalse("Client encountered writing error!", writerError.get());
+
+ restartNameNode();
+ cluster.waitActive();
+ }
+
private void restartNameNode() throws Exception {
cluster.triggerBlockReports();
NameNode nameNode = cluster.getNameNode();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[30/50] [abbrv] hadoop git commit: YARN-5603. Metrics for Federation
StateStore. (Ellen Hui via asuresh)
Posted by as...@apache.org.
YARN-5603. Metrics for Federation StateStore. (Ellen Hui via asuresh)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/75abc9a8
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/75abc9a8
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/75abc9a8
Branch: refs/heads/YARN-5972
Commit: 75abc9a8e2cf1c7d2c574ede720df59421512be3
Parents: b6bfb2f
Author: Arun Suresh <as...@apache.org>
Authored: Mon Aug 21 22:43:08 2017 -0700
Committer: Arun Suresh <as...@apache.org>
Committed: Mon Aug 21 22:43:08 2017 -0700
----------------------------------------------------------------------
.../store/impl/SQLFederationStateStore.java | 79 ++++++++
.../FederationStateStoreClientMetrics.java | 184 +++++++++++++++++++
.../federation/store/metrics/package-info.java | 17 ++
.../TestFederationStateStoreClientMetrics.java | 146 +++++++++++++++
4 files changed, 426 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/75abc9a8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/SQLFederationStateStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/SQLFederationStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/SQLFederationStateStore.java
index 63d8e42..533f9c8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/SQLFederationStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/SQLFederationStateStore.java
@@ -36,6 +36,7 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.server.federation.store.FederationStateStore;
import org.apache.hadoop.yarn.server.federation.store.exception.FederationStateStoreInvalidInputException;
+import org.apache.hadoop.yarn.server.federation.store.metrics.FederationStateStoreClientMetrics;
import org.apache.hadoop.yarn.server.federation.store.records.AddApplicationHomeSubClusterRequest;
import org.apache.hadoop.yarn.server.federation.store.records.AddApplicationHomeSubClusterResponse;
import org.apache.hadoop.yarn.server.federation.store.records.ApplicationHomeSubCluster;
@@ -72,6 +73,8 @@ import org.apache.hadoop.yarn.server.federation.store.utils.FederationMembership
import org.apache.hadoop.yarn.server.federation.store.utils.FederationPolicyStoreInputValidator;
import org.apache.hadoop.yarn.server.federation.store.utils.FederationStateStoreUtils;
import org.apache.hadoop.yarn.server.records.Version;
+import org.apache.hadoop.yarn.util.Clock;
+import org.apache.hadoop.yarn.util.MonotonicClock;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -137,6 +140,7 @@ public class SQLFederationStateStore implements FederationStateStore {
private String url;
private int maximumPoolSize;
private HikariDataSource dataSource = null;
+ private final Clock clock = new MonotonicClock();
@Override
public void init(Configuration conf) throws YarnException {
@@ -203,7 +207,9 @@ public class SQLFederationStateStore implements FederationStateStore {
cstmt.registerOutParameter(9, java.sql.Types.INTEGER);
// Execute the query
+ long startTime = clock.getTime();
cstmt.executeUpdate();
+ long stopTime = clock.getTime();
// Check the ROWCOUNT value, if it is equal to 0 it means the call
// did not add a new subcluster into FederationStateStore
@@ -222,8 +228,11 @@ public class SQLFederationStateStore implements FederationStateStore {
LOG.info(
"Registered the SubCluster " + subClusterId + " into the StateStore");
+ FederationStateStoreClientMetrics
+ .succeededStateStoreCall(stopTime - startTime);
} catch (SQLException e) {
+ FederationStateStoreClientMetrics.failedStateStoreCall();
FederationStateStoreUtils.logAndThrowRetriableException(LOG,
"Unable to register the SubCluster " + subClusterId
+ " into the StateStore",
@@ -260,7 +269,9 @@ public class SQLFederationStateStore implements FederationStateStore {
cstmt.registerOutParameter(3, java.sql.Types.INTEGER);
// Execute the query
+ long startTime = clock.getTime();
cstmt.executeUpdate();
+ long stopTime = clock.getTime();
// Check the ROWCOUNT value, if it is equal to 0 it means the call
// did not deregister the subcluster into FederationStateStore
@@ -278,8 +289,11 @@ public class SQLFederationStateStore implements FederationStateStore {
LOG.info("Deregistered the SubCluster " + subClusterId + " state to "
+ state.toString());
+ FederationStateStoreClientMetrics
+ .succeededStateStoreCall(stopTime - startTime);
} catch (SQLException e) {
+ FederationStateStoreClientMetrics.failedStateStoreCall();
FederationStateStoreUtils.logAndThrowRetriableException(LOG,
"Unable to deregister the sub-cluster " + subClusterId + " state to "
+ state.toString(),
@@ -317,7 +331,9 @@ public class SQLFederationStateStore implements FederationStateStore {
cstmt.registerOutParameter(4, java.sql.Types.INTEGER);
// Execute the query
+ long startTime = clock.getTime();
cstmt.executeUpdate();
+ long stopTime = clock.getTime();
// Check the ROWCOUNT value, if it is equal to 0 it means the call
// did not update the subcluster into FederationStateStore
@@ -336,8 +352,11 @@ public class SQLFederationStateStore implements FederationStateStore {
LOG.info("Heartbeated the StateStore for the specified SubCluster "
+ subClusterId);
+ FederationStateStoreClientMetrics
+ .succeededStateStoreCall(stopTime - startTime);
} catch (SQLException e) {
+ FederationStateStoreClientMetrics.failedStateStoreCall();
FederationStateStoreUtils.logAndThrowRetriableException(LOG,
"Unable to heartbeat the StateStore for the specified SubCluster "
+ subClusterId,
@@ -378,7 +397,9 @@ public class SQLFederationStateStore implements FederationStateStore {
cstmt.registerOutParameter(9, java.sql.Types.VARCHAR);
// Execute the query
+ long startTime = clock.getTime();
cstmt.execute();
+ long stopTime = clock.getTime();
String amRMAddress = cstmt.getString(2);
String clientRMAddress = cstmt.getString(3);
@@ -403,6 +424,9 @@ public class SQLFederationStateStore implements FederationStateStore {
clientRMAddress, rmAdminAddress, webAppAddress, lastHeartBeat, state,
lastStartTime, capability);
+ FederationStateStoreClientMetrics
+ .succeededStateStoreCall(stopTime - startTime);
+
// Check if the output it is a valid subcluster
try {
FederationMembershipStateStoreInputValidator
@@ -417,6 +441,7 @@ public class SQLFederationStateStore implements FederationStateStore {
+ subClusterInfo.toString());
}
} catch (SQLException e) {
+ FederationStateStoreClientMetrics.failedStateStoreCall();
FederationStateStoreUtils.logAndThrowRetriableException(LOG,
"Unable to obtain the SubCluster information for " + subClusterId, e);
} finally {
@@ -439,7 +464,9 @@ public class SQLFederationStateStore implements FederationStateStore {
cstmt = conn.prepareCall(CALL_SP_GET_SUBCLUSTERS);
// Execute the query
+ long startTime = clock.getTime();
rs = cstmt.executeQuery();
+ long stopTime = clock.getTime();
while (rs.next()) {
@@ -459,6 +486,10 @@ public class SQLFederationStateStore implements FederationStateStore {
amRMAddress, clientRMAddress, rmAdminAddress, webAppAddress,
lastHeartBeat, state, lastStartTime, capability);
+ FederationStateStoreClientMetrics
+ .succeededStateStoreCall(stopTime - startTime);
+
+
// Check if the output it is a valid subcluster
try {
FederationMembershipStateStoreInputValidator
@@ -477,6 +508,7 @@ public class SQLFederationStateStore implements FederationStateStore {
}
} catch (SQLException e) {
+ FederationStateStoreClientMetrics.failedStateStoreCall();
FederationStateStoreUtils.logAndThrowRetriableException(LOG,
"Unable to obtain the information for all the SubClusters ", e);
} finally {
@@ -513,11 +545,16 @@ public class SQLFederationStateStore implements FederationStateStore {
cstmt.registerOutParameter(4, java.sql.Types.INTEGER);
// Execute the query
+ long startTime = clock.getTime();
cstmt.executeUpdate();
+ long stopTime = clock.getTime();
subClusterHome = cstmt.getString(3);
SubClusterId subClusterIdHome = SubClusterId.newInstance(subClusterHome);
+ FederationStateStoreClientMetrics
+ .succeededStateStoreCall(stopTime - startTime);
+
// For failover reason, we check the returned SubClusterId.
// If it is equal to the subclusterId we sent, the call added the new
// application into FederationStateStore. If the call returns a different
@@ -554,6 +591,7 @@ public class SQLFederationStateStore implements FederationStateStore {
}
} catch (SQLException e) {
+ FederationStateStoreClientMetrics.failedStateStoreCall();
FederationStateStoreUtils
.logAndThrowRetriableException(LOG,
"Unable to insert the newly generated application "
@@ -592,7 +630,9 @@ public class SQLFederationStateStore implements FederationStateStore {
cstmt.registerOutParameter(3, java.sql.Types.INTEGER);
// Execute the query
+ long startTime = clock.getTime();
cstmt.executeUpdate();
+ long stopTime = clock.getTime();
// Check the ROWCOUNT value, if it is equal to 0 it means the call
// did not update the application into FederationStateStore
@@ -611,8 +651,11 @@ public class SQLFederationStateStore implements FederationStateStore {
LOG.info(
"Update the SubCluster to {} for application {} in the StateStore",
subClusterId, appId);
+ FederationStateStoreClientMetrics
+ .succeededStateStoreCall(stopTime - startTime);
} catch (SQLException e) {
+ FederationStateStoreClientMetrics.failedStateStoreCall();
FederationStateStoreUtils
.logAndThrowRetriableException(LOG,
"Unable to update the application "
@@ -645,7 +688,9 @@ public class SQLFederationStateStore implements FederationStateStore {
cstmt.registerOutParameter(2, java.sql.Types.VARCHAR);
// Execute the query
+ long startTime = clock.getTime();
cstmt.execute();
+ long stopTime = clock.getTime();
if (cstmt.getString(2) != null) {
homeRM = SubClusterId.newInstance(cstmt.getString(2));
@@ -659,7 +704,12 @@ public class SQLFederationStateStore implements FederationStateStore {
LOG.debug("Got the information about the specified application "
+ request.getApplicationId() + ". The AM is running in " + homeRM);
}
+
+ FederationStateStoreClientMetrics
+ .succeededStateStoreCall(stopTime - startTime);
+
} catch (SQLException e) {
+ FederationStateStoreClientMetrics.failedStateStoreCall();
FederationStateStoreUtils.logAndThrowRetriableException(LOG,
"Unable to obtain the application information "
+ "for the specified application " + request.getApplicationId(),
@@ -688,7 +738,9 @@ public class SQLFederationStateStore implements FederationStateStore {
cstmt = conn.prepareCall(CALL_SP_GET_APPLICATIONS_HOME_SUBCLUSTER);
// Execute the query
+ long startTime = clock.getTime();
rs = cstmt.executeQuery();
+ long stopTime = clock.getTime();
while (rs.next()) {
@@ -701,7 +753,11 @@ public class SQLFederationStateStore implements FederationStateStore {
SubClusterId.newInstance(homeSubCluster)));
}
+ FederationStateStoreClientMetrics
+ .succeededStateStoreCall(stopTime - startTime);
+
} catch (SQLException e) {
+ FederationStateStoreClientMetrics.failedStateStoreCall();
FederationStateStoreUtils.logAndThrowRetriableException(LOG,
"Unable to obtain the information for all the applications ", e);
} finally {
@@ -731,7 +787,9 @@ public class SQLFederationStateStore implements FederationStateStore {
cstmt.registerOutParameter(2, java.sql.Types.INTEGER);
// Execute the query
+ long startTime = clock.getTime();
cstmt.executeUpdate();
+ long stopTime = clock.getTime();
// Check the ROWCOUNT value, if it is equal to 0 it means the call
// did not delete the application from FederationStateStore
@@ -750,8 +808,11 @@ public class SQLFederationStateStore implements FederationStateStore {
LOG.info("Delete from the StateStore the application: {}",
request.getApplicationId());
+ FederationStateStoreClientMetrics
+ .succeededStateStoreCall(stopTime - startTime);
} catch (SQLException e) {
+ FederationStateStoreClientMetrics.failedStateStoreCall();
FederationStateStoreUtils.logAndThrowRetriableException(LOG,
"Unable to delete the application " + request.getApplicationId(), e);
} finally {
@@ -782,7 +843,9 @@ public class SQLFederationStateStore implements FederationStateStore {
cstmt.registerOutParameter(3, java.sql.Types.VARBINARY);
// Execute the query
+ long startTime = clock.getTime();
cstmt.executeUpdate();
+ long stopTime = clock.getTime();
// Check if the output it is a valid policy
if (cstmt.getString(2) != null && cstmt.getBytes(3) != null) {
@@ -798,7 +861,11 @@ public class SQLFederationStateStore implements FederationStateStore {
return null;
}
+ FederationStateStoreClientMetrics
+ .succeededStateStoreCall(stopTime - startTime);
+
} catch (SQLException e) {
+ FederationStateStoreClientMetrics.failedStateStoreCall();
FederationStateStoreUtils.logAndThrowRetriableException(LOG,
"Unable to select the policy for the queue :" + request.getQueue(),
e);
@@ -833,7 +900,9 @@ public class SQLFederationStateStore implements FederationStateStore {
cstmt.registerOutParameter(4, java.sql.Types.INTEGER);
// Execute the query
+ long startTime = clock.getTime();
cstmt.executeUpdate();
+ long stopTime = clock.getTime();
// Check the ROWCOUNT value, if it is equal to 0 it means the call
// did not add a new policy into FederationStateStore
@@ -852,8 +921,11 @@ public class SQLFederationStateStore implements FederationStateStore {
LOG.info("Insert into the state store the policy for the queue: "
+ policyConf.getQueue());
+ FederationStateStoreClientMetrics
+ .succeededStateStoreCall(stopTime - startTime);
} catch (SQLException e) {
+ FederationStateStoreClientMetrics.failedStateStoreCall();
FederationStateStoreUtils.logAndThrowRetriableException(LOG,
"Unable to insert the newly generated policy for the queue :"
+ policyConf.getQueue(),
@@ -880,7 +952,9 @@ public class SQLFederationStateStore implements FederationStateStore {
cstmt = conn.prepareCall(CALL_SP_GET_POLICIES_CONFIGURATIONS);
// Execute the query
+ long startTime = clock.getTime();
rs = cstmt.executeQuery();
+ long stopTime = clock.getTime();
while (rs.next()) {
@@ -894,7 +968,12 @@ public class SQLFederationStateStore implements FederationStateStore {
ByteBuffer.wrap(policyInfo));
policyConfigurations.add(subClusterPolicyConfiguration);
}
+
+ FederationStateStoreClientMetrics
+ .succeededStateStoreCall(stopTime - startTime);
+
} catch (SQLException e) {
+ FederationStateStoreClientMetrics.failedStateStoreCall();
FederationStateStoreUtils.logAndThrowRetriableException(LOG,
"Unable to obtain the policy information for all the queues.", e);
} finally {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/75abc9a8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/metrics/FederationStateStoreClientMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/metrics/FederationStateStoreClientMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/metrics/FederationStateStoreClientMetrics.java
new file mode 100644
index 0000000..27b46cd
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/metrics/FederationStateStoreClientMetrics.java
@@ -0,0 +1,184 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.metrics;
+
+import java.lang.reflect.Method;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+import org.apache.hadoop.metrics2.MetricsCollector;
+import org.apache.hadoop.metrics2.MetricsSource;
+import org.apache.hadoop.metrics2.annotation.Metric;
+import org.apache.hadoop.metrics2.annotation.Metrics;
+import org.apache.hadoop.metrics2.lib.DefaultMetricsSystem;
+import org.apache.hadoop.metrics2.lib.MetricsRegistry;
+import org.apache.hadoop.metrics2.lib.MutableCounterLong;
+import org.apache.hadoop.metrics2.lib.MutableQuantiles;
+import org.apache.hadoop.metrics2.lib.MutableRate;
+import org.apache.hadoop.yarn.server.federation.store.FederationStateStore;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.common.annotations.VisibleForTesting;
+
+/**
+ * Performance metrics for FederationStateStore implementations.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+@Metrics(about = "Performance and usage metrics for Federation StateStore",
+ context = "fedr")
+public final class FederationStateStoreClientMetrics implements MetricsSource {
+ public static final Logger LOG =
+ LoggerFactory.getLogger(FederationStateStoreClientMetrics.class);
+
+ private static final MetricsRegistry REGISTRY =
+ new MetricsRegistry("FederationStateStoreClientMetrics");
+ private final static Method[] STATESTORE_API_METHODS =
+ FederationStateStore.class.getMethods();
+
+ // Map method names to counter objects
+ private static final Map<String, MutableCounterLong> API_TO_FAILED_CALLS =
+ new HashMap<String, MutableCounterLong>();
+ private static final Map<String, MutableRate> API_TO_SUCCESSFUL_CALLS =
+ new HashMap<String, MutableRate>();
+
+ // Provide quantile latency for each api call.
+ private static final Map<String, MutableQuantiles> API_TO_QUANTILE_METRICS =
+ new HashMap<String, MutableQuantiles>();
+
+ // Error string templates for logging calls from methods not in
+ // FederationStateStore API
+ private static final String UNKOWN_FAIL_ERROR_MSG =
+ "Not recording failed call for unknown FederationStateStore method {}";
+ private static final String UNKNOWN_SUCCESS_ERROR_MSG =
+ "Not recording successful call for unknown "
+ + "FederationStateStore method {}";
+
+ // Aggregate metrics are shared, and don't have to be looked up per call
+ @Metric("Total number of successful calls and latency(ms)")
+ private static MutableRate totalSucceededCalls;
+
+ @Metric("Total number of failed StateStore calls")
+ private static MutableCounterLong totalFailedCalls;
+
+ // This after the static members are initialized, or the constructor will
+ // throw a NullPointerException
+ private static final FederationStateStoreClientMetrics S_INSTANCE =
+ DefaultMetricsSystem.instance()
+ .register(new FederationStateStoreClientMetrics());
+
+ synchronized public static FederationStateStoreClientMetrics getInstance() {
+ return S_INSTANCE;
+ }
+
+ private FederationStateStoreClientMetrics() {
+ // Create the metrics for each method and put them into the map
+ for (Method m : STATESTORE_API_METHODS) {
+ String methodName = m.getName();
+ LOG.debug("Registering Federation StateStore Client metrics for {}",
+ methodName);
+
+ // This metric only records the number of failed calls; it does not
+ // capture latency information
+ API_TO_FAILED_CALLS.put(methodName,
+ REGISTRY.newCounter(methodName + "_numFailedCalls",
+ "# failed calls to " + methodName, 0L));
+
+ // This metric records both the number and average latency of successful
+ // calls.
+ API_TO_SUCCESSFUL_CALLS.put(methodName,
+ REGISTRY.newRate(methodName + "_successfulCalls",
+ "# successful calls and latency(ms) for" + methodName));
+
+ // This metric records the quantile-based latency of each successful call,
+ // re-sampled every 10 seconds.
+ API_TO_QUANTILE_METRICS.put(methodName,
+ REGISTRY.newQuantiles(methodName + "Latency",
+ "Quantile latency (ms) for " + methodName, "ops", "latency", 10));
+ }
+ }
+
+ public static void failedStateStoreCall() {
+ String methodName =
+ Thread.currentThread().getStackTrace()[2].getMethodName();
+ MutableCounterLong methodMetric = API_TO_FAILED_CALLS.get(methodName);
+ if (methodMetric == null) {
+ LOG.error(UNKOWN_FAIL_ERROR_MSG, methodName);
+ return;
+ }
+
+ totalFailedCalls.incr();
+ methodMetric.incr();
+ }
+
+ public static void succeededStateStoreCall(long duration) {
+ String methodName =
+ Thread.currentThread().getStackTrace()[2].getMethodName();
+ MutableRate methodMetric = API_TO_SUCCESSFUL_CALLS.get(methodName);
+ MutableQuantiles methodQuantileMetric =
+ API_TO_QUANTILE_METRICS.get(methodName);
+ if (methodMetric == null || methodQuantileMetric == null) {
+ LOG.error(UNKNOWN_SUCCESS_ERROR_MSG, methodName);
+ return;
+ }
+
+ totalSucceededCalls.add(duration);
+ methodMetric.add(duration);
+ methodQuantileMetric.add(duration);
+ }
+
+ @Override
+ public void getMetrics(MetricsCollector collector, boolean all) {
+ REGISTRY.snapshot(collector.addRecord(REGISTRY.info()), all);
+ }
+
+ // Getters for unit testing
+ @VisibleForTesting
+ static long getNumFailedCallsForMethod(String methodName) {
+ return API_TO_FAILED_CALLS.get(methodName).value();
+ }
+
+ @VisibleForTesting
+ static long getNumSucceessfulCallsForMethod(String methodName) {
+ return API_TO_SUCCESSFUL_CALLS.get(methodName).lastStat().numSamples();
+ }
+
+ @VisibleForTesting
+ static double getLatencySucceessfulCallsForMethod(String methodName) {
+ return API_TO_SUCCESSFUL_CALLS.get(methodName).lastStat().mean();
+ }
+
+ @VisibleForTesting
+ static long getNumFailedCalls() {
+ return totalFailedCalls.value();
+ }
+
+ @VisibleForTesting
+ static long getNumSucceededCalls() {
+ return totalSucceededCalls.lastStat().numSamples();
+ }
+
+ @VisibleForTesting
+ static double getLatencySucceededCalls() {
+ return totalSucceededCalls.lastStat().mean();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/75abc9a8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/metrics/package-info.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/metrics/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/metrics/package-info.java
new file mode 100644
index 0000000..eb548f4
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/metrics/package-info.java
@@ -0,0 +1,17 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.yarn.server.federation.store.metrics;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/75abc9a8/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/metrics/TestFederationStateStoreClientMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/metrics/TestFederationStateStoreClientMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/metrics/TestFederationStateStoreClientMetrics.java
new file mode 100644
index 0000000..241d5e2
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/metrics/TestFederationStateStoreClientMetrics.java
@@ -0,0 +1,146 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.metrics;
+
+import org.junit.Assert;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Unittests for {@link FederationStateStoreClientMetrics}.
+ *
+ */
+public class TestFederationStateStoreClientMetrics {
+ public static final Logger LOG =
+ LoggerFactory.getLogger(TestFederationStateStoreClientMetrics.class);
+
+ private MockBadFederationStateStore badStateStore =
+ new MockBadFederationStateStore();
+ private MockGoodFederationStateStore goodStateStore =
+ new MockGoodFederationStateStore();
+
+ @Test
+ public void testAggregateMetricInit() {
+ LOG.info("Test: aggregate metrics are initialized correctly");
+
+ Assert.assertEquals(0,
+ FederationStateStoreClientMetrics.getNumSucceededCalls());
+ Assert.assertEquals(0,
+ FederationStateStoreClientMetrics.getNumFailedCalls());
+
+ LOG.info("Test: aggregate metrics are updated correctly");
+ }
+
+ @Test
+ public void testSuccessfulCalls() {
+ LOG.info("Test: Aggregate and method successful calls updated correctly");
+
+ long totalGoodBefore =
+ FederationStateStoreClientMetrics.getNumSucceededCalls();
+ long apiGoodBefore = FederationStateStoreClientMetrics
+ .getNumSucceessfulCallsForMethod("registerSubCluster");
+
+ goodStateStore.registerSubCluster(100);
+
+ Assert.assertEquals(totalGoodBefore + 1,
+ FederationStateStoreClientMetrics.getNumSucceededCalls());
+ Assert.assertEquals(100,
+ FederationStateStoreClientMetrics.getLatencySucceededCalls(), 0);
+ Assert.assertEquals(apiGoodBefore + 1,
+ FederationStateStoreClientMetrics.getNumSucceededCalls());
+ Assert.assertEquals(100, FederationStateStoreClientMetrics
+ .getLatencySucceessfulCallsForMethod("registerSubCluster"), 0);
+
+ LOG.info("Test: Running stats correctly calculated for 2 metrics");
+
+ goodStateStore.registerSubCluster(200);
+
+ Assert.assertEquals(totalGoodBefore + 2,
+ FederationStateStoreClientMetrics.getNumSucceededCalls());
+ Assert.assertEquals(150,
+ FederationStateStoreClientMetrics.getLatencySucceededCalls(), 0);
+ Assert.assertEquals(apiGoodBefore + 2,
+ FederationStateStoreClientMetrics.getNumSucceededCalls());
+ Assert.assertEquals(150, FederationStateStoreClientMetrics
+ .getLatencySucceessfulCallsForMethod("registerSubCluster"), 0);
+
+ }
+
+ @Test
+ public void testFailedCalls() {
+
+ long totalBadbefore = FederationStateStoreClientMetrics.getNumFailedCalls();
+ long apiBadBefore = FederationStateStoreClientMetrics
+ .getNumFailedCallsForMethod("registerSubCluster");
+
+ badStateStore.registerSubCluster();
+
+ LOG.info("Test: Aggregate and method failed calls updated correctly");
+ Assert.assertEquals(totalBadbefore + 1,
+ FederationStateStoreClientMetrics.getNumFailedCalls());
+ Assert.assertEquals(apiBadBefore + 1, FederationStateStoreClientMetrics
+ .getNumFailedCallsForMethod("registerSubCluster"));
+
+ }
+
+ @Test
+ public void testCallsUnknownMethod() {
+
+ long totalBadbefore = FederationStateStoreClientMetrics.getNumFailedCalls();
+ long apiBadBefore = FederationStateStoreClientMetrics
+ .getNumFailedCallsForMethod("registerSubCluster");
+ long totalGoodBefore =
+ FederationStateStoreClientMetrics.getNumSucceededCalls();
+ long apiGoodBefore = FederationStateStoreClientMetrics
+ .getNumSucceessfulCallsForMethod("registerSubCluster");
+
+ LOG.info("Calling Metrics class directly");
+ FederationStateStoreClientMetrics.failedStateStoreCall();
+ FederationStateStoreClientMetrics.succeededStateStoreCall(100);
+
+ LOG.info("Test: Aggregate and method calls did not update");
+ Assert.assertEquals(totalBadbefore,
+ FederationStateStoreClientMetrics.getNumFailedCalls());
+ Assert.assertEquals(apiBadBefore, FederationStateStoreClientMetrics
+ .getNumFailedCallsForMethod("registerSubCluster"));
+
+ Assert.assertEquals(totalGoodBefore,
+ FederationStateStoreClientMetrics.getNumSucceededCalls());
+ Assert.assertEquals(apiGoodBefore, FederationStateStoreClientMetrics
+ .getNumSucceessfulCallsForMethod("registerSubCluster"));
+
+ }
+
+ // Records failures for all calls
+ private class MockBadFederationStateStore {
+ public void registerSubCluster() {
+ LOG.info("Mocked: failed registerSubCluster call");
+ FederationStateStoreClientMetrics.failedStateStoreCall();
+ }
+ }
+
+ // Records successes for all calls
+ private class MockGoodFederationStateStore {
+ public void registerSubCluster(long duration) {
+ LOG.info("Mocked: successful registerSubCluster call with duration {}",
+ duration);
+ FederationStateStoreClientMetrics.succeededStateStoreCall(duration);
+ }
+ }
+}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[29/50] [abbrv] hadoop git commit: HDFS-11738. Hedged pread takes
more time when block moved from initial locations. Contributed by Vinayakumar
B.
Posted by as...@apache.org.
HDFS-11738. Hedged pread takes more time when block moved from initial locations. Contributed by Vinayakumar B.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b6bfb2fc
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b6bfb2fc
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b6bfb2fc
Branch: refs/heads/YARN-5972
Commit: b6bfb2fcb2391d51b8de97c01c1290880779132e
Parents: 736ceab
Author: John Zhuge <jz...@apache.org>
Authored: Mon Aug 21 13:44:32 2017 -0700
Committer: John Zhuge <jz...@apache.org>
Committed: Mon Aug 21 13:45:30 2017 -0700
----------------------------------------------------------------------
.../hadoop/hdfs/DFSClientFaultInjector.java | 2 +
.../org/apache/hadoop/hdfs/DFSInputStream.java | 145 +++++++++++--------
.../java/org/apache/hadoop/hdfs/TestPread.java | 26 +++-
3 files changed, 112 insertions(+), 61 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6bfb2fc/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClientFaultInjector.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClientFaultInjector.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClientFaultInjector.java
index 748edcd..b58cf16 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClientFaultInjector.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSClientFaultInjector.java
@@ -61,4 +61,6 @@ public class DFSClientFaultInjector {
public boolean skipRollingRestartWait() {
return false;
}
+
+ public void sleepBeforeHedgedGet() {}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6bfb2fc/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
index 6bff172..97d3de4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-client/src/main/java/org/apache/hadoop/hdfs/DFSInputStream.java
@@ -830,60 +830,85 @@ public class DFSInputStream extends FSInputStream
private DNAddrPair chooseDataNode(LocatedBlock block,
Collection<DatanodeInfo> ignoredNodes) throws IOException {
+ return chooseDataNode(block, ignoredNodes, true);
+ }
+
+ /**
+ * Choose datanode to read from.
+ *
+ * @param block Block to choose datanode addr from
+ * @param ignoredNodes Ignored nodes inside.
+ * @param refetchIfRequired Whether to refetch if no nodes to chose
+ * from.
+ * @return Returns chosen DNAddrPair; Can be null if refetchIfRequired is
+ * false.
+ */
+ private DNAddrPair chooseDataNode(LocatedBlock block,
+ Collection<DatanodeInfo> ignoredNodes, boolean refetchIfRequired)
+ throws IOException {
while (true) {
DNAddrPair result = getBestNodeDNAddrPair(block, ignoredNodes);
if (result != null) {
return result;
+ } else if (refetchIfRequired) {
+ block = refetchLocations(block, ignoredNodes);
} else {
- String errMsg = getBestNodeDNAddrPairErrorString(block.getLocations(),
- deadNodes, ignoredNodes);
- String blockInfo = block.getBlock() + " file=" + src;
- if (failures >= dfsClient.getConf().getMaxBlockAcquireFailures()) {
- String description = "Could not obtain block: " + blockInfo;
- DFSClient.LOG.warn(description + errMsg
- + ". Throwing a BlockMissingException");
- throw new BlockMissingException(src, description,
- block.getStartOffset());
- }
-
- DatanodeInfo[] nodes = block.getLocations();
- if (nodes == null || nodes.length == 0) {
- DFSClient.LOG.info("No node available for " + blockInfo);
- }
- DFSClient.LOG.info("Could not obtain " + block.getBlock()
- + " from any node: " + errMsg
- + ". Will get new block locations from namenode and retry...");
- try {
- // Introducing a random factor to the wait time before another retry.
- // The wait time is dependent on # of failures and a random factor.
- // At the first time of getting a BlockMissingException, the wait time
- // is a random number between 0..3000 ms. If the first retry
- // still fails, we will wait 3000 ms grace period before the 2nd retry.
- // Also at the second retry, the waiting window is expanded to 6000 ms
- // alleviating the request rate from the server. Similarly the 3rd retry
- // will wait 6000ms grace period before retry and the waiting window is
- // expanded to 9000ms.
- final int timeWindow = dfsClient.getConf().getTimeWindow();
- double waitTime = timeWindow * failures + // grace period for the last round of attempt
- // expanding time window for each failure
- timeWindow * (failures + 1) *
- ThreadLocalRandom.current().nextDouble();
- DFSClient.LOG.warn("DFS chooseDataNode: got # " + (failures + 1) +
- " IOException, will wait for " + waitTime + " msec.");
- Thread.sleep((long)waitTime);
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- throw new InterruptedIOException(
- "Interrupted while choosing DataNode for read.");
- }
- deadNodes.clear(); //2nd option is to remove only nodes[blockId]
- openInfo(true);
- block = refreshLocatedBlock(block);
- failures++;
+ return null;
}
}
}
+ private LocatedBlock refetchLocations(LocatedBlock block,
+ Collection<DatanodeInfo> ignoredNodes) throws IOException {
+ String errMsg = getBestNodeDNAddrPairErrorString(block.getLocations(),
+ deadNodes, ignoredNodes);
+ String blockInfo = block.getBlock() + " file=" + src;
+ if (failures >= dfsClient.getConf().getMaxBlockAcquireFailures()) {
+ String description = "Could not obtain block: " + blockInfo;
+ DFSClient.LOG.warn(description + errMsg
+ + ". Throwing a BlockMissingException");
+ throw new BlockMissingException(src, description,
+ block.getStartOffset());
+ }
+
+ DatanodeInfo[] nodes = block.getLocations();
+ if (nodes == null || nodes.length == 0) {
+ DFSClient.LOG.info("No node available for " + blockInfo);
+ }
+ DFSClient.LOG.info("Could not obtain " + block.getBlock()
+ + " from any node: " + errMsg
+ + ". Will get new block locations from namenode and retry...");
+ try {
+ // Introducing a random factor to the wait time before another retry.
+ // The wait time is dependent on # of failures and a random factor.
+ // At the first time of getting a BlockMissingException, the wait time
+ // is a random number between 0..3000 ms. If the first retry
+ // still fails, we will wait 3000 ms grace period before the 2nd retry.
+ // Also at the second retry, the waiting window is expanded to 6000 ms
+ // alleviating the request rate from the server. Similarly the 3rd retry
+ // will wait 6000ms grace period before retry and the waiting window is
+ // expanded to 9000ms.
+ final int timeWindow = dfsClient.getConf().getTimeWindow();
+ // grace period for the last round of attempt
+ double waitTime = timeWindow * failures +
+ // expanding time window for each failure
+ timeWindow * (failures + 1) *
+ ThreadLocalRandom.current().nextDouble();
+ DFSClient.LOG.warn("DFS chooseDataNode: got # " + (failures + 1) +
+ " IOException, will wait for " + waitTime + " msec.");
+ Thread.sleep((long)waitTime);
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ throw new InterruptedIOException(
+ "Interrupted while choosing DataNode for read.");
+ }
+ deadNodes.clear(); //2nd option is to remove only nodes[blockId]
+ openInfo(true);
+ block = refreshLocatedBlock(block);
+ failures++;
+ return block;
+ }
+
/**
* Get the best node from which to stream the data.
* @param block LocatedBlock, containing nodes in priority order.
@@ -985,6 +1010,7 @@ public class DFSInputStream extends FSInputStream
return new Callable<ByteBuffer>() {
@Override
public ByteBuffer call() throws Exception {
+ DFSClientFaultInjector.get().sleepBeforeHedgedGet();
try (TraceScope ignored = dfsClient.getTracer().
newScope("hedgedRead" + hedgedReadId, parentSpanId)) {
actualGetFromOneDataNode(datanode, start, end, bb, corruptedBlocks);
@@ -1159,20 +1185,22 @@ public class DFSInputStream extends FSInputStream
// We are starting up a 'hedged' read. We have a read already
// ongoing. Call getBestNodeDNAddrPair instead of chooseDataNode.
// If no nodes to do hedged reads against, pass.
+ boolean refetch = false;
try {
- chosenNode = getBestNodeDNAddrPair(block, ignored);
- if (chosenNode == null) {
- chosenNode = chooseDataNode(block, ignored);
+ chosenNode = chooseDataNode(block, ignored, false);
+ if (chosenNode != null) {
+ // Latest block, if refreshed internally
+ block = chosenNode.block;
+ bb = ByteBuffer.allocate(len);
+ Callable<ByteBuffer> getFromDataNodeCallable =
+ getFromOneDataNode(chosenNode, block, start, end, bb,
+ corruptedBlocks, hedgedReadId++);
+ Future<ByteBuffer> oneMoreRequest =
+ hedgedService.submit(getFromDataNodeCallable);
+ futures.add(oneMoreRequest);
+ } else {
+ refetch = true;
}
- // Latest block, if refreshed internally
- block = chosenNode.block;
- bb = ByteBuffer.allocate(len);
- Callable<ByteBuffer> getFromDataNodeCallable = getFromOneDataNode(
- chosenNode, block, start, end, bb,
- corruptedBlocks, hedgedReadId++);
- Future<ByteBuffer> oneMoreRequest = hedgedService
- .submit(getFromDataNodeCallable);
- futures.add(oneMoreRequest);
} catch (IOException ioe) {
DFSClient.LOG.debug("Failed getting node for hedged read: {}",
ioe.getMessage());
@@ -1190,6 +1218,9 @@ public class DFSInputStream extends FSInputStream
} catch (InterruptedException ie) {
// Ignore and retry
}
+ if (refetch) {
+ refetchLocations(block, ignored);
+ }
// We got here if exception. Ignore this node on next go around IFF
// we found a chosenNode to hedge read against.
if (chosenNode != null && chosenNode.info != null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b6bfb2fc/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestPread.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestPread.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestPread.java
index bcb02b3..0834d30 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestPread.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestPread.java
@@ -626,7 +626,7 @@ public class TestPread {
*/
@Test
public void testPreadFailureWithChangedBlockLocations() throws Exception {
- doPreadTestWithChangedLocations();
+ doPreadTestWithChangedLocations(1);
}
/**
@@ -639,21 +639,36 @@ public class TestPread {
* 7. Consider next calls to getBlockLocations() always returns DN3 as last
* location.<br>
*/
- @Test
+ @Test(timeout = 60000)
public void testPreadHedgedFailureWithChangedBlockLocations()
throws Exception {
isHedgedRead = true;
- doPreadTestWithChangedLocations();
+ DFSClientFaultInjector old = DFSClientFaultInjector.get();
+ try {
+ DFSClientFaultInjector.set(new DFSClientFaultInjector() {
+ public void sleepBeforeHedgedGet() {
+ try {
+ Thread.sleep(500);
+ } catch (InterruptedException e) {
+ }
+ }
+ });
+ doPreadTestWithChangedLocations(2);
+ } finally {
+ DFSClientFaultInjector.set(old);
+ }
}
- private void doPreadTestWithChangedLocations()
+ private void doPreadTestWithChangedLocations(int maxFailures)
throws IOException, TimeoutException, InterruptedException {
GenericTestUtils.setLogLevel(DFSClient.LOG, Level.DEBUG);
Configuration conf = new HdfsConfiguration();
conf.setInt(DFSConfigKeys.DFS_REPLICATION_KEY, 2);
conf.setInt(DFSConfigKeys.DFS_HEARTBEAT_INTERVAL_KEY, 1);
if (isHedgedRead) {
+ conf.setInt(HdfsClientConfigKeys.HedgedRead.THRESHOLD_MILLIS_KEY, 100);
conf.setInt(HdfsClientConfigKeys.HedgedRead.THREADPOOL_SIZE_KEY, 2);
+ conf.setInt(HdfsClientConfigKeys.Retry.WINDOW_BASE_KEY, 1000);
}
try (MiniDFSCluster cluster =
new MiniDFSCluster.Builder(conf).numDataNodes(3).build()) {
@@ -747,6 +762,9 @@ public class TestPread {
int n = din.read(0, buf, 0, data.length());
assertEquals(data.length(), n);
assertEquals("Data should be read", data, new String(buf, 0, n));
+ assertTrue("Read should complete with maximum " + maxFailures
+ + " failures, but completed with " + din.failures,
+ din.failures <= maxFailures);
DFSClient.LOG.info("Read completed");
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[16/50] [abbrv] hadoop git commit: HDFS-12072. Provide fairness
between EC and non-EC recovery tasks. Contributed by Eddy Xu.
Posted by as...@apache.org.
HDFS-12072. Provide fairness between EC and non-EC recovery tasks. Contributed by Eddy Xu.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/b2989488
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/b2989488
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/b2989488
Branch: refs/heads/YARN-5972
Commit: b29894889742dda654cd88a7ce72a4e51fccb328
Parents: ab1a8ae
Author: Andrew Wang <wa...@apache.org>
Authored: Thu Aug 17 15:26:11 2017 -0700
Committer: Andrew Wang <wa...@apache.org>
Committed: Thu Aug 17 15:26:11 2017 -0700
----------------------------------------------------------------------
.../blockmanagement/DatanodeDescriptor.java | 6 +-
.../server/blockmanagement/DatanodeManager.java | 45 ++++++---
.../blockmanagement/TestDatanodeManager.java | 96 +++++++++++++++-----
3 files changed, 108 insertions(+), 39 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2989488/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
index 2bd4a20..d35894c 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeDescriptor.java
@@ -661,7 +661,11 @@ public class DatanodeDescriptor extends DatanodeInfo {
return erasurecodeBlocks.size();
}
- public List<BlockTargetPair> getReplicationCommand(int maxTransfers) {
+ int getNumberOfReplicateBlocks() {
+ return replicateBlocks.size();
+ }
+
+ List<BlockTargetPair> getReplicationCommand(int maxTransfers) {
return replicateBlocks.poll(maxTransfers);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2989488/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
index 78783ca..c75bcea 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/blockmanagement/DatanodeManager.java
@@ -1663,21 +1663,38 @@ public class DatanodeManager {
}
final List<DatanodeCommand> cmds = new ArrayList<>();
- // check pending replication
- List<BlockTargetPair> pendingList = nodeinfo.getReplicationCommand(
- maxTransfers);
- if (pendingList != null) {
- cmds.add(new BlockCommand(DatanodeProtocol.DNA_TRANSFER, blockPoolId,
- pendingList));
- maxTransfers -= pendingList.size();
- }
- // check pending erasure coding tasks
- List<BlockECReconstructionInfo> pendingECList = nodeinfo
- .getErasureCodeCommand(maxTransfers);
- if (pendingECList != null) {
- cmds.add(new BlockECReconstructionCommand(
- DNA_ERASURE_CODING_RECONSTRUCTION, pendingECList));
+ // Allocate _approximately_ maxTransfers pending tasks to DataNode.
+ // NN chooses pending tasks based on the ratio between the lengths of
+ // replication and erasure-coded block queues.
+ int totalReplicateBlocks = nodeinfo.getNumberOfReplicateBlocks();
+ int totalECBlocks = nodeinfo.getNumberOfBlocksToBeErasureCoded();
+ int totalBlocks = totalReplicateBlocks + totalECBlocks;
+ if (totalBlocks > 0) {
+ int numReplicationTasks = (int) Math.ceil(
+ (double) (totalReplicateBlocks * maxTransfers) / totalBlocks);
+ int numECTasks = (int) Math.ceil(
+ (double) (totalECBlocks * maxTransfers) / totalBlocks);
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Pending replication tasks: " + numReplicationTasks
+ + " erasure-coded tasks: " + numECTasks);
+ }
+ // check pending replication tasks
+ List<BlockTargetPair> pendingList = nodeinfo.getReplicationCommand(
+ numReplicationTasks);
+ if (pendingList != null && !pendingList.isEmpty()) {
+ cmds.add(new BlockCommand(DatanodeProtocol.DNA_TRANSFER, blockPoolId,
+ pendingList));
+ }
+ // check pending erasure coding tasks
+ List<BlockECReconstructionInfo> pendingECList = nodeinfo
+ .getErasureCodeCommand(numECTasks);
+ if (pendingECList != null && !pendingECList.isEmpty()) {
+ cmds.add(new BlockECReconstructionCommand(
+ DNA_ERASURE_CODING_RECONSTRUCTION, pendingECList));
+ }
}
+
// check block invalidation
Block[] blks = nodeinfo.getInvalidateBlocks(blockInvalidateLimit);
if (blks != null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/b2989488/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeManager.java
index de002f4..286f4a4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/blockmanagement/TestDatanodeManager.java
@@ -25,6 +25,7 @@ import java.net.URL;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.Iterator;
@@ -500,46 +501,93 @@ public class TestDatanodeManager {
"127.0.0.1:23456", bothAgain.get(1).getInfoAddr());
}
- @Test
- public void testPendingRecoveryTasks() throws IOException {
+ /**
+ * Verify the correctness of pending recovery process.
+ *
+ * @param numReplicationBlocks the number of replication blocks in the queue.
+ * @param numECBlocks number of EC blocks in the queue.
+ * @param maxTransfers the maxTransfer value.
+ * @param numReplicationTasks the number of replication tasks polled from
+ * the queue.
+ * @param numECTasks the number of EC tasks polled from the queue.
+ *
+ * @throws IOException
+ */
+ private void verifyPendingRecoveryTasks(
+ int numReplicationBlocks, int numECBlocks,
+ int maxTransfers, int numReplicationTasks, int numECTasks)
+ throws IOException {
FSNamesystem fsn = Mockito.mock(FSNamesystem.class);
Mockito.when(fsn.hasWriteLock()).thenReturn(true);
Configuration conf = new Configuration();
DatanodeManager dm = Mockito.spy(mockDatanodeManager(fsn, conf));
- int maxTransfers = 20;
- int numPendingTasks = 7;
- int numECTasks = maxTransfers - numPendingTasks;
-
DatanodeDescriptor nodeInfo = Mockito.mock(DatanodeDescriptor.class);
Mockito.when(nodeInfo.isRegistered()).thenReturn(true);
Mockito.when(nodeInfo.getStorageInfos())
.thenReturn(new DatanodeStorageInfo[0]);
- List<BlockTargetPair> pendingList =
- Collections.nCopies(numPendingTasks, new BlockTargetPair(null, null));
- Mockito.when(nodeInfo.getReplicationCommand(maxTransfers))
- .thenReturn(pendingList);
- List<BlockECReconstructionInfo> ecPendingList =
- Collections.nCopies(numECTasks, null);
+ if (numReplicationBlocks > 0) {
+ Mockito.when(nodeInfo.getNumberOfReplicateBlocks())
+ .thenReturn(numReplicationBlocks);
+
+ List<BlockTargetPair> tasks =
+ Collections.nCopies(
+ Math.min(numReplicationTasks, numReplicationBlocks),
+ new BlockTargetPair(null, null));
+ Mockito.when(nodeInfo.getReplicationCommand(numReplicationTasks))
+ .thenReturn(tasks);
+ }
+
+ if (numECBlocks > 0) {
+ Mockito.when(nodeInfo.getNumberOfBlocksToBeErasureCoded())
+ .thenReturn(numECBlocks);
+
+ List<BlockECReconstructionInfo> tasks =
+ Collections.nCopies(numECTasks, null);
+ Mockito.when(nodeInfo.getErasureCodeCommand(numECTasks))
+ .thenReturn(tasks);
+ }
- Mockito.when(nodeInfo.getErasureCodeCommand(numECTasks))
- .thenReturn(ecPendingList);
DatanodeRegistration dnReg = Mockito.mock(DatanodeRegistration.class);
Mockito.when(dm.getDatanode(dnReg)).thenReturn(nodeInfo);
-
DatanodeCommand[] cmds = dm.handleHeartbeat(
dnReg, new StorageReport[1], "bp-123", 0, 0, 10, maxTransfers, 0, null,
SlowPeerReports.EMPTY_REPORT, SlowDiskReports.EMPTY_REPORT);
- assertEquals(2, cmds.length);
- assertTrue(cmds[0] instanceof BlockCommand);
- BlockCommand replicaCmd = (BlockCommand) cmds[0];
- assertEquals(numPendingTasks, replicaCmd.getBlocks().length);
- assertEquals(numPendingTasks, replicaCmd.getTargets().length);
- assertTrue(cmds[1] instanceof BlockECReconstructionCommand);
- BlockECReconstructionCommand ecRecoveryCmd =
- (BlockECReconstructionCommand) cmds[1];
- assertEquals(numECTasks, ecRecoveryCmd.getECTasks().size());
+ long expectedNumCmds = Arrays.stream(
+ new int[]{numReplicationTasks, numECTasks})
+ .filter(x -> x > 0)
+ .count();
+ assertEquals(expectedNumCmds, cmds.length);
+
+ int idx = 0;
+ if (numReplicationTasks > 0) {
+ assertTrue(cmds[idx] instanceof BlockCommand);
+ BlockCommand cmd = (BlockCommand) cmds[0];
+ assertEquals(numReplicationTasks, cmd.getBlocks().length);
+ assertEquals(numReplicationTasks, cmd.getTargets().length);
+ idx++;
+ }
+
+ if (numECTasks > 0) {
+ assertTrue(cmds[idx] instanceof BlockECReconstructionCommand);
+ BlockECReconstructionCommand cmd =
+ (BlockECReconstructionCommand) cmds[idx];
+ assertEquals(numECTasks, cmd.getECTasks().size());
+ }
+
+ Mockito.verify(nodeInfo).getReplicationCommand(numReplicationTasks);
+ Mockito.verify(nodeInfo).getErasureCodeCommand(numECTasks);
+ }
+
+ @Test
+ public void testPendingRecoveryTasks() throws IOException {
+ // Tasks are slitted according to the ratio between queue lengths.
+ verifyPendingRecoveryTasks(20, 20, 20, 10, 10);
+ verifyPendingRecoveryTasks(40, 10, 20, 16, 4);
+
+ // Approximately load tasks if the ratio between queue length is large.
+ verifyPendingRecoveryTasks(400, 1, 20, 20, 1);
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[07/50] [abbrv] hadoop git commit: YARN-6900. ZooKeeper based implementation of the FederationStateStore. (Íñigo Goiri via Subru).
Posted by as...@apache.org.
YARN-6900. ZooKeeper based implementation of the FederationStateStore. (Íñigo Goiri via Subru).
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/de462da0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/de462da0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/de462da0
Branch: refs/heads/YARN-5972
Commit: de462da04e167a04b89ecf0f40d464cf39dc6549
Parents: 1455306
Author: Subru Krishnan <su...@apache.org>
Authored: Wed Aug 16 11:43:24 2017 -0700
Committer: Subru Krishnan <su...@apache.org>
Committed: Wed Aug 16 11:43:24 2017 -0700
----------------------------------------------------------------------
.../hadoop/yarn/conf/YarnConfiguration.java | 8 +
.../yarn/conf/TestYarnConfigurationFields.java | 4 +
.../hadoop-yarn-server-common/pom.xml | 5 +
.../impl/ZookeeperFederationStateStore.java | 634 +++++++++++++++++++
.../impl/TestZookeeperFederationStateStore.java | 89 +++
.../TestFederationStateStoreFacadeRetry.java | 20 +-
.../src/site/markdown/Federation.md | 56 +-
7 files changed, 785 insertions(+), 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de462da0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
index 8acaef8..8515e0a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
@@ -2629,6 +2629,14 @@ public class YarnConfiguration extends Configuration {
public static final String DEFAULT_FEDERATION_POLICY_MANAGER_PARAMS = "";
+ public static final String FEDERATION_STATESTORE_ZK_PREFIX =
+ FEDERATION_PREFIX + "zk-state-store.";
+ /** Parent znode path under which ZKRMStateStore will create znodes. */
+ public static final String FEDERATION_STATESTORE_ZK_PARENT_PATH =
+ FEDERATION_STATESTORE_ZK_PREFIX + "parent-path";
+ public static final String DEFAULT_FEDERATION_STATESTORE_ZK_PARENT_PATH =
+ "/federationstore";
+
private static final String FEDERATION_STATESTORE_SQL_PREFIX =
FEDERATION_PREFIX + "state-store.sql.";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de462da0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
index 91a8b0a..c40c2c5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-api/src/test/java/org/apache/hadoop/yarn/conf/TestYarnConfigurationFields.java
@@ -96,6 +96,10 @@ public class TestYarnConfigurationFields extends TestConfigurationFieldsBase {
configurationPropsToSkipCompare
.add(YarnConfiguration.DEFAULT_FEDERATION_POLICY_MANAGER_PARAMS);
+ // Federation StateStore ZK implementation configs to be ignored
+ configurationPropsToSkipCompare.add(
+ YarnConfiguration.FEDERATION_STATESTORE_ZK_PARENT_PATH);
+
// Federation StateStore SQL implementation configs to be ignored
configurationPropsToSkipCompare
.add(YarnConfiguration.FEDERATION_STATESTORE_SQL_JDBC_CLASS);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de462da0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml
index 441a574..e8d3880 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/pom.xml
@@ -130,6 +130,11 @@
</exclusion>
</exclusions>
</dependency>
+ <dependency>
+ <groupId>org.apache.curator</groupId>
+ <artifactId>curator-test</artifactId>
+ <scope>test</scope>
+ </dependency>
</dependencies>
<build>
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de462da0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/ZookeeperFederationStateStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/ZookeeperFederationStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/ZookeeperFederationStateStore.java
new file mode 100644
index 0000000..6ae7d3c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/ZookeeperFederationStateStore.java
@@ -0,0 +1,634 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.impl;
+
+import static org.apache.hadoop.util.curator.ZKCuratorManager.getNodePath;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Calendar;
+import java.util.List;
+import java.util.TimeZone;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.util.curator.ZKCuratorManager;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SubClusterIdProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SubClusterInfoProto;
+import org.apache.hadoop.yarn.federation.proto.YarnServerFederationProtos.SubClusterPolicyConfigurationProto;
+import org.apache.hadoop.yarn.server.federation.store.FederationStateStore;
+import org.apache.hadoop.yarn.server.federation.store.records.AddApplicationHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.AddApplicationHomeSubClusterResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.ApplicationHomeSubCluster;
+import org.apache.hadoop.yarn.server.federation.store.records.DeleteApplicationHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.DeleteApplicationHomeSubClusterResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationHomeSubClusterResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationsHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.GetApplicationsHomeSubClusterResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterInfoRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterInfoResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterPoliciesConfigurationsRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterPoliciesConfigurationsResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterPolicyConfigurationRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClusterPolicyConfigurationResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClustersInfoRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.GetSubClustersInfoResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.SetSubClusterPolicyConfigurationRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.SetSubClusterPolicyConfigurationResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterDeregisterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterDeregisterResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterHeartbeatRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterHeartbeatResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterPolicyConfiguration;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterRegisterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterRegisterResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.SubClusterState;
+import org.apache.hadoop.yarn.server.federation.store.records.UpdateApplicationHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.UpdateApplicationHomeSubClusterResponse;
+import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.SubClusterIdPBImpl;
+import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.SubClusterInfoPBImpl;
+import org.apache.hadoop.yarn.server.federation.store.records.impl.pb.SubClusterPolicyConfigurationPBImpl;
+import org.apache.hadoop.yarn.server.federation.store.utils.FederationApplicationHomeSubClusterStoreInputValidator;
+import org.apache.hadoop.yarn.server.federation.store.utils.FederationMembershipStateStoreInputValidator;
+import org.apache.hadoop.yarn.server.federation.store.utils.FederationPolicyStoreInputValidator;
+import org.apache.hadoop.yarn.server.federation.store.utils.FederationStateStoreUtils;
+import org.apache.hadoop.yarn.server.records.Version;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import com.google.protobuf.InvalidProtocolBufferException;
+
+/**
+ * ZooKeeper implementation of {@link FederationStateStore}.
+ *
+ * The znode structure is as follows:
+ * ROOT_DIR_PATH
+ * |--- MEMBERSHIP
+ * | |----- SC1
+ * | |----- SC2
+ * |--- APPLICATION
+ * | |----- APP1
+ * | |----- APP2
+ * |--- POLICY
+ * |----- QUEUE1
+ * |----- QUEUE1
+ */
+public class ZookeeperFederationStateStore implements FederationStateStore {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ZookeeperFederationStateStore.class);
+
+ private final static String ROOT_ZNODE_NAME_MEMBERSHIP = "memberships";
+ private final static String ROOT_ZNODE_NAME_APPLICATION = "applications";
+ private final static String ROOT_ZNODE_NAME_POLICY = "policies";
+
+ /** Interface to Zookeeper. */
+ private ZKCuratorManager zkManager;
+
+ /** Directory to store the state store data. */
+ private String baseZNode;
+
+ private String appsZNode;
+ private String membershipZNode;
+ private String policiesZNode;
+
+ @Override
+ public void init(Configuration conf) throws YarnException {
+ LOG.info("Initializing ZooKeeper connection");
+
+ baseZNode = conf.get(
+ YarnConfiguration.FEDERATION_STATESTORE_ZK_PARENT_PATH,
+ YarnConfiguration.DEFAULT_FEDERATION_STATESTORE_ZK_PARENT_PATH);
+ try {
+ this.zkManager = new ZKCuratorManager(conf);
+ this.zkManager.start();
+ } catch (IOException e) {
+ LOG.error("Cannot initialize the ZK connection", e);
+ }
+
+ // Base znodes
+ membershipZNode = getNodePath(baseZNode, ROOT_ZNODE_NAME_MEMBERSHIP);
+ appsZNode = getNodePath(baseZNode, ROOT_ZNODE_NAME_APPLICATION);
+ policiesZNode = getNodePath(baseZNode, ROOT_ZNODE_NAME_POLICY);
+
+ // Create base znode for each entity
+ try {
+ zkManager.createRootDirRecursively(membershipZNode);
+ zkManager.createRootDirRecursively(appsZNode);
+ zkManager.createRootDirRecursively(policiesZNode);
+ } catch (Exception e) {
+ String errMsg = "Cannot create base directories: " + e.getMessage();
+ FederationStateStoreUtils.logAndThrowStoreException(LOG, errMsg);
+ }
+
+ }
+
+ @Override
+ public void close() throws Exception {
+ if (zkManager != null) {
+ zkManager.close();
+ }
+ }
+
+ @Override
+ public AddApplicationHomeSubClusterResponse addApplicationHomeSubCluster(
+ AddApplicationHomeSubClusterRequest request) throws YarnException {
+
+ FederationApplicationHomeSubClusterStoreInputValidator.validate(request);
+ ApplicationHomeSubCluster app = request.getApplicationHomeSubCluster();
+ ApplicationId appId = app.getApplicationId();
+
+ // Try to write the subcluster
+ SubClusterId homeSubCluster = app.getHomeSubCluster();
+ try {
+ putApp(appId, homeSubCluster, false);
+ } catch (Exception e) {
+ String errMsg = "Cannot add application home subcluster for " + appId;
+ FederationStateStoreUtils.logAndThrowStoreException(LOG, errMsg);
+ }
+
+ // Check for the actual subcluster
+ try {
+ homeSubCluster = getApp(appId);
+ } catch (Exception e) {
+ String errMsg = "Cannot check app home subcluster for " + appId;
+ FederationStateStoreUtils.logAndThrowStoreException(LOG, errMsg);
+ }
+
+ return AddApplicationHomeSubClusterResponse
+ .newInstance(homeSubCluster);
+ }
+
+ @Override
+ public UpdateApplicationHomeSubClusterResponse
+ updateApplicationHomeSubCluster(
+ UpdateApplicationHomeSubClusterRequest request)
+ throws YarnException {
+
+ FederationApplicationHomeSubClusterStoreInputValidator.validate(request);
+ ApplicationHomeSubCluster app = request.getApplicationHomeSubCluster();
+ ApplicationId appId = app.getApplicationId();
+ SubClusterId homeSubCluster = getApp(appId);
+ if (homeSubCluster == null) {
+ String errMsg = "Application " + appId + " does not exist";
+ FederationStateStoreUtils.logAndThrowStoreException(LOG, errMsg);
+ }
+ SubClusterId newSubClusterId =
+ request.getApplicationHomeSubCluster().getHomeSubCluster();
+ putApp(appId, newSubClusterId, true);
+ return UpdateApplicationHomeSubClusterResponse.newInstance();
+ }
+
+ @Override
+ public GetApplicationHomeSubClusterResponse getApplicationHomeSubCluster(
+ GetApplicationHomeSubClusterRequest request) throws YarnException {
+
+ FederationApplicationHomeSubClusterStoreInputValidator.validate(request);
+ ApplicationId appId = request.getApplicationId();
+ SubClusterId homeSubCluster = getApp(appId);
+ if (homeSubCluster == null) {
+ String errMsg = "Application " + appId + " does not exist";
+ FederationStateStoreUtils.logAndThrowStoreException(LOG, errMsg);
+ }
+ return GetApplicationHomeSubClusterResponse.newInstance(
+ ApplicationHomeSubCluster.newInstance(appId, homeSubCluster));
+ }
+
+ @Override
+ public GetApplicationsHomeSubClusterResponse getApplicationsHomeSubCluster(
+ GetApplicationsHomeSubClusterRequest request) throws YarnException {
+ List<ApplicationHomeSubCluster> result = new ArrayList<>();
+
+ try {
+ for (String child : zkManager.getChildren(appsZNode)) {
+ ApplicationId appId = ApplicationId.fromString(child);
+ SubClusterId homeSubCluster = getApp(appId);
+ ApplicationHomeSubCluster app =
+ ApplicationHomeSubCluster.newInstance(appId, homeSubCluster);
+ result.add(app);
+ }
+ } catch (Exception e) {
+ String errMsg = "Cannot get apps: " + e.getMessage();
+ FederationStateStoreUtils.logAndThrowStoreException(LOG, errMsg);
+ }
+
+ return GetApplicationsHomeSubClusterResponse.newInstance(result);
+ }
+
+ @Override
+ public DeleteApplicationHomeSubClusterResponse
+ deleteApplicationHomeSubCluster(
+ DeleteApplicationHomeSubClusterRequest request)
+ throws YarnException {
+
+ FederationApplicationHomeSubClusterStoreInputValidator.validate(request);
+ ApplicationId appId = request.getApplicationId();
+ String appZNode = getNodePath(appsZNode, appId.toString());
+
+ boolean exists = false;
+ try {
+ exists = zkManager.exists(appZNode);
+ } catch (Exception e) {
+ String errMsg = "Cannot check app: " + e.getMessage();
+ FederationStateStoreUtils.logAndThrowStoreException(LOG, errMsg);
+ }
+ if (!exists) {
+ String errMsg = "Application " + appId + " does not exist";
+ FederationStateStoreUtils.logAndThrowStoreException(LOG, errMsg);
+ }
+
+ try {
+ zkManager.delete(appZNode);
+ } catch (Exception e) {
+ String errMsg = "Cannot delete app: " + e.getMessage();
+ FederationStateStoreUtils.logAndThrowStoreException(LOG, errMsg);
+ }
+
+ return DeleteApplicationHomeSubClusterResponse.newInstance();
+ }
+
+ @Override
+ public SubClusterRegisterResponse registerSubCluster(
+ SubClusterRegisterRequest request) throws YarnException {
+ FederationMembershipStateStoreInputValidator.validate(request);
+ SubClusterInfo subClusterInfo = request.getSubClusterInfo();
+ SubClusterId subclusterId = subClusterInfo.getSubClusterId();
+
+ // Update the heartbeat time
+ long currentTime = getCurrentTime();
+ subClusterInfo.setLastHeartBeat(currentTime);
+
+ try {
+ putSubclusterInfo(subclusterId, subClusterInfo, true);
+ } catch (Exception e) {
+ String errMsg = "Cannot register subcluster: " + e.getMessage();
+ FederationStateStoreUtils.logAndThrowStoreException(LOG, errMsg);
+ }
+ return SubClusterRegisterResponse.newInstance();
+ }
+
+ @Override
+ public SubClusterDeregisterResponse deregisterSubCluster(
+ SubClusterDeregisterRequest request) throws YarnException {
+ FederationMembershipStateStoreInputValidator.validate(request);
+ SubClusterId subClusterId = request.getSubClusterId();
+ SubClusterState state = request.getState();
+
+ // Get the current information and update it
+ SubClusterInfo subClusterInfo = getSubclusterInfo(subClusterId);
+ if (subClusterInfo == null) {
+ String errMsg = "SubCluster " + subClusterId + " not found";
+ FederationStateStoreUtils.logAndThrowStoreException(LOG, errMsg);
+ } else {
+ subClusterInfo.setState(state);
+ putSubclusterInfo(subClusterId, subClusterInfo, true);
+ }
+
+ return SubClusterDeregisterResponse.newInstance();
+ }
+
+ @Override
+ public SubClusterHeartbeatResponse subClusterHeartbeat(
+ SubClusterHeartbeatRequest request) throws YarnException {
+
+ FederationMembershipStateStoreInputValidator.validate(request);
+ SubClusterId subClusterId = request.getSubClusterId();
+
+ SubClusterInfo subClusterInfo = getSubclusterInfo(subClusterId);
+ if (subClusterInfo == null) {
+ String errMsg = "SubCluster " + subClusterId
+ + " does not exist; cannot heartbeat";
+ FederationStateStoreUtils.logAndThrowStoreException(LOG, errMsg);
+ }
+
+ long currentTime = getCurrentTime();
+ subClusterInfo.setLastHeartBeat(currentTime);
+ subClusterInfo.setState(request.getState());
+ subClusterInfo.setCapability(request.getCapability());
+
+ putSubclusterInfo(subClusterId, subClusterInfo, true);
+
+ return SubClusterHeartbeatResponse.newInstance();
+ }
+
+ @Override
+ public GetSubClusterInfoResponse getSubCluster(
+ GetSubClusterInfoRequest request) throws YarnException {
+
+ FederationMembershipStateStoreInputValidator.validate(request);
+ SubClusterId subClusterId = request.getSubClusterId();
+ SubClusterInfo subClusterInfo = null;
+ try {
+ subClusterInfo = getSubclusterInfo(subClusterId);
+ if (subClusterInfo == null) {
+ LOG.warn("The queried SubCluster: {} does not exist.", subClusterId);
+ return null;
+ }
+ } catch (Exception e) {
+ String errMsg = "Cannot get subcluster: " + e.getMessage();
+ FederationStateStoreUtils.logAndThrowStoreException(LOG, errMsg);
+ }
+ return GetSubClusterInfoResponse.newInstance(subClusterInfo);
+ }
+
+ @Override
+ public GetSubClustersInfoResponse getSubClusters(
+ GetSubClustersInfoRequest request) throws YarnException {
+ List<SubClusterInfo> result = new ArrayList<>();
+
+ try {
+ for (String child : zkManager.getChildren(membershipZNode)) {
+ SubClusterId subClusterId = SubClusterId.newInstance(child);
+ SubClusterInfo info = getSubclusterInfo(subClusterId);
+ if (!request.getFilterInactiveSubClusters() ||
+ info.getState().isActive()) {
+ result.add(info);
+ }
+ }
+ } catch (Exception e) {
+ String errMsg = "Cannot get subclusters: " + e.getMessage();
+ FederationStateStoreUtils.logAndThrowStoreException(LOG, errMsg);
+ }
+ return GetSubClustersInfoResponse.newInstance(result);
+ }
+
+
+ @Override
+ public GetSubClusterPolicyConfigurationResponse getPolicyConfiguration(
+ GetSubClusterPolicyConfigurationRequest request) throws YarnException {
+
+ FederationPolicyStoreInputValidator.validate(request);
+ String queue = request.getQueue();
+ SubClusterPolicyConfiguration policy = null;
+ try {
+ policy = getPolicy(queue);
+ } catch (Exception e) {
+ String errMsg = "Cannot get policy: " + e.getMessage();
+ FederationStateStoreUtils.logAndThrowStoreException(LOG, errMsg);
+ }
+
+ if (policy == null) {
+ LOG.warn("Policy for queue: {} does not exist.", queue);
+ return null;
+ }
+ return GetSubClusterPolicyConfigurationResponse
+ .newInstance(policy);
+ }
+
+ @Override
+ public SetSubClusterPolicyConfigurationResponse setPolicyConfiguration(
+ SetSubClusterPolicyConfigurationRequest request) throws YarnException {
+
+ FederationPolicyStoreInputValidator.validate(request);
+ SubClusterPolicyConfiguration policy =
+ request.getPolicyConfiguration();
+ try {
+ String queue = policy.getQueue();
+ putPolicy(queue, policy, true);
+ } catch (Exception e) {
+ String errMsg = "Cannot set policy: " + e.getMessage();
+ FederationStateStoreUtils.logAndThrowStoreException(LOG, errMsg);
+ }
+ return SetSubClusterPolicyConfigurationResponse.newInstance();
+ }
+
+ @Override
+ public GetSubClusterPoliciesConfigurationsResponse getPoliciesConfigurations(
+ GetSubClusterPoliciesConfigurationsRequest request) throws YarnException {
+ List<SubClusterPolicyConfiguration> result = new ArrayList<>();
+
+ try {
+ for (String child : zkManager.getChildren(policiesZNode)) {
+ SubClusterPolicyConfiguration policy = getPolicy(child);
+ result.add(policy);
+ }
+ } catch (Exception e) {
+ String errMsg = "Cannot get policies: " + e.getMessage();
+ FederationStateStoreUtils.logAndThrowStoreException(LOG, errMsg);
+ }
+ return GetSubClusterPoliciesConfigurationsResponse.newInstance(result);
+ }
+
+ @Override
+ public Version getCurrentVersion() {
+ return null;
+ }
+
+ @Override
+ public Version loadVersion() {
+ return null;
+ }
+
+ /**
+ * Get the subcluster for an application.
+ * @param appId Application identifier.
+ * @return Subcluster identifier.
+ * @throws Exception If it cannot contact ZooKeeper.
+ */
+ private SubClusterId getApp(final ApplicationId appId) throws YarnException {
+ String appZNode = getNodePath(appsZNode, appId.toString());
+
+ SubClusterId subClusterId = null;
+ byte[] data = get(appZNode);
+ if (data != null) {
+ try {
+ subClusterId = new SubClusterIdPBImpl(
+ SubClusterIdProto.parseFrom(data));
+ } catch (InvalidProtocolBufferException e) {
+ String errMsg = "Cannot parse application at " + appZNode;
+ FederationStateStoreUtils.logAndThrowStoreException(LOG, errMsg);
+ }
+ }
+ return subClusterId;
+ }
+
+ /**
+ * Put an application.
+ * @param appId Application identifier.
+ * @param subClusterId Subcluster identifier.
+ * @throws Exception If it cannot contact ZooKeeper.
+ */
+ private void putApp(final ApplicationId appId,
+ final SubClusterId subClusterId, boolean update)
+ throws YarnException {
+ String appZNode = getNodePath(appsZNode, appId.toString());
+ SubClusterIdProto proto =
+ ((SubClusterIdPBImpl)subClusterId).getProto();
+ byte[] data = proto.toByteArray();
+ put(appZNode, data, update);
+ }
+
+ /**
+ * Get the current information for a subcluster from Zookeeper.
+ * @param subclusterId Subcluster identifier.
+ * @return Subcluster information or null if it doesn't exist.
+ * @throws Exception If it cannot contact ZooKeeper.
+ */
+ private SubClusterInfo getSubclusterInfo(final SubClusterId subclusterId)
+ throws YarnException {
+ String memberZNode = getNodePath(membershipZNode, subclusterId.toString());
+
+ SubClusterInfo policy = null;
+ byte[] data = get(memberZNode);
+ if (data != null) {
+ try {
+ policy = new SubClusterInfoPBImpl(
+ SubClusterInfoProto.parseFrom(data));
+ } catch (InvalidProtocolBufferException e) {
+ String errMsg = "Cannot parse subcluster info at " + memberZNode;
+ FederationStateStoreUtils.logAndThrowStoreException(LOG, errMsg);
+ }
+ }
+ return policy;
+ }
+
+ /**
+ * Put the subcluster information in Zookeeper.
+ * @param subclusterId Subcluster identifier.
+ * @param subClusterInfo Subcluster information.
+ * @throws Exception If it cannot contact ZooKeeper.
+ */
+ private void putSubclusterInfo(final SubClusterId subclusterId,
+ final SubClusterInfo subClusterInfo, final boolean update)
+ throws YarnException {
+ String memberZNode = getNodePath(membershipZNode, subclusterId.toString());
+ SubClusterInfoProto proto =
+ ((SubClusterInfoPBImpl)subClusterInfo).getProto();
+ byte[] data = proto.toByteArray();
+ put(memberZNode, data, update);
+ }
+
+ /**
+ * Get the queue policy from Zookeeper.
+ * @param queue Name of the queue.
+ * @return Subcluster policy configuration.
+ * @throws YarnException If it cannot contact ZooKeeper.
+ */
+ private SubClusterPolicyConfiguration getPolicy(final String queue)
+ throws YarnException {
+ String policyZNode = getNodePath(policiesZNode, queue);
+
+ SubClusterPolicyConfiguration policy = null;
+ byte[] data = get(policyZNode);
+ if (data != null) {
+ try {
+ policy = new SubClusterPolicyConfigurationPBImpl(
+ SubClusterPolicyConfigurationProto.parseFrom(data));
+ } catch (InvalidProtocolBufferException e) {
+ String errMsg = "Cannot parse policy at " + policyZNode;
+ FederationStateStoreUtils.logAndThrowStoreException(LOG, errMsg);
+ }
+ }
+ return policy;
+ }
+
+ /**
+ * Put the subcluster information in Zookeeper.
+ * @param queue Name of the queue.
+ * @param policy Subcluster policy configuration.
+ * @throws YarnException If it cannot contact ZooKeeper.
+ */
+ private void putPolicy(final String queue,
+ final SubClusterPolicyConfiguration policy, boolean update)
+ throws YarnException {
+ String policyZNode = getNodePath(policiesZNode, queue);
+
+ SubClusterPolicyConfigurationProto proto =
+ ((SubClusterPolicyConfigurationPBImpl)policy).getProto();
+ byte[] data = proto.toByteArray();
+ put(policyZNode, data, update);
+ }
+
+ /**
+ * Get data from a znode in Zookeeper.
+ * @param znode Path of the znode.
+ * @return Data in the znode.
+ * @throws YarnException If it cannot contact ZooKeeper.
+ */
+ private byte[] get(String znode) throws YarnException {
+ boolean exists = false;
+ try {
+ exists = zkManager.exists(znode);
+ } catch (Exception e) {
+ String errMsg = "Cannot find znode " + znode;
+ FederationStateStoreUtils.logAndThrowStoreException(LOG, errMsg);
+ }
+ if (!exists) {
+ LOG.error("{} does not exist", znode);
+ return null;
+ }
+
+ byte[] data = null;
+ try {
+ data = zkManager.getData(znode);
+ } catch (Exception e) {
+ String errMsg = "Cannot get data from znode " + znode
+ + ": " + e.getMessage();
+ FederationStateStoreUtils.logAndThrowStoreException(LOG, errMsg);
+ }
+ return data;
+ }
+
+ /**
+ * Put data into a znode in Zookeeper.
+ * @param znode Path of the znode.
+ * @param data Data to write.
+ * @throws YarnException If it cannot contact ZooKeeper.
+ */
+ private void put(String znode, byte[] data, boolean update)
+ throws YarnException {
+ // Create the znode
+ boolean created = false;
+ try {
+ created = zkManager.create(znode);
+ } catch (Exception e) {
+ String errMsg = "Cannot create znode " + znode + ": " + e.getMessage();
+ FederationStateStoreUtils.logAndThrowStoreException(LOG, errMsg);
+ }
+ if (!created) {
+ LOG.debug("{} not created", znode);
+ if (!update) {
+ LOG.info("{} already existed and we are not updating", znode);
+ return;
+ }
+ }
+
+ // Write the data into the znode
+ try {
+ zkManager.setData(znode, data, -1);
+ } catch (Exception e) {
+ String errMsg = "Cannot write data into znode " + znode
+ + ": " + e.getMessage();
+ FederationStateStoreUtils.logAndThrowStoreException(LOG, errMsg);
+ }
+ }
+
+ /**
+ * Get the current time.
+ * @return Current time in milliseconds.
+ */
+ private static long getCurrentTime() {
+ Calendar cal = Calendar.getInstance(TimeZone.getTimeZone("UTC"));
+ return cal.getTimeInMillis();
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de462da0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/TestZookeeperFederationStateStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/TestZookeeperFederationStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/TestZookeeperFederationStateStore.java
new file mode 100644
index 0000000..390b803
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/TestZookeeperFederationStateStore.java
@@ -0,0 +1,89 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+package org.apache.hadoop.yarn.server.federation.store.impl;
+
+import java.io.IOException;
+
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.CuratorFrameworkFactory;
+import org.apache.curator.retry.RetryNTimes;
+import org.apache.curator.test.TestingServer;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.server.federation.store.FederationStateStore;
+import org.junit.After;
+import org.junit.Before;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Unit tests for ZookeeperFederationStateStore.
+ */
+public class TestZookeeperFederationStateStore
+ extends FederationStateStoreBaseTest {
+
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TestZookeeperFederationStateStore.class);
+
+ /** Zookeeper test server. */
+ private static TestingServer curatorTestingServer;
+ private static CuratorFramework curatorFramework;
+
+ @Before
+ public void before() throws IOException, YarnException {
+ try {
+ curatorTestingServer = new TestingServer();
+ curatorTestingServer.start();
+ String connectString = curatorTestingServer.getConnectString();
+ curatorFramework = CuratorFrameworkFactory.builder()
+ .connectString(connectString)
+ .retryPolicy(new RetryNTimes(100, 100))
+ .build();
+ curatorFramework.start();
+
+ Configuration conf = new YarnConfiguration();
+ conf.set(CommonConfigurationKeys.ZK_ADDRESS, connectString);
+ setConf(conf);
+ } catch (Exception e) {
+ LOG.error("Cannot initialize ZooKeeper store", e);
+ throw new IOException(e);
+ }
+
+ super.before();
+ }
+
+ @After
+ public void after() throws Exception {
+ super.after();
+
+ curatorFramework.close();
+ try {
+ curatorTestingServer.stop();
+ } catch (IOException e) {
+ }
+ }
+
+ @Override
+ protected FederationStateStore createStateStore() {
+ Configuration conf = new Configuration();
+ super.setConf(conf);
+ return new ZookeeperFederationStateStore();
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de462da0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/TestFederationStateStoreFacadeRetry.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/TestFederationStateStoreFacadeRetry.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/TestFederationStateStoreFacadeRetry.java
index ea43268..868e771 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/TestFederationStateStoreFacadeRetry.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/utils/TestFederationStateStoreFacadeRetry.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.yarn.server.federation.store.exception.FederationStateS
import org.apache.hadoop.yarn.server.federation.store.exception.FederationStateStoreInvalidInputException;
import org.apache.hadoop.yarn.server.federation.store.exception.FederationStateStoreRetriableException;
import org.junit.Assert;
+import org.junit.Before;
import org.junit.Test;
import com.zaxxer.hikari.pool.HikariPool.PoolInitializationException;
@@ -40,14 +41,18 @@ public class TestFederationStateStoreFacadeRetry {
private int maxRetries = 4;
private Configuration conf;
+ @Before
+ public void setup() {
+ conf = new Configuration();
+ conf.setInt(YarnConfiguration.CLIENT_FAILOVER_RETRIES, maxRetries);
+ }
+
/*
* Test to validate that FederationStateStoreRetriableException is a retriable
* exception.
*/
@Test
public void testFacadeRetriableException() throws Exception {
- conf = new Configuration();
- conf.setInt(YarnConfiguration.CLIENT_FAILOVER_RETRIES, maxRetries);
RetryPolicy policy = FederationStateStoreFacade.createRetryPolicy(conf);
RetryAction action = policy.shouldRetry(
new FederationStateStoreRetriableException(""), 0, 0, false);
@@ -66,9 +71,6 @@ public class TestFederationStateStoreFacadeRetry {
*/
@Test
public void testFacadeYarnException() throws Exception {
-
- conf = new Configuration();
- conf.setInt(YarnConfiguration.CLIENT_FAILOVER_RETRIES, maxRetries);
RetryPolicy policy = FederationStateStoreFacade.createRetryPolicy(conf);
RetryAction action = policy.shouldRetry(new YarnException(), 0, 0, false);
Assert.assertEquals(RetryAction.FAIL.action, action.action);
@@ -80,8 +82,6 @@ public class TestFederationStateStoreFacadeRetry {
*/
@Test
public void testFacadeStateStoreException() throws Exception {
- conf = new Configuration();
- conf.setInt(YarnConfiguration.CLIENT_FAILOVER_RETRIES, maxRetries);
RetryPolicy policy = FederationStateStoreFacade.createRetryPolicy(conf);
RetryAction action = policy
.shouldRetry(new FederationStateStoreException("Error"), 0, 0, false);
@@ -94,8 +94,6 @@ public class TestFederationStateStoreFacadeRetry {
*/
@Test
public void testFacadeInvalidInputException() throws Exception {
- conf = new Configuration();
- conf.setInt(YarnConfiguration.CLIENT_FAILOVER_RETRIES, maxRetries);
RetryPolicy policy = FederationStateStoreFacade.createRetryPolicy(conf);
RetryAction action = policy.shouldRetry(
new FederationStateStoreInvalidInputException(""), 0, 0, false);
@@ -107,8 +105,6 @@ public class TestFederationStateStoreFacadeRetry {
*/
@Test
public void testFacadeCacheRetriableException() throws Exception {
- conf = new Configuration();
- conf.setInt(YarnConfiguration.CLIENT_FAILOVER_RETRIES, maxRetries);
RetryPolicy policy = FederationStateStoreFacade.createRetryPolicy(conf);
RetryAction action =
policy.shouldRetry(new CacheLoaderException(""), 0, 0, false);
@@ -128,8 +124,6 @@ public class TestFederationStateStoreFacadeRetry {
@Test
public void testFacadePoolInitRetriableException() throws Exception {
// PoolInitializationException is a retriable exception
- conf = new Configuration();
- conf.setInt(YarnConfiguration.CLIENT_FAILOVER_RETRIES, maxRetries);
RetryPolicy policy = FederationStateStoreFacade.createRetryPolicy(conf);
RetryAction action = policy.shouldRetry(
new PoolInitializationException(new YarnException()), 0, 0, false);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/de462da0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md
index 3e3580c..8a6c137 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-site/src/site/markdown/Federation.md
@@ -129,6 +129,7 @@ AMRMProxy, Global Policy Generator (GPG) and Router work together to make this h
The figure shows a sequence diagram for the following job execution flow:
+
1. The Router receives an application submission request that is complaint to the YARN Application Client Protocol.
2. The router interrogates a routing table / policy to choose the “home RM” for the job (the policy configuration is received from the state-store on heartbeat).
3. The router queries the membership state to determine the endpoint of the home RM.
@@ -160,15 +161,50 @@ These are common configurations that should appear in the **conf/yarn-site.xml**
| Property | Example | Description |
|:---- |:---- |
|`yarn.federation.enabled` | `true` | Whether federation is enabled or not |
+|`yarn.resourcemanager.cluster-id` | `<unique-subcluster-id>` | The unique subcluster identifier for this RM (same as the one used for HA). |
+
+####State-Store:
+
+Currently, we support ZooKeeper and SQL based implementations of the state-store.
+
+**Note:** The State-Store implementation must always be overwritten with one of the below.
+
+ZooKeeper: one must set the ZooKeeper settings for Hadoop:
+
+| Property | Example | Description |
+|:---- |:---- |
+|`yarn.federation.state-store.class` | `org.apache.hadoop.yarn.server.federation.store.impl.ZookeeperFederationStateStore` | The type of state-store to use. |
+|`hadoop.zk.address` | `host:port` | The address for the ZooKeeper ensemble. |
+
+SQL: one must setup the following parameters:
+
+| Property | Example | Description |
+|:---- |:---- |
|`yarn.federation.state-store.class` | `org.apache.hadoop.yarn.server.federation.store.impl.SQLFederationStateStore` | The type of state-store to use. |
|`yarn.federation.state-store.sql.url` | `jdbc:mysql://<host>:<port>/FederationStateStore` | For SQLFederationStateStore the name of the DB where the state is stored. |
|`yarn.federation.state-store.sql.jdbc-class` | `com.mysql.jdbc.jdbc2.optional.MysqlDataSource` | For SQLFederationStateStore the jdbc class to use. |
|`yarn.federation.state-store.sql.username` | `<dbuser>` | For SQLFederationStateStore the username for the DB connection. |
|`yarn.federation.state-store.sql.password` | `<dbpass>` | For SQLFederationStateStore the password for the DB connection. |
-|`yarn.resourcemanager.cluster-id` | `<unique-subcluster-id>` | The unique subcluster identifier for this RM (same as the one used for HA). |
+We provide scripts for MySQL and Microsoft SQL Server.
-Optional:
+For MySQL, one must download the latest jar version 5.x from [MVN Repository](https://mvnrepository.com/artifact/mysql/mysql-connector-java) and add it to the CLASSPATH.
+Then the DB schema is created by executing the following SQL scripts in the database:
+
+1. **sbin/FederationStateStore/MySQL/FederationStateStoreDatabase.sql**.
+2. **sbin/FederationStateStore/MySQL/FederationStateStoreUser.sql**.
+3. **sbin/FederationStateStore/MySQL/FederationStateStoreTables.sql**.
+4. **sbin/FederationStateStore/MySQL/FederationStateStoreStoredProcs.sql**.
+
+In the same directory we provide scripts to drop the Stored Procedures, the Tables, the User and the Database.
+
+**Note:** the FederationStateStoreUser.sql defines a default user/password for the DB that you are **highly encouraged** to set this to a proper strong password.
+
+For SQL-Server, the process is similar, but the jdbc driver is already included.
+SQL-Server scripts are located in **sbin/FederationStateStore/SQLServer/**.
+
+
+####Optional:
| Property | Example | Description |
|:---- |:---- |
@@ -236,22 +272,6 @@ Optional:
|`yarn.federation.statestore.max-connections` | `1` | The maximum number of parallel connections from each AMRMProxy to the state-store. This value is typically lower than the router one, since we have many AMRMProxy that could burn-through many DB connections quickly. |
|`yarn.federation.cache-ttl.secs` | `300` | The time to leave for the AMRMProxy cache. Typically larger than at the router, as the number of AMRMProxy is large, and we want to limit the load to the centralized state-store. |
-###State-Store:
-
-Currently, we support only SQL based implementation of state-store (ZooKeeper is in the works), i.e. either MySQL or Microsoft SQL Server.
-
-For MySQL, one must download the latest jar version 5.x from [MVN Repository](https://mvnrepository.com/artifact/mysql/mysql-connector-java) and add it to the CLASSPATH.
-Then the DB schema is created by executing the following SQL scripts in the database:
-1. **sbin/FederationStateStore/MySQL/FederationStateStoreDatabase.sql**.
-2. **sbin/FederationStateStore/MySQL/FederationStateStoreUser.sql**.
-3. **sbin/FederationStateStore/MySQL/FederationStateStoreTables.sql**.
-4. **sbin/FederationStateStore/MySQL/FederationStateStoreStoredProcs.sql**.
-In the same directory we provide scripts to drop the Stored Procedures, the Tables, the User and the Database.
-**Note:** the FederationStateStoreUser.sql defines a default user/password for the DB that you are **highly encouraged** to set this to a proper strong password.
-
-For SQL-Server, the process is similar, but the jdbc driver is already included in the pom (license allows it).
-SQL-Server scripts are located in **sbin/FederationStateStore/SQLServer/**.
-
Running a Sample Job
--------------------
In order to submit jobs to a Federation cluster one must create a seperate set of configs for the client from which jobs will be submitted. In these, the **conf/yarn-site.xml** should have the following additional configurations:
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[09/50] [abbrv] hadoop git commit: MAPREDUCE-6936. Remove unnecessary
dependency of hadoop-yarn-server-common from hadoop-mapreduce-client-common
(haibochen via rkanter)
Posted by as...@apache.org.
MAPREDUCE-6936. Remove unnecessary dependency of hadoop-yarn-server-common from hadoop-mapreduce-client-common (haibochen via rkanter)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ab051bd4
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ab051bd4
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ab051bd4
Branch: refs/heads/YARN-5972
Commit: ab051bd42ee1d7c4d3b7cc71e6b2734a0955e767
Parents: 0acc5e0
Author: Robert Kanter <rk...@apache.org>
Authored: Wed Aug 16 16:14:04 2017 -0700
Committer: Robert Kanter <rk...@apache.org>
Committed: Wed Aug 16 16:14:04 2017 -0700
----------------------------------------------------------------------
.../hadoop-mapreduce-client-common/pom.xml | 4 ----
1 file changed, 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ab051bd4/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml
index db8ae49..b88b012 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-common/pom.xml
@@ -46,10 +46,6 @@
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-mapreduce-client-core</artifactId>
</dependency>
- <dependency>
- <groupId>org.apache.hadoop</groupId>
- <artifactId>hadoop-yarn-server-common</artifactId>
- </dependency>
</dependencies>
<build>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[12/50] [abbrv] hadoop git commit: HADOOP-14560. Make HttpServer2
backlog size configurable. Contributed by Alexander Krasheninnikov.
Posted by as...@apache.org.
HADOOP-14560. Make HttpServer2 backlog size configurable. Contributed by Alexander Krasheninnikov.
This closes #242.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/1f04cb45
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/1f04cb45
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/1f04cb45
Branch: refs/heads/YARN-5972
Commit: 1f04cb45f70648678840cdafbec68d534b03fe95
Parents: 96b3a6b
Author: Alexandr Krasheninnikov <a....@corp.badoo.com>
Authored: Wed Jun 21 12:57:34 2017 +0300
Committer: John Zhuge <jz...@apache.org>
Committed: Thu Aug 17 01:05:19 2017 -0700
----------------------------------------------------------------------
.../main/java/org/apache/hadoop/http/HttpServer2.java | 9 ++++++++-
.../java/org/apache/hadoop/http/TestHttpServer.java | 13 +++++++++++++
2 files changed, 21 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1f04cb45/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java
index 28b9bb0..a450f66 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/http/HttpServer2.java
@@ -128,6 +128,10 @@ public final class HttpServer2 implements FilterContainer {
public static final String HTTP_MAX_RESPONSE_HEADER_SIZE_KEY =
"hadoop.http.max.response.header.size";
public static final int HTTP_MAX_RESPONSE_HEADER_SIZE_DEFAULT = 65536;
+
+ public static final String HTTP_SOCKET_BACKLOG_SIZE_KEY =
+ "hadoop.http.socket.backlog.size";
+ public static final int HTTP_SOCKET_BACKLOG_SIZE_DEFAULT = 128;
public static final String HTTP_MAX_THREADS_KEY = "hadoop.http.max.threads";
public static final String HTTP_TEMP_DIR_KEY = "hadoop.http.temp.dir";
@@ -433,6 +437,9 @@ public final class HttpServer2 implements FilterContainer {
httpConfig.setResponseHeaderSize(responseHeaderSize);
httpConfig.setSendServerVersion(false);
+ int backlogSize = conf.getInt(HTTP_SOCKET_BACKLOG_SIZE_KEY,
+ HTTP_SOCKET_BACKLOG_SIZE_DEFAULT);
+
for (URI ep : endpoints) {
final ServerConnector connector;
String scheme = ep.getScheme();
@@ -448,6 +455,7 @@ public final class HttpServer2 implements FilterContainer {
}
connector.setHost(ep.getHost());
connector.setPort(ep.getPort() == -1 ? 0 : ep.getPort());
+ connector.setAcceptQueueSize(backlogSize);
server.addListener(connector);
}
server.loadListeners();
@@ -640,7 +648,6 @@ public final class HttpServer2 implements FilterContainer {
private static void configureChannelConnector(ServerConnector c) {
c.setIdleTimeout(10000);
- c.setAcceptQueueSize(128);
if(Shell.WINDOWS) {
// result of setting the SO_REUSEADDR flag is different on Windows
// http://msdn.microsoft.com/en-us/library/ms740621(v=vs.85).aspx
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1f04cb45/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServer.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServer.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServer.java
index 6ec6e0f..ca7e466 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServer.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/http/TestHttpServer.java
@@ -682,4 +682,17 @@ public class TestHttpServer extends HttpServerFunctionalTest {
stopHttpServer(myServer2);
}
}
+
+ @Test
+ public void testBacklogSize() throws Exception
+ {
+ final int backlogSize = 2048;
+ Configuration conf = new Configuration();
+ conf.setInt(HttpServer2.HTTP_SOCKET_BACKLOG_SIZE_KEY, backlogSize);
+ HttpServer2 srv = createServer("test", conf);
+ List<?> listeners = (List<?>) Whitebox.getInternalState(srv,
+ "listeners");
+ ServerConnector listener = (ServerConnector)listeners.get(0);
+ assertEquals(backlogSize, listener.getAcceptQueueSize());
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[38/50] [abbrv] hadoop git commit: YARN-7048. Fix tests faking
kerberos to explicitly set ugi auth type. Contributed by Daryn Sharp
Posted by as...@apache.org.
YARN-7048. Fix tests faking kerberos to explicitly set ugi auth type. Contributed by Daryn Sharp
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/657dd59c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/657dd59c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/657dd59c
Branch: refs/heads/YARN-5972
Commit: 657dd59cc840bce3426170baf68f9d46842f4f53
Parents: 3efcd51
Author: Jason Lowe <jl...@apache.org>
Authored: Tue Aug 22 13:16:24 2017 -0500
Committer: Jason Lowe <jl...@apache.org>
Committed: Tue Aug 22 13:16:24 2017 -0500
----------------------------------------------------------------------
.../yarn/server/resourcemanager/TestTokenClientRMService.java | 3 +++
.../server/resourcemanager/security/TestRMDelegationTokens.java | 4 ++++
2 files changed, 7 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/657dd59c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestTokenClientRMService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestTokenClientRMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestTokenClientRMService.java
index 2a4e49d..78271c6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestTokenClientRMService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestTokenClientRMService.java
@@ -28,6 +28,7 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.security.SaslRpcServer.AuthMethod;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
import org.apache.hadoop.security.authentication.util.KerberosName;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.yarn.api.protocolrecords.CancelDelegationTokenRequest;
@@ -80,6 +81,8 @@ public class TestTokenClientRMService {
conf.set("hadoop.security.authentication", "kerberos");
conf.set("hadoop.security.auth_to_local", kerberosRule);
UserGroupInformation.setConfiguration(conf);
+ UserGroupInformation.getLoginUser()
+ .setAuthenticationMethod(AuthenticationMethod.KERBEROS);
}
@AfterClass
http://git-wip-us.apache.org/repos/asf/hadoop/blob/657dd59c/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestRMDelegationTokens.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestRMDelegationTokens.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestRMDelegationTokens.java
index 06c642a..640293c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestRMDelegationTokens.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/security/TestRMDelegationTokens.java
@@ -29,6 +29,7 @@ import java.util.concurrent.atomic.AtomicInteger;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.delegation.DelegationKey;
import org.apache.hadoop.util.ExitUtil;
@@ -74,6 +75,9 @@ public class TestRMDelegationTokens {
Configuration conf = new Configuration(testConf);
conf.set("hadoop.security.authentication", "kerberos");
UserGroupInformation.setConfiguration(conf);
+ UserGroupInformation.getLoginUser()
+ .setAuthenticationMethod(AuthenticationMethod.KERBEROS);
+
MemoryRMStateStore memStore = new MockRMMemoryStateStore();
memStore.init(conf);
RMState rmState = memStore.getState();
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[20/50] [abbrv] hadoop git commit: YARN-7007. NPE in RM while using
YarnClient.getApplications(). Contributed by Lingfeng Su.
Posted by as...@apache.org.
YARN-7007. NPE in RM while using YarnClient.getApplications(). Contributed by Lingfeng Su.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/e05fa345
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/e05fa345
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/e05fa345
Branch: refs/heads/YARN-5972
Commit: e05fa3451db343c0d22496b332910874b6be5b7f
Parents: c6b4e65
Author: bibinchundatt <bi...@apache.org>
Authored: Fri Aug 18 20:28:50 2017 +0530
Committer: bibinchundatt <bi...@apache.org>
Committed: Fri Aug 18 20:28:50 2017 +0530
----------------------------------------------------------------------
.../rmapp/attempt/RMAppAttemptMetrics.java | 19 +++++++++++--------
1 file changed, 11 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/e05fa345/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java
index e089050..0655609 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/RMAppAttemptMetrics.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationResourceUsageReport;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
+import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType;
import org.apache.hadoop.yarn.util.resource.Resources;
@@ -125,14 +126,16 @@ public class RMAppAttemptMetrics {
long vcoreSeconds = finishedVcoreSeconds.get();
// Only add in the running containers if this is the active attempt.
- RMAppAttempt currentAttempt = rmContext.getRMApps()
- .get(attemptId.getApplicationId()).getCurrentAppAttempt();
- if (currentAttempt.getAppAttemptId().equals(attemptId)) {
- ApplicationResourceUsageReport appResUsageReport = rmContext
- .getScheduler().getAppResourceUsageReport(attemptId);
- if (appResUsageReport != null) {
- memorySeconds += appResUsageReport.getMemorySeconds();
- vcoreSeconds += appResUsageReport.getVcoreSeconds();
+ RMApp rmApp = rmContext.getRMApps().get(attemptId.getApplicationId());
+ if (null != rmApp) {
+ RMAppAttempt currentAttempt = rmApp.getCurrentAppAttempt();
+ if (currentAttempt.getAppAttemptId().equals(attemptId)) {
+ ApplicationResourceUsageReport appResUsageReport = rmContext
+ .getScheduler().getAppResourceUsageReport(attemptId);
+ if (appResUsageReport != null) {
+ memorySeconds += appResUsageReport.getMemorySeconds();
+ vcoreSeconds += appResUsageReport.getVcoreSeconds();
+ }
}
}
return new AggregateAppResourceUsage(memorySeconds, vcoreSeconds);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[43/50] [abbrv] hadoop git commit: HADOOP-14649. Update
aliyun-sdk-oss version to 2.8.1. (Genmao Yu via rchiang)
Posted by as...@apache.org.
HADOOP-14649. Update aliyun-sdk-oss version to 2.8.1. (Genmao Yu via rchiang)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/26d8c8fa
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/26d8c8fa
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/26d8c8fa
Branch: refs/heads/YARN-5972
Commit: 26d8c8fa586a634ae91993940b28b3a1452d4be6
Parents: 7e6463d
Author: Ray Chiang <rc...@apache.org>
Authored: Wed Aug 23 15:22:24 2017 -0700
Committer: Ray Chiang <rc...@apache.org>
Committed: Wed Aug 23 15:22:24 2017 -0700
----------------------------------------------------------------------
hadoop-project/pom.xml | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/26d8c8fa/hadoop-project/pom.xml
----------------------------------------------------------------------
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 8c1d374..938ef05 100755
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -1140,7 +1140,7 @@
<dependency>
<groupId>com.aliyun.oss</groupId>
<artifactId>aliyun-sdk-oss</artifactId>
- <version>2.4.1</version>
+ <version>2.8.1</version>
<exclusions>
<exclusion>
<groupId>org.apache.httpcomponents</groupId>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[06/50] [abbrv] hadoop git commit: YARN-7020.
TestAMRMProxy#testAMRMProxyTokenRenewal is flakey. Contributed by Robert
Kanter
Posted by as...@apache.org.
YARN-7020. TestAMRMProxy#testAMRMProxyTokenRenewal is flakey. Contributed by Robert Kanter
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/14553061
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/14553061
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/14553061
Branch: refs/heads/YARN-5972
Commit: 14553061be0a341df3e628dcaf06717b4630b05e
Parents: 588c190
Author: Jason Lowe <jl...@yahoo-inc.com>
Authored: Wed Aug 16 13:04:36 2017 -0500
Committer: Jason Lowe <jl...@yahoo-inc.com>
Committed: Wed Aug 16 13:04:36 2017 -0500
----------------------------------------------------------------------
.../apache/hadoop/yarn/client/api/impl/TestAMRMProxy.java | 10 +++++-----
1 file changed, 5 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/14553061/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMProxy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMProxy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMProxy.java
index 14df94a..6a063e6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMProxy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMProxy.java
@@ -151,13 +151,13 @@ public class TestAMRMProxy extends BaseAMRMProxyE2ETest {
YarnClient rmClient = YarnClient.createYarnClient()) {
Configuration conf = new YarnConfiguration();
conf.setBoolean(YarnConfiguration.AMRM_PROXY_ENABLED, true);
- conf.setInt(YarnConfiguration.RM_NM_EXPIRY_INTERVAL_MS, 1500);
- conf.setInt(YarnConfiguration.RM_NM_HEARTBEAT_INTERVAL_MS, 1500);
- conf.setInt(YarnConfiguration.RM_AM_EXPIRY_INTERVAL_MS, 1500);
+ conf.setInt(YarnConfiguration.RM_NM_EXPIRY_INTERVAL_MS, 4500);
+ conf.setInt(YarnConfiguration.RM_NM_HEARTBEAT_INTERVAL_MS, 4500);
+ conf.setInt(YarnConfiguration.RM_AM_EXPIRY_INTERVAL_MS, 4500);
// RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS should be at least
// RM_AM_EXPIRY_INTERVAL_MS * 1.5 *3
conf.setInt(
- YarnConfiguration.RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS, 6);
+ YarnConfiguration.RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS, 20);
cluster.init(conf);
cluster.start();
final Configuration yarnConf = cluster.getConfig();
@@ -198,7 +198,7 @@ public class TestAMRMProxy extends BaseAMRMProxyE2ETest {
lastToken = response.getAMRMToken();
// Time slot to be sure the AMRMProxy renew the token
- Thread.sleep(1500);
+ Thread.sleep(4500);
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[14/50] [abbrv] hadoop git commit: HDFS-12250. Reduce usage of
FsPermissionExtension in unit tests. Contributed by Chris Douglas.
Posted by as...@apache.org.
HDFS-12250. Reduce usage of FsPermissionExtension in unit tests. Contributed by Chris Douglas.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/dd7916d3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/dd7916d3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/dd7916d3
Branch: refs/heads/YARN-5972
Commit: dd7916d3cd5d880d0b257d229f43f10feff04c93
Parents: f9a0e23
Author: Andrew Wang <wa...@apache.org>
Authored: Thu Aug 17 09:35:36 2017 -0700
Committer: Andrew Wang <wa...@apache.org>
Committed: Thu Aug 17 09:35:36 2017 -0700
----------------------------------------------------------------------
.../hadoop/fs/permission/FsPermission.java | 2 +-
.../org/apache/hadoop/fs/shell/AclCommands.java | 6 ++---
.../hadoop/fs/shell/CommandWithDestination.java | 4 ++--
.../java/org/apache/hadoop/fs/shell/Ls.java | 4 ++--
.../fs/http/client/BaseTestHttpFSWith.java | 1 +
.../org/apache/hadoop/hdfs/TestDFSShell.java | 24 ++++++++++----------
.../hdfs/server/namenode/FSAclBaseTest.java | 6 +++++
.../ClientDistributedCacheManager.java | 6 ++---
.../apache/hadoop/fs/adl/TestGetFileStatus.java | 1 +
.../hadoop/tools/CopyListingFileStatus.java | 4 ++--
.../apache/hadoop/tools/util/DistCpUtils.java | 4 +---
11 files changed, 33 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dd7916d3/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java
index 23692de..031092b 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/permission/FsPermission.java
@@ -163,7 +163,7 @@ public class FsPermission implements Writable, Serializable,
*/
public static FsPermission read(DataInput in) throws IOException {
FsPermission p = new FsPermission();
- p.readFields(in);
+ p.fromShort(in.readShort());
return p;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dd7916d3/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/AclCommands.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/AclCommands.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/AclCommands.java
index a5e386c..701c9de 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/AclCommands.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/AclCommands.java
@@ -86,9 +86,9 @@ class AclCommands extends FsCommand {
(perm.getOtherAction().implies(FsAction.EXECUTE) ? "t" : "T"));
}
- AclStatus aclStatus = null;
- List<AclEntry> entries = null;
- if (perm.getAclBit()) {
+ final AclStatus aclStatus;
+ final List<AclEntry> entries;
+ if (item.stat.hasAcl()) {
aclStatus = item.fs.getAclStatus(item.path);
entries = aclStatus.getEntries();
} else {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dd7916d3/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java
index 2a483c0..0bd4882 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/CommandWithDestination.java
@@ -444,8 +444,8 @@ abstract class CommandWithDestination extends FsCommand {
src.stat.getPermission());
}
if (shouldPreserve(FileAttribute.ACL)) {
- FsPermission perm = src.stat.getPermission();
- if (perm.getAclBit()) {
+ if (src.stat.hasAcl()) {
+ FsPermission perm = src.stat.getPermission();
List<AclEntry> srcEntries =
src.fs.getAclStatus(src.path).getEntries();
List<AclEntry> srcFullEntries =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dd7916d3/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java
index 221b3cb..a2d5017 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/shell/Ls.java
@@ -255,7 +255,7 @@ class Ls extends FsCommand {
ContentSummary contentSummary = item.fs.getContentSummary(item.path);
String line = String.format(lineFormat,
(stat.isDirectory() ? "d" : "-"),
- stat.getPermission() + (stat.getPermission().getAclBit() ? "+" : " "),
+ stat.getPermission() + (stat.hasAcl() ? "+" : " "),
(stat.isFile() ? stat.getReplication() : "-"),
stat.getOwner(),
stat.getGroup(),
@@ -269,7 +269,7 @@ class Ls extends FsCommand {
} else {
String line = String.format(lineFormat,
(stat.isDirectory() ? "d" : "-"),
- stat.getPermission() + (stat.getPermission().getAclBit() ? "+" : " "),
+ stat.getPermission() + (stat.hasAcl() ? "+" : " "),
(stat.isFile() ? stat.getReplication() : "-"),
stat.getOwner(),
stat.getGroup(),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dd7916d3/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java
index 553bbce..2cd8934 100644
--- a/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java
+++ b/hadoop-hdfs-project/hadoop-hdfs-httpfs/src/test/java/org/apache/hadoop/fs/http/client/BaseTestHttpFSWith.java
@@ -859,6 +859,7 @@ public abstract class BaseTestHttpFSWith extends HFSTestCase {
FileStatus expectedFileStatus = expected.getFileStatus(path);
FileStatus actualFileStatus = actual.getFileStatus(path);
assertEquals(actualFileStatus.hasAcl(), expectedFileStatus.hasAcl());
+ // backwards compat
assertEquals(actualFileStatus.getPermission().getAclBit(),
expectedFileStatus.getPermission().getAclBit());
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dd7916d3/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java
index 27d41b4..9ae49aa 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestDFSShell.java
@@ -2189,7 +2189,7 @@ public class TestDFSShell {
assertTrue(xattrs.isEmpty());
List<AclEntry> acls = dfs.getAclStatus(target1).getEntries();
assertTrue(acls.isEmpty());
- assertFalse(targetPerm.getAclBit());
+ assertFalse(targetStatus.hasAcl());
// -ptop
Path target2 = new Path(hdfsTestDir, "targetfile2");
@@ -2208,7 +2208,7 @@ public class TestDFSShell {
assertTrue(xattrs.isEmpty());
acls = dfs.getAclStatus(target2).getEntries();
assertTrue(acls.isEmpty());
- assertFalse(targetPerm.getAclBit());
+ assertFalse(targetStatus.hasAcl());
// -ptopx
Path target3 = new Path(hdfsTestDir, "targetfile3");
@@ -2229,7 +2229,7 @@ public class TestDFSShell {
assertArrayEquals(TRUSTED_A1_VALUE, xattrs.get(TRUSTED_A1));
acls = dfs.getAclStatus(target3).getEntries();
assertTrue(acls.isEmpty());
- assertFalse(targetPerm.getAclBit());
+ assertFalse(targetStatus.hasAcl());
// -ptopa
Path target4 = new Path(hdfsTestDir, "targetfile4");
@@ -2248,7 +2248,7 @@ public class TestDFSShell {
assertTrue(xattrs.isEmpty());
acls = dfs.getAclStatus(target4).getEntries();
assertFalse(acls.isEmpty());
- assertTrue(targetPerm.getAclBit());
+ assertTrue(targetStatus.hasAcl());
assertEquals(dfs.getAclStatus(src), dfs.getAclStatus(target4));
// -ptoa (verify -pa option will preserve permissions also)
@@ -2268,7 +2268,7 @@ public class TestDFSShell {
assertTrue(xattrs.isEmpty());
acls = dfs.getAclStatus(target5).getEntries();
assertFalse(acls.isEmpty());
- assertTrue(targetPerm.getAclBit());
+ assertTrue(targetStatus.hasAcl());
assertEquals(dfs.getAclStatus(src), dfs.getAclStatus(target5));
} finally {
if (null != shell) {
@@ -2480,7 +2480,7 @@ public class TestDFSShell {
assertTrue(xattrs.isEmpty());
List<AclEntry> acls = dfs.getAclStatus(targetDir1).getEntries();
assertTrue(acls.isEmpty());
- assertFalse(targetPerm.getAclBit());
+ assertFalse(targetStatus.hasAcl());
// -ptop
Path targetDir2 = new Path(hdfsTestDir, "targetDir2");
@@ -2499,7 +2499,7 @@ public class TestDFSShell {
assertTrue(xattrs.isEmpty());
acls = dfs.getAclStatus(targetDir2).getEntries();
assertTrue(acls.isEmpty());
- assertFalse(targetPerm.getAclBit());
+ assertFalse(targetStatus.hasAcl());
// -ptopx
Path targetDir3 = new Path(hdfsTestDir, "targetDir3");
@@ -2520,7 +2520,7 @@ public class TestDFSShell {
assertArrayEquals(TRUSTED_A1_VALUE, xattrs.get(TRUSTED_A1));
acls = dfs.getAclStatus(targetDir3).getEntries();
assertTrue(acls.isEmpty());
- assertFalse(targetPerm.getAclBit());
+ assertFalse(targetStatus.hasAcl());
// -ptopa
Path targetDir4 = new Path(hdfsTestDir, "targetDir4");
@@ -2539,7 +2539,7 @@ public class TestDFSShell {
assertTrue(xattrs.isEmpty());
acls = dfs.getAclStatus(targetDir4).getEntries();
assertFalse(acls.isEmpty());
- assertTrue(targetPerm.getAclBit());
+ assertTrue(targetStatus.hasAcl());
assertEquals(dfs.getAclStatus(srcDir), dfs.getAclStatus(targetDir4));
// -ptoa (verify -pa option will preserve permissions also)
@@ -2559,7 +2559,7 @@ public class TestDFSShell {
assertTrue(xattrs.isEmpty());
acls = dfs.getAclStatus(targetDir5).getEntries();
assertFalse(acls.isEmpty());
- assertTrue(targetPerm.getAclBit());
+ assertTrue(targetStatus.hasAcl());
assertEquals(dfs.getAclStatus(srcDir), dfs.getAclStatus(targetDir5));
} finally {
if (shell != null) {
@@ -2615,7 +2615,7 @@ public class TestDFSShell {
assertTrue(perm.equals(targetPerm));
List<AclEntry> acls = dfs.getAclStatus(target1).getEntries();
assertTrue(acls.isEmpty());
- assertFalse(targetPerm.getAclBit());
+ assertFalse(targetStatus.hasAcl());
// -ptopa preserves both sticky bit and ACL
Path target2 = new Path(hdfsTestDir, "targetfile2");
@@ -2632,7 +2632,7 @@ public class TestDFSShell {
assertTrue(perm.equals(targetPerm));
acls = dfs.getAclStatus(target2).getEntries();
assertFalse(acls.isEmpty());
- assertTrue(targetPerm.getAclBit());
+ assertTrue(targetStatus.hasAcl());
assertEquals(dfs.getAclStatus(src), dfs.getAclStatus(target2));
} finally {
if (null != shell) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dd7916d3/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSAclBaseTest.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSAclBaseTest.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSAclBaseTest.java
index 93a83fd..ee92217 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSAclBaseTest.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/FSAclBaseTest.java
@@ -32,6 +32,7 @@ import java.util.Map;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
+import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.AclEntry;
@@ -886,6 +887,11 @@ public abstract class FSAclBaseTest {
FsPermission perm = inode.getFsPermission();
assertNotNull(perm);
assertEquals(0755, perm.toShort());
+ FileStatus stat = fs.getFileStatus(path);
+ assertFalse(stat.hasAcl());
+ assertFalse(stat.isEncrypted());
+ assertFalse(stat.isErasureCoded());
+ // backwards-compat check
assertEquals(0755, perm.toExtendedShort());
assertAclFeature(false);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dd7916d3/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/filecache/ClientDistributedCacheManager.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/filecache/ClientDistributedCacheManager.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/filecache/ClientDistributedCacheManager.java
index 9f8edb5..ada14db 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/filecache/ClientDistributedCacheManager.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/filecache/ClientDistributedCacheManager.java
@@ -28,7 +28,6 @@ import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsAction;
-import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.mapreduce.MRJobConfig;
import org.apache.hadoop.mapreduce.security.TokenCache;
import org.apache.hadoop.security.Credentials;
@@ -293,7 +292,6 @@ public class ClientDistributedCacheManager {
private static boolean checkPermissionOfOther(FileSystem fs, Path path,
FsAction action, Map<URI, FileStatus> statCache) throws IOException {
FileStatus status = getFileStatus(fs, path.toUri(), statCache);
- FsPermission perms = status.getPermission();
// Encrypted files are always treated as private. This stance has two
// important side effects. The first is that the encrypted files will be
@@ -302,8 +300,8 @@ public class ClientDistributedCacheManager {
// world readable permissions that is stored in an encryption zone from
// being localized as a publicly shared file with world readable
// permissions.
- if (!perms.getEncryptedBit()) {
- FsAction otherAction = perms.getOtherAction();
+ if (!status.isEncrypted()) {
+ FsAction otherAction = status.getPermission().getOtherAction();
if (otherAction.implies(action)) {
return true;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dd7916d3/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestGetFileStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestGetFileStatus.java b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestGetFileStatus.java
index d9e22db..95c2363 100644
--- a/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestGetFileStatus.java
+++ b/hadoop-tools/hadoop-azure-datalake/src/test/java/org/apache/hadoop/fs/adl/TestGetFileStatus.java
@@ -98,4 +98,5 @@ public class TestGetFileStatus extends AdlMockWebServer {
Assert.assertFalse(fileStatus.hasAcl());
Assert.assertFalse(fileStatus.getPermission().getAclBit());
}
+
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dd7916d3/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListingFileStatus.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListingFileStatus.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListingFileStatus.java
index 29c59ac..138b491 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListingFileStatus.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/CopyListingFileStatus.java
@@ -280,7 +280,7 @@ public final class CopyListingFileStatus implements Writable {
out.writeLong(getBlockSize());
out.writeLong(getModificationTime());
out.writeLong(getAccessTime());
- getPermission().write(out);
+ out.writeShort(getPermission().toShort());
Text.writeString(out, getOwner(), Text.DEFAULT_MAX_LEN);
Text.writeString(out, getGroup(), Text.DEFAULT_MAX_LEN);
if (aclEntries != null) {
@@ -330,7 +330,7 @@ public final class CopyListingFileStatus implements Writable {
blocksize = in.readLong();
modificationTime = in.readLong();
accessTime = in.readLong();
- permission.readFields(in);
+ permission.fromShort(in.readShort());
owner = Text.readString(in, Text.DEFAULT_MAX_LEN);
group = Text.readString(in, Text.DEFAULT_MAX_LEN);
byte aclEntriesSize = in.readByte();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dd7916d3/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/util/DistCpUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/util/DistCpUtils.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/util/DistCpUtils.java
index dbe750a..2b3b529 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/util/DistCpUtils.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/util/DistCpUtils.java
@@ -31,7 +31,6 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.XAttr;
import org.apache.hadoop.fs.permission.AclEntry;
import org.apache.hadoop.fs.permission.AclUtil;
-import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.io.SequenceFile;
import org.apache.hadoop.io.Text;
@@ -403,8 +402,7 @@ public class DistCpUtils {
CopyListingFileStatus copyListingFileStatus =
new CopyListingFileStatus(fileStatus, chunkOffset, chunkLength);
if (preserveAcls) {
- FsPermission perm = fileStatus.getPermission();
- if (perm.getAclBit()) {
+ if (fileStatus.hasAcl()) {
List<AclEntry> aclEntries = fileSystem.getAclStatus(
fileStatus.getPath()).getEntries();
copyListingFileStatus.setAclEntries(aclEntries);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[31/50] [abbrv] hadoop git commit: YARN-6923. Metrics for Federation
Router. (Giovanni Matteo Fumarola via asuresh)
Posted by as...@apache.org.
YARN-6923. Metrics for Federation Router. (Giovanni Matteo Fumarola via asuresh)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/ae8fb13b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/ae8fb13b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/ae8fb13b
Branch: refs/heads/YARN-5972
Commit: ae8fb13b312b30de50d65b5450b565d50d690e9e
Parents: 75abc9a
Author: Arun Suresh <as...@apache.org>
Authored: Mon Aug 21 22:50:24 2017 -0700
Committer: Arun Suresh <as...@apache.org>
Committed: Mon Aug 21 22:50:24 2017 -0700
----------------------------------------------------------------------
.../yarn/server/router/RouterMetrics.java | 203 +++++++++++++++
.../clientrm/FederationClientInterceptor.java | 37 ++-
.../webapp/FederationInterceptorREST.java | 116 +++++++--
.../yarn/server/router/TestRouterMetrics.java | 248 +++++++++++++++++++
.../webapp/TestFederationInterceptorREST.java | 12 +-
5 files changed, 593 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ae8fb13b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterMetrics.java
new file mode 100644
index 0000000..42361a3
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/RouterMetrics.java
@@ -0,0 +1,203 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.router;
+
+import com.google.common.annotations.VisibleForTesting;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.metrics2.MetricsInfo;
+import org.apache.hadoop.metrics2.annotation.Metric;
+import org.apache.hadoop.metrics2.annotation.Metrics;
+import org.apache.hadoop.metrics2.lib.*;
+
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import static org.apache.hadoop.metrics2.lib.Interns.info;
+
+/**
+ * This class is for maintaining the various Router Federation Interceptor
+ * activity statistics and publishing them through the metrics interfaces.
+ */
+@InterfaceAudience.Private
+@Metrics(about = "Metrics for Router Federation Interceptor", context = "fedr")
+public final class RouterMetrics {
+
+ private static final MetricsInfo RECORD_INFO =
+ info("RouterMetrics", "Router Federation Interceptor");
+ private static AtomicBoolean isInitialized = new AtomicBoolean(false);
+
+ // Metrics for operation failed
+ @Metric("# of applications failed to be submitted")
+ private MutableGaugeInt numAppsFailedSubmitted;
+ @Metric("# of applications failed to be created")
+ private MutableGaugeInt numAppsFailedCreated;
+ @Metric("# of applications failed to be killed")
+ private MutableGaugeInt numAppsFailedKilled;
+ @Metric("# of application reports failed to be retrieved")
+ private MutableGaugeInt numAppsFailedRetrieved;
+
+ // Aggregate metrics are shared, and don't have to be looked up per call
+ @Metric("Total number of successful Submitted apps and latency(ms)")
+ private MutableRate totalSucceededAppsSubmitted;
+ @Metric("Total number of successful Killed apps and latency(ms)")
+ private MutableRate totalSucceededAppsKilled;
+ @Metric("Total number of successful Created apps and latency(ms)")
+ private MutableRate totalSucceededAppsCreated;
+ @Metric("Total number of successful Retrieved app reports and latency(ms)")
+ private MutableRate totalSucceededAppsRetrieved;
+
+ /**
+ * Provide quantile counters for all latencies.
+ */
+ private MutableQuantiles submitApplicationLatency;
+ private MutableQuantiles getNewApplicationLatency;
+ private MutableQuantiles killApplicationLatency;
+ private MutableQuantiles getApplicationReportLatency;
+
+ private static volatile RouterMetrics INSTANCE = null;
+ private static MetricsRegistry registry;
+
+ private RouterMetrics() {
+ registry = new MetricsRegistry(RECORD_INFO);
+ registry.tag(RECORD_INFO, "Router");
+ getNewApplicationLatency = registry.newQuantiles("getNewApplicationLatency",
+ "latency of get new application", "ops", "latency", 10);
+ submitApplicationLatency = registry.newQuantiles("submitApplicationLatency",
+ "latency of submit application", "ops", "latency", 10);
+ killApplicationLatency = registry.newQuantiles("killApplicationLatency",
+ "latency of kill application", "ops", "latency", 10);
+ getApplicationReportLatency =
+ registry.newQuantiles("getApplicationReportLatency",
+ "latency of get application report", "ops", "latency", 10);
+ }
+
+ public static RouterMetrics getMetrics() {
+ if (!isInitialized.get()) {
+ synchronized (RouterMetrics.class) {
+ if (INSTANCE == null) {
+ INSTANCE = DefaultMetricsSystem.instance().register("RouterMetrics",
+ "Metrics for the Yarn Router", new RouterMetrics());
+ isInitialized.set(true);
+ }
+ }
+ }
+ return INSTANCE;
+ }
+
+ @VisibleForTesting
+ synchronized static void destroy() {
+ isInitialized.set(false);
+ INSTANCE = null;
+ }
+
+ @VisibleForTesting
+ public long getNumSucceededAppsCreated() {
+ return totalSucceededAppsCreated.lastStat().numSamples();
+ }
+
+ @VisibleForTesting
+ public long getNumSucceededAppsSubmitted() {
+ return totalSucceededAppsSubmitted.lastStat().numSamples();
+ }
+
+ @VisibleForTesting
+ public long getNumSucceededAppsKilled() {
+ return totalSucceededAppsKilled.lastStat().numSamples();
+ }
+
+ @VisibleForTesting
+ public long getNumSucceededAppsRetrieved() {
+ return totalSucceededAppsRetrieved.lastStat().numSamples();
+ }
+
+ @VisibleForTesting
+ public double getLatencySucceededAppsCreated() {
+ return totalSucceededAppsCreated.lastStat().mean();
+ }
+
+ @VisibleForTesting
+ public double getLatencySucceededAppsSubmitted() {
+ return totalSucceededAppsSubmitted.lastStat().mean();
+ }
+
+ @VisibleForTesting
+ public double getLatencySucceededAppsKilled() {
+ return totalSucceededAppsKilled.lastStat().mean();
+ }
+
+ @VisibleForTesting
+ public double getLatencySucceededGetAppReport() {
+ return totalSucceededAppsRetrieved.lastStat().mean();
+ }
+
+ @VisibleForTesting
+ public int getAppsFailedCreated() {
+ return numAppsFailedCreated.value();
+ }
+
+ @VisibleForTesting
+ public int getAppsFailedSubmitted() {
+ return numAppsFailedSubmitted.value();
+ }
+
+ @VisibleForTesting
+ public int getAppsFailedKilled() {
+ return numAppsFailedKilled.value();
+ }
+
+ @VisibleForTesting
+ public int getAppsFailedRetrieved() {
+ return numAppsFailedRetrieved.value();
+ }
+
+ public void succeededAppsCreated(long duration) {
+ totalSucceededAppsCreated.add(duration);
+ getNewApplicationLatency.add(duration);
+ }
+
+ public void succeededAppsSubmitted(long duration) {
+ totalSucceededAppsSubmitted.add(duration);
+ submitApplicationLatency.add(duration);
+ }
+
+ public void succeededAppsKilled(long duration) {
+ totalSucceededAppsKilled.add(duration);
+ killApplicationLatency.add(duration);
+ }
+
+ public void succeededAppsRetrieved(long duration) {
+ totalSucceededAppsRetrieved.add(duration);
+ getApplicationReportLatency.add(duration);
+ }
+
+ public void incrAppsFailedCreated() {
+ numAppsFailedCreated.incr();
+ }
+
+ public void incrAppsFailedSubmitted() {
+ numAppsFailedSubmitted.incr();
+ }
+
+ public void incrAppsFailedKilled() {
+ numAppsFailedKilled.incr();
+ }
+
+ public void incrAppsFailedRetrieved() {
+ numAppsFailedRetrieved.incr();
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ae8fb13b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java
index 7268ebd..3a36eec 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/clientrm/FederationClientInterceptor.java
@@ -98,7 +98,10 @@ import org.apache.hadoop.yarn.server.federation.store.records.ApplicationHomeSub
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade;
+import org.apache.hadoop.yarn.server.router.RouterMetrics;
import org.apache.hadoop.yarn.server.router.RouterServerUtil;
+import org.apache.hadoop.yarn.util.Clock;
+import org.apache.hadoop.yarn.util.MonotonicClock;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -130,6 +133,8 @@ public class FederationClientInterceptor
private FederationStateStoreFacade federationFacade;
private Random rand;
private RouterPolicyFacade policyFacade;
+ private RouterMetrics routerMetrics;
+ private final Clock clock = new MonotonicClock();
@Override
public void init(String userName) {
@@ -153,7 +158,7 @@ public class FederationClientInterceptor
clientRMProxies =
new ConcurrentHashMap<SubClusterId, ApplicationClientProtocol>();
-
+ routerMetrics = RouterMetrics.getMetrics();
}
@Override
@@ -220,6 +225,9 @@ public class FederationClientInterceptor
@Override
public GetNewApplicationResponse getNewApplication(
GetNewApplicationRequest request) throws YarnException, IOException {
+
+ long startTime = clock.getTime();
+
Map<SubClusterId, SubClusterInfo> subClustersActive =
federationFacade.getSubClusters(true);
@@ -238,6 +246,9 @@ public class FederationClientInterceptor
}
if (response != null) {
+
+ long stopTime = clock.getTime();
+ routerMetrics.succeededAppsCreated(stopTime - startTime);
return response;
} else {
// Empty response from the ResourceManager.
@@ -247,6 +258,7 @@ public class FederationClientInterceptor
}
+ routerMetrics.incrAppsFailedCreated();
String errMsg = "Fail to create a new application.";
LOG.error(errMsg);
throw new YarnException(errMsg);
@@ -320,9 +332,13 @@ public class FederationClientInterceptor
@Override
public SubmitApplicationResponse submitApplication(
SubmitApplicationRequest request) throws YarnException, IOException {
+
+ long startTime = clock.getTime();
+
if (request == null || request.getApplicationSubmissionContext() == null
|| request.getApplicationSubmissionContext()
.getApplicationId() == null) {
+ routerMetrics.incrAppsFailedSubmitted();
RouterServerUtil
.logAndThrowException("Missing submitApplication request or "
+ "applicationSubmissionContex information.", null);
@@ -350,6 +366,7 @@ public class FederationClientInterceptor
subClusterId =
federationFacade.addApplicationHomeSubCluster(appHomeSubCluster);
} catch (YarnException e) {
+ routerMetrics.incrAppsFailedSubmitted();
String message = "Unable to insert the ApplicationId " + applicationId
+ " into the FederationStateStore";
RouterServerUtil.logAndThrowException(message, e);
@@ -368,6 +385,7 @@ public class FederationClientInterceptor
LOG.info("Application " + applicationId
+ " already submitted on SubCluster " + subClusterId);
} else {
+ routerMetrics.incrAppsFailedSubmitted();
RouterServerUtil.logAndThrowException(message, e);
}
}
@@ -388,6 +406,8 @@ public class FederationClientInterceptor
LOG.info("Application "
+ request.getApplicationSubmissionContext().getApplicationName()
+ " with appId " + applicationId + " submitted on " + subClusterId);
+ long stopTime = clock.getTime();
+ routerMetrics.succeededAppsSubmitted(stopTime - startTime);
return response;
} else {
// Empty response from the ResourceManager.
@@ -396,6 +416,7 @@ public class FederationClientInterceptor
}
}
+ routerMetrics.incrAppsFailedSubmitted();
String errMsg = "Application "
+ request.getApplicationSubmissionContext().getApplicationName()
+ " with appId " + applicationId + " failed to be submitted.";
@@ -423,7 +444,10 @@ public class FederationClientInterceptor
public KillApplicationResponse forceKillApplication(
KillApplicationRequest request) throws YarnException, IOException {
+ long startTime = clock.getTime();
+
if (request == null || request.getApplicationId() == null) {
+ routerMetrics.incrAppsFailedKilled();
RouterServerUtil.logAndThrowException(
"Missing forceKillApplication request or ApplicationId.", null);
}
@@ -434,6 +458,7 @@ public class FederationClientInterceptor
subClusterId = federationFacade
.getApplicationHomeSubCluster(request.getApplicationId());
} catch (YarnException e) {
+ routerMetrics.incrAppsFailedKilled();
RouterServerUtil.logAndThrowException("Application " + applicationId
+ " does not exist in FederationStateStore", e);
}
@@ -447,6 +472,7 @@ public class FederationClientInterceptor
+ subClusterId);
response = clientRMProxy.forceKillApplication(request);
} catch (Exception e) {
+ routerMetrics.incrAppsFailedKilled();
LOG.error("Unable to kill the application report for "
+ request.getApplicationId() + "to SubCluster "
+ subClusterId.getId(), e);
@@ -458,6 +484,8 @@ public class FederationClientInterceptor
+ applicationId + " to SubCluster " + subClusterId.getId());
}
+ long stopTime = clock.getTime();
+ routerMetrics.succeededAppsKilled(stopTime - startTime);
return response;
}
@@ -481,7 +509,10 @@ public class FederationClientInterceptor
public GetApplicationReportResponse getApplicationReport(
GetApplicationReportRequest request) throws YarnException, IOException {
+ long startTime = clock.getTime();
+
if (request == null || request.getApplicationId() == null) {
+ routerMetrics.incrAppsFailedRetrieved();
RouterServerUtil.logAndThrowException(
"Missing getApplicationReport request or applicationId information.",
null);
@@ -493,6 +524,7 @@ public class FederationClientInterceptor
subClusterId = federationFacade
.getApplicationHomeSubCluster(request.getApplicationId());
} catch (YarnException e) {
+ routerMetrics.incrAppsFailedRetrieved();
RouterServerUtil
.logAndThrowException("Application " + request.getApplicationId()
+ " does not exist in FederationStateStore", e);
@@ -505,6 +537,7 @@ public class FederationClientInterceptor
try {
response = clientRMProxy.getApplicationReport(request);
} catch (Exception e) {
+ routerMetrics.incrAppsFailedRetrieved();
LOG.error("Unable to get the application report for "
+ request.getApplicationId() + "to SubCluster "
+ subClusterId.getId(), e);
@@ -517,6 +550,8 @@ public class FederationClientInterceptor
+ subClusterId.getId());
}
+ long stopTime = clock.getTime();
+ routerMetrics.succeededAppsRetrieved(stopTime - startTime);
return response;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ae8fb13b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/FederationInterceptorREST.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/FederationInterceptorREST.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/FederationInterceptorREST.java
index 8ecc19d..4c7d4b1 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/FederationInterceptorREST.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/main/java/org/apache/hadoop/yarn/server/router/webapp/FederationInterceptorREST.java
@@ -18,7 +18,19 @@
package org.apache.hadoop.yarn.server.router.webapp;
-import com.google.common.annotations.VisibleForTesting;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Random;
+import java.util.Set;
+
+import javax.servlet.http.HttpServletRequest;
+import javax.servlet.http.HttpServletResponse;
+import javax.ws.rs.core.Response;
+import javax.ws.rs.core.Response.Status;
+
import org.apache.commons.lang.NotImplementedException;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.authorize.AuthorizationException;
@@ -36,20 +48,42 @@ import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
import org.apache.hadoop.yarn.server.federation.utils.FederationStateStoreFacade;
import org.apache.hadoop.yarn.server.resourcemanager.webapp.RMWebAppUtil;
-import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.*;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ActivitiesInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppActivitiesInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppAttemptsInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppPriority;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppQueue;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppState;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppTimeoutInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppTimeoutsInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ApplicationStatisticsInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ApplicationSubmissionContextInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.AppsInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ClusterMetricsInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.DelegationToken;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.LabelsToNodesInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeLabelsInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeToLabelsEntryList;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodeToLabelsInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.NodesInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationDeleteRequestInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationSubmissionRequestInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.ReservationUpdateRequestInfo;
+import org.apache.hadoop.yarn.server.resourcemanager.webapp.dao.SchedulerTypeInfo;
+import org.apache.hadoop.yarn.server.router.RouterMetrics;
import org.apache.hadoop.yarn.server.router.RouterServerUtil;
import org.apache.hadoop.yarn.server.webapp.dao.AppAttemptInfo;
import org.apache.hadoop.yarn.server.webapp.dao.ContainerInfo;
import org.apache.hadoop.yarn.server.webapp.dao.ContainersInfo;
+import org.apache.hadoop.yarn.util.Clock;
+import org.apache.hadoop.yarn.util.MonotonicClock;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import javax.servlet.http.HttpServletRequest;
-import javax.servlet.http.HttpServletResponse;
-import javax.ws.rs.core.Response;
-import javax.ws.rs.core.Response.Status;
-import java.io.IOException;
-import java.util.*;
+import com.google.common.annotations.VisibleForTesting;
/**
* Extends the {@code AbstractRESTRequestInterceptor} class and provides an
@@ -66,6 +100,8 @@ public class FederationInterceptorREST extends AbstractRESTRequestInterceptor {
private FederationStateStoreFacade federationFacade;
private Random rand;
private RouterPolicyFacade policyFacade;
+ private RouterMetrics routerMetrics;
+ private final Clock clock = new MonotonicClock();
private Map<SubClusterId, DefaultRequestInterceptorREST> interceptors;
@@ -88,6 +124,7 @@ public class FederationInterceptorREST extends AbstractRESTRequestInterceptor {
YarnConfiguration.DEFAULT_ROUTER_CLIENTRM_SUBMIT_RETRY);
interceptors = new HashMap<SubClusterId, DefaultRequestInterceptorREST>();
+ routerMetrics = RouterMetrics.getMetrics();
}
private SubClusterId getRandomActiveSubCluster(
@@ -191,10 +228,14 @@ public class FederationInterceptorREST extends AbstractRESTRequestInterceptor {
@Override
public Response createNewApplication(HttpServletRequest hsr)
throws AuthorizationException, IOException, InterruptedException {
+
+ long startTime = clock.getTime();
+
Map<SubClusterId, SubClusterInfo> subClustersActive;
try {
subClustersActive = federationFacade.getSubClusters(true);
} catch (YarnException e) {
+ routerMetrics.incrAppsFailedCreated();
return Response.status(Status.INTERNAL_SERVER_ERROR)
.entity(e.getLocalizedMessage()).build();
}
@@ -207,6 +248,7 @@ public class FederationInterceptorREST extends AbstractRESTRequestInterceptor {
try {
subClusterId = getRandomActiveSubCluster(subClustersActive, blacklist);
} catch (YarnException e) {
+ routerMetrics.incrAppsFailedCreated();
return Response.status(Status.SERVICE_UNAVAILABLE)
.entity(e.getLocalizedMessage()).build();
}
@@ -226,6 +268,10 @@ public class FederationInterceptorREST extends AbstractRESTRequestInterceptor {
}
if (response != null && response.getStatus() == 200) {
+
+ long stopTime = clock.getTime();
+ routerMetrics.succeededAppsCreated(stopTime - startTime);
+
return response;
} else {
// Empty response from the ResourceManager.
@@ -236,6 +282,7 @@ public class FederationInterceptorREST extends AbstractRESTRequestInterceptor {
String errMsg = "Fail to create a new application.";
LOG.error(errMsg);
+ routerMetrics.incrAppsFailedCreated();
return Response.status(Status.INTERNAL_SERVER_ERROR).entity(errMsg).build();
}
@@ -308,7 +355,11 @@ public class FederationInterceptorREST extends AbstractRESTRequestInterceptor {
public Response submitApplication(ApplicationSubmissionContextInfo newApp,
HttpServletRequest hsr)
throws AuthorizationException, IOException, InterruptedException {
+
+ long startTime = clock.getTime();
+
if (newApp == null || newApp.getApplicationId() == null) {
+ routerMetrics.incrAppsFailedSubmitted();
String errMsg = "Missing ApplicationSubmissionContextInfo or "
+ "applicationSubmissionContex information.";
return Response.status(Status.BAD_REQUEST).entity(errMsg).build();
@@ -318,6 +369,7 @@ public class FederationInterceptorREST extends AbstractRESTRequestInterceptor {
try {
applicationId = ApplicationId.fromString(newApp.getApplicationId());
} catch (IllegalArgumentException e) {
+ routerMetrics.incrAppsFailedSubmitted();
return Response.status(Status.BAD_REQUEST).entity(e.getLocalizedMessage())
.build();
}
@@ -333,6 +385,7 @@ public class FederationInterceptorREST extends AbstractRESTRequestInterceptor {
try {
subClusterId = policyFacade.getHomeSubcluster(context, blacklist);
} catch (YarnException e) {
+ routerMetrics.incrAppsFailedSubmitted();
return Response.status(Status.SERVICE_UNAVAILABLE)
.entity(e.getLocalizedMessage()).build();
}
@@ -349,6 +402,7 @@ public class FederationInterceptorREST extends AbstractRESTRequestInterceptor {
subClusterId =
federationFacade.addApplicationHomeSubCluster(appHomeSubCluster);
} catch (YarnException e) {
+ routerMetrics.incrAppsFailedSubmitted();
String errMsg = "Unable to insert the ApplicationId " + applicationId
+ " into the FederationStateStore";
return Response.status(Status.SERVICE_UNAVAILABLE)
@@ -367,6 +421,7 @@ public class FederationInterceptorREST extends AbstractRESTRequestInterceptor {
subClusterIdInStateStore =
federationFacade.getApplicationHomeSubCluster(applicationId);
} catch (YarnException e1) {
+ routerMetrics.incrAppsFailedSubmitted();
return Response.status(Status.SERVICE_UNAVAILABLE)
.entity(e1.getLocalizedMessage()).build();
}
@@ -374,6 +429,7 @@ public class FederationInterceptorREST extends AbstractRESTRequestInterceptor {
LOG.info("Application " + applicationId
+ " already submitted on SubCluster " + subClusterId);
} else {
+ routerMetrics.incrAppsFailedSubmitted();
return Response.status(Status.SERVICE_UNAVAILABLE).entity(errMsg)
.build();
}
@@ -384,6 +440,7 @@ public class FederationInterceptorREST extends AbstractRESTRequestInterceptor {
try {
subClusterInfo = federationFacade.getSubCluster(subClusterId);
} catch (YarnException e) {
+ routerMetrics.incrAppsFailedSubmitted();
return Response.status(Status.SERVICE_UNAVAILABLE)
.entity(e.getLocalizedMessage()).build();
}
@@ -401,6 +458,10 @@ public class FederationInterceptorREST extends AbstractRESTRequestInterceptor {
if (response != null && response.getStatus() == 202) {
LOG.info("Application " + context.getApplicationName() + " with appId "
+ applicationId + " submitted on " + subClusterId);
+
+ long stopTime = clock.getTime();
+ routerMetrics.succeededAppsSubmitted(stopTime - startTime);
+
return response;
} else {
// Empty response from the ResourceManager.
@@ -409,6 +470,7 @@ public class FederationInterceptorREST extends AbstractRESTRequestInterceptor {
}
}
+ routerMetrics.incrAppsFailedSubmitted();
String errMsg = "Application " + newApp.getApplicationName()
+ " with appId " + applicationId + " failed to be submitted.";
LOG.error(errMsg);
@@ -435,10 +497,13 @@ public class FederationInterceptorREST extends AbstractRESTRequestInterceptor {
public AppInfo getApp(HttpServletRequest hsr, String appId,
Set<String> unselectedFields) {
+ long startTime = clock.getTime();
+
ApplicationId applicationId = null;
try {
applicationId = ApplicationId.fromString(appId);
} catch (IllegalArgumentException e) {
+ routerMetrics.incrAppsFailedRetrieved();
return null;
}
@@ -448,16 +513,23 @@ public class FederationInterceptorREST extends AbstractRESTRequestInterceptor {
subClusterId =
federationFacade.getApplicationHomeSubCluster(applicationId);
if (subClusterId == null) {
+ routerMetrics.incrAppsFailedRetrieved();
return null;
}
subClusterInfo = federationFacade.getSubCluster(subClusterId);
} catch (YarnException e) {
+ routerMetrics.incrAppsFailedRetrieved();
return null;
}
- return getOrCreateInterceptorForSubCluster(subClusterId,
+ AppInfo response = getOrCreateInterceptorForSubCluster(subClusterId,
subClusterInfo.getRMWebServiceAddress()).getApp(hsr, appId,
unselectedFields);
+
+ long stopTime = clock.getTime();
+ routerMetrics.succeededAppsRetrieved(stopTime - startTime);
+
+ return response;
}
/**
@@ -481,23 +553,37 @@ public class FederationInterceptorREST extends AbstractRESTRequestInterceptor {
String appId) throws AuthorizationException, YarnException,
InterruptedException, IOException {
+ long startTime = clock.getTime();
+
ApplicationId applicationId = null;
try {
applicationId = ApplicationId.fromString(appId);
} catch (IllegalArgumentException e) {
+ routerMetrics.incrAppsFailedKilled();
return Response.status(Status.BAD_REQUEST).entity(e.getLocalizedMessage())
.build();
}
- SubClusterId subClusterId =
- federationFacade.getApplicationHomeSubCluster(applicationId);
-
- SubClusterInfo subClusterInfo =
- federationFacade.getSubCluster(subClusterId);
+ SubClusterInfo subClusterInfo = null;
+ SubClusterId subClusterId = null;
+ try {
+ subClusterId =
+ federationFacade.getApplicationHomeSubCluster(applicationId);
+ subClusterInfo = federationFacade.getSubCluster(subClusterId);
+ } catch (YarnException e) {
+ routerMetrics.incrAppsFailedKilled();
+ return Response.status(Status.BAD_REQUEST).entity(e.getLocalizedMessage())
+ .build();
+ }
- return getOrCreateInterceptorForSubCluster(subClusterId,
+ Response response = getOrCreateInterceptorForSubCluster(subClusterId,
subClusterInfo.getRMWebServiceAddress()).updateAppState(targetState,
hsr, appId);
+
+ long stopTime = clock.getTime();
+ routerMetrics.succeededAppsRetrieved(stopTime - startTime);
+
+ return response;
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ae8fb13b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/TestRouterMetrics.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/TestRouterMetrics.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/TestRouterMetrics.java
new file mode 100644
index 0000000..3cdafd8
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/TestRouterMetrics.java
@@ -0,0 +1,248 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.yarn.server.router;
+
+import org.junit.Assert;
+import org.junit.BeforeClass;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * This class validates the correctness of Router Federation Interceptor
+ * Metrics.
+ */
+public class TestRouterMetrics {
+ public static final Logger LOG =
+ LoggerFactory.getLogger(TestRouterMetrics.class);
+
+ // All the operations in the bad subcluster failed.
+ private MockBadSubCluster badSubCluster = new MockBadSubCluster();
+ // All the operations in the bad subcluster succeed.
+ private MockGoodSubCluster goodSubCluster = new MockGoodSubCluster();
+
+ private static RouterMetrics metrics = RouterMetrics.getMetrics();
+
+ @BeforeClass
+ public static void init() {
+
+ LOG.info("Test: aggregate metrics are initialized correctly");
+
+ Assert.assertEquals(0, metrics.getNumSucceededAppsCreated());
+ Assert.assertEquals(0, metrics.getNumSucceededAppsSubmitted());
+ Assert.assertEquals(0, metrics.getNumSucceededAppsKilled());
+ Assert.assertEquals(0, metrics.getNumSucceededAppsRetrieved());
+
+ Assert.assertEquals(0, metrics.getAppsFailedCreated());
+ Assert.assertEquals(0, metrics.getAppsFailedSubmitted());
+ Assert.assertEquals(0, metrics.getAppsFailedKilled());
+ Assert.assertEquals(0, metrics.getAppsFailedRetrieved());
+
+ LOG.info("Test: aggregate metrics are updated correctly");
+ }
+
+ /**
+ * This test validates the correctness of the metric: Created Apps
+ * successfully.
+ */
+ @Test
+ public void testSucceededAppsCreated() {
+
+ long totalGoodBefore = metrics.getNumSucceededAppsCreated();
+
+ goodSubCluster.getNewApplication(100);
+
+ Assert.assertEquals(totalGoodBefore + 1,
+ metrics.getNumSucceededAppsCreated());
+ Assert.assertEquals(100, metrics.getLatencySucceededAppsCreated(), 0);
+
+ goodSubCluster.getNewApplication(200);
+
+ Assert.assertEquals(totalGoodBefore + 2,
+ metrics.getNumSucceededAppsCreated());
+ Assert.assertEquals(150, metrics.getLatencySucceededAppsCreated(), 0);
+ }
+
+ /**
+ * This test validates the correctness of the metric: Failed to create Apps.
+ */
+ @Test
+ public void testAppsFailedCreated() {
+
+ long totalBadbefore = metrics.getAppsFailedCreated();
+
+ badSubCluster.getNewApplication();
+
+ Assert.assertEquals(totalBadbefore + 1, metrics.getAppsFailedCreated());
+ }
+
+ /**
+ * This test validates the correctness of the metric: Submitted Apps
+ * successfully.
+ */
+ @Test
+ public void testSucceededAppsSubmitted() {
+
+ long totalGoodBefore = metrics.getNumSucceededAppsSubmitted();
+
+ goodSubCluster.submitApplication(100);
+
+ Assert.assertEquals(totalGoodBefore + 1,
+ metrics.getNumSucceededAppsSubmitted());
+ Assert.assertEquals(100, metrics.getLatencySucceededAppsSubmitted(), 0);
+
+ goodSubCluster.submitApplication(200);
+
+ Assert.assertEquals(totalGoodBefore + 2,
+ metrics.getNumSucceededAppsSubmitted());
+ Assert.assertEquals(150, metrics.getLatencySucceededAppsSubmitted(), 0);
+ }
+
+ /**
+ * This test validates the correctness of the metric: Failed to submit Apps.
+ */
+ @Test
+ public void testAppsFailedSubmitted() {
+
+ long totalBadbefore = metrics.getAppsFailedSubmitted();
+
+ badSubCluster.submitApplication();
+
+ Assert.assertEquals(totalBadbefore + 1, metrics.getAppsFailedSubmitted());
+ }
+
+ /**
+ * This test validates the correctness of the metric: Killed Apps
+ * successfully.
+ */
+ @Test
+ public void testSucceededAppsKilled() {
+
+ long totalGoodBefore = metrics.getNumSucceededAppsKilled();
+
+ goodSubCluster.forceKillApplication(100);
+
+ Assert.assertEquals(totalGoodBefore + 1,
+ metrics.getNumSucceededAppsKilled());
+ Assert.assertEquals(100, metrics.getLatencySucceededAppsKilled(), 0);
+
+ goodSubCluster.forceKillApplication(200);
+
+ Assert.assertEquals(totalGoodBefore + 2,
+ metrics.getNumSucceededAppsKilled());
+ Assert.assertEquals(150, metrics.getLatencySucceededAppsKilled(), 0);
+ }
+
+ /**
+ * This test validates the correctness of the metric: Failed to kill Apps.
+ */
+ @Test
+ public void testAppsFailedKilled() {
+
+ long totalBadbefore = metrics.getAppsFailedKilled();
+
+ badSubCluster.forceKillApplication();
+
+ Assert.assertEquals(totalBadbefore + 1, metrics.getAppsFailedKilled());
+ }
+
+ /**
+ * This test validates the correctness of the metric: Retrieved Apps
+ * successfully.
+ */
+ @Test
+ public void testSucceededAppsReport() {
+
+ long totalGoodBefore = metrics.getNumSucceededAppsRetrieved();
+
+ goodSubCluster.getApplicationReport(100);
+
+ Assert.assertEquals(totalGoodBefore + 1,
+ metrics.getNumSucceededAppsRetrieved());
+ Assert.assertEquals(100, metrics.getLatencySucceededGetAppReport(), 0);
+
+ goodSubCluster.getApplicationReport(200);
+
+ Assert.assertEquals(totalGoodBefore + 2,
+ metrics.getNumSucceededAppsRetrieved());
+ Assert.assertEquals(150, metrics.getLatencySucceededGetAppReport(), 0);
+ }
+
+ /**
+ * This test validates the correctness of the metric: Failed to retrieve Apps.
+ */
+ @Test
+ public void testAppsReportFailed() {
+
+ long totalBadbefore = metrics.getAppsFailedRetrieved();
+
+ badSubCluster.getApplicationReport();
+
+ Assert.assertEquals(totalBadbefore + 1, metrics.getAppsFailedRetrieved());
+ }
+
+ // Records failures for all calls
+ private class MockBadSubCluster {
+ public void getNewApplication() {
+ LOG.info("Mocked: failed getNewApplication call");
+ metrics.incrAppsFailedCreated();
+ }
+
+ public void submitApplication() {
+ LOG.info("Mocked: failed submitApplication call");
+ metrics.incrAppsFailedSubmitted();
+ }
+
+ public void forceKillApplication() {
+ LOG.info("Mocked: failed forceKillApplication call");
+ metrics.incrAppsFailedKilled();
+ }
+
+ public void getApplicationReport() {
+ LOG.info("Mocked: failed getApplicationReport call");
+ metrics.incrAppsFailedRetrieved();
+ }
+ }
+
+ // Records successes for all calls
+ private class MockGoodSubCluster {
+ public void getNewApplication(long duration) {
+ LOG.info("Mocked: successful getNewApplication call with duration {}",
+ duration);
+ metrics.succeededAppsCreated(duration);
+ }
+
+ public void submitApplication(long duration) {
+ LOG.info("Mocked: successful submitApplication call with duration {}",
+ duration);
+ metrics.succeededAppsSubmitted(duration);
+ }
+
+ public void forceKillApplication(long duration) {
+ LOG.info("Mocked: successful forceKillApplication call with duration {}",
+ duration);
+ metrics.succeededAppsKilled(duration);
+ }
+
+ public void getApplicationReport(long duration) {
+ LOG.info("Mocked: successful getApplicationReport call with duration {}",
+ duration);
+ metrics.succeededAppsRetrieved(duration);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/ae8fb13b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestFederationInterceptorREST.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestFederationInterceptorREST.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestFederationInterceptorREST.java
index d918149..fb6cdd8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestFederationInterceptorREST.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router/src/test/java/org/apache/hadoop/yarn/server/router/webapp/TestFederationInterceptorREST.java
@@ -276,13 +276,11 @@ public class TestFederationInterceptorREST extends BaseRouterWebServicesTest {
ApplicationId appId =
ApplicationId.newInstance(System.currentTimeMillis(), 1);
AppState appState = new AppState("KILLED");
- try {
- interceptor.updateAppState(appState, null, appId.toString());
- Assert.fail();
- } catch (YarnException e) {
- Assert.assertTrue(
- e.getMessage().equals("Application " + appId + " does not exist"));
- }
+
+ Response response =
+ interceptor.updateAppState(appState, null, appId.toString());
+ Assert.assertEquals(BAD_REQUEST, response.getStatus());
+
}
/**
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[45/50] [abbrv] hadoop git commit: HDFS-10899. Add functionality to
re-encrypt EDEKs.
Posted by as...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1000a2af/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CryptoAdmin.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CryptoAdmin.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CryptoAdmin.java
index 14abf6e..4b0b083 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CryptoAdmin.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/CryptoAdmin.java
@@ -32,8 +32,11 @@ import org.apache.hadoop.fs.RemoteIterator;
import org.apache.hadoop.hdfs.client.CreateEncryptionZoneFlag;
import org.apache.hadoop.hdfs.client.HdfsAdmin;
import org.apache.hadoop.hdfs.protocol.EncryptionZone;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants.ReencryptAction;
+import org.apache.hadoop.hdfs.protocol.ZoneReencryptionStatus;
import org.apache.hadoop.tools.TableListing;
import org.apache.hadoop.util.StringUtils;
+import org.apache.hadoop.util.Time;
import org.apache.hadoop.util.Tool;
import org.apache.hadoop.util.ToolRunner;
@@ -286,10 +289,139 @@ public class CryptoAdmin extends Configured implements Tool {
}
}
+ private static class ReencryptZoneCommand implements AdminHelper.Command {
+ @Override
+ public String getName() {
+ return "-reencryptZone";
+ }
+
+ @Override
+ public String getShortUsage() {
+ return "[" + getName() + " <action> -path <zone>]\n";
+ }
+
+ @Override
+ public String getLongUsage() {
+ final TableListing listing = AdminHelper.getOptionDescriptionListing();
+ listing.addRow("<action>",
+ "The re-encrypt action to perform. Must be -start or -cancel.");
+ listing.addRow("<zone>", "The path to the zone to be re-encrypted.");
+ return getShortUsage() + "\n" + "Issue a re-encryption command for"
+ + " an encryption zone. Requires superuser permissions.\n\n"
+ + listing.toString();
+ }
+
+ @Override
+ public int run(Configuration conf, List<String> args) throws IOException {
+ final String path = StringUtils.popOptionWithArgument("-path", args);
+ final boolean start = StringUtils.popOption("-start", args);
+ final boolean cancel = StringUtils.popOption("-cancel", args);
+
+ if (!args.isEmpty()) {
+ System.err.println("Can't understand argument: " + args.get(0));
+ getLongUsage();
+ return 1;
+ }
+ if (!(start ^ cancel)) {
+ System.err.println("You must specify either [-start] or [-cancel]. ");
+ getLongUsage();
+ return 2;
+ }
+ if (path == null) {
+ System.err.println("You must specify a zone directory with [-path]. ");
+ getLongUsage();
+ return 3;
+ }
+ ReencryptAction action = ReencryptAction.START;
+ if (cancel) {
+ action = ReencryptAction.CANCEL;
+ }
+
+ final HdfsAdmin admin =
+ new HdfsAdmin(FileSystem.getDefaultUri(conf), conf);
+ try {
+ admin.reencryptEncryptionZone(new Path(path), action);
+ System.out.println("re-encrypt command successfully submitted for "
+ + "zone: " + path + " action: " + action);
+ } catch (IOException e) {
+ System.err.println(prettifyException(e));
+ return 4;
+ }
+ return 0;
+ }
+ }
+
+ private static class ListReencryptionStatusCommand
+ implements AdminHelper.Command {
+ @Override
+ public String getName() {
+ return "-listReencryptionStatus";
+ }
+
+ @Override
+ public String getShortUsage() {
+ return "[" + getName()+ "]\n";
+ }
+
+ @Override
+ public String getLongUsage() {
+ return getShortUsage() + "\n" +
+ "List re-encryption statuses of encryption zones. "
+ + "Requires superuser permissions.\n\n";
+ }
+
+ @Override
+ public int run(Configuration conf, List<String> args) throws IOException {
+ HdfsAdmin admin = new HdfsAdmin(FileSystem.getDefaultUri(conf), conf);
+ try {
+ final TableListing listing =
+ new TableListing.Builder().addField("Zone Name").addField("Status")
+ .addField("EZKey Version Name").addField("Submission Time")
+ .addField("Is Canceled?").addField("Completion Time")
+ .addField("Number of files re-encrypted")
+ .addField("Number of failures")
+ .addField("Last File Checkpointed")
+ .wrapWidth(AdminHelper.MAX_LINE_WIDTH).showHeaders().build();
+ final RemoteIterator<ZoneReencryptionStatus> it =
+ admin.listReencryptionStatus();
+ boolean failuresMet = false;
+ while (it.hasNext()) {
+ ZoneReencryptionStatus zs = it.next();
+ final long completion = zs.getCompletionTime();
+ listing.addRow(zs.getZoneName(), zs.getState().toString(),
+ zs.getEzKeyVersionName(), Time.formatTime(zs.getSubmissionTime()),
+ Boolean.toString(zs.isCanceled()),
+ completion == 0 ? "N/A" : Time.formatTime(completion),
+ Long.toString(zs.getFilesReencrypted()),
+ Long.toString(zs.getNumReencryptionFailures()),
+ zs.getLastCheckpointFile());
+ if (zs.getNumReencryptionFailures() > 0) {
+ failuresMet = true;
+ }
+ }
+ System.out.println(listing.toString());
+ if (failuresMet) {
+ System.out.println("There are re-encryption failures. Files that are"
+ + " failed to re-encrypt are still using the old EDEKs. "
+ + "Please check NameNode log to see which files failed,"
+ + " then either fix the error and re-encrypt again,"
+ + " or manually copy the failed files to use new EDEKs.");
+ }
+ } catch (IOException e) {
+ System.err.println(prettifyException(e));
+ return 2;
+ }
+
+ return 0;
+ }
+ }
+
private static final AdminHelper.Command[] COMMANDS = {
new CreateZoneCommand(),
new ListZonesCommand(),
new ProvisionTrashCommand(),
- new GetFileEncryptionInfoCommand()
+ new GetFileEncryptionInfoCommand(),
+ new ReencryptZoneCommand(),
+ new ListReencryptionStatusCommand()
};
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1000a2af/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
index 03becc9..aedc7e8 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/resources/hdfs-default.xml
@@ -2800,6 +2800,15 @@
</description>
</property>
+<property>
+ <name>dfs.namenode.list.reencryption.status.num.responses</name>
+ <value>100</value>
+ <description>When listing re-encryption status, the maximum number of zones
+ that will be returned in a batch. Fetching the list incrementally in
+ batches improves namenode performance.
+ </description>
+</property>
+
<property>
<name>dfs.namenode.list.openfiles.num.responses</name>
<value>1000</value>
@@ -2829,6 +2838,49 @@
</property>
<property>
+ <name>dfs.namenode.reencrypt.sleep.interval</name>
+ <value>1m</value>
+ <description>Interval the re-encrypt EDEK thread sleeps in the main loop. The
+ interval accepts units. If none given, millisecond is assumed.
+ </description>
+</property>
+
+<property>
+ <name>dfs.namenode.reencrypt.batch.size</name>
+ <value>1000</value>
+ <description>How many EDEKs should the re-encrypt thread process in one batch.
+ </description>
+</property>
+
+<property>
+ <name>dfs.namenode.reencrypt.throttle.limit.handler.ratio</name>
+ <value>1.0</value>
+ <description>Throttling ratio for the re-encryption, indicating what fraction
+ of time should the re-encrypt handler thread work under NN read lock.
+ Larger than 1.0 values are interpreted as 1.0. Negative value or 0 are
+ invalid values and will fail NN startup.
+ </description>
+</property>
+
+<property>
+ <name>dfs.namenode.reencrypt.throttle.limit.updater.ratio</name>
+ <value>1.0</value>
+ <description>Throttling ratio for the re-encryption, indicating what fraction
+ of time should the re-encrypt updater thread work under NN write lock.
+ Larger than 1.0 values are interpreted as 1.0. Negative value or 0 are
+ invalid values and will fail NN startup.
+ </description>
+</property>
+
+<property>
+ <name>dfs.namenode.reencrypt.edek.threads</name>
+ <value>10</value>
+ <description>Maximum number of re-encrypt threads to contact the KMS
+ and re-encrypt the edeks.
+ </description>
+</property>
+
+<property>
<name>dfs.namenode.inotify.max.events.per.rpc</name>
<value>1000</value>
<description>Maximum number of events that will be sent to an inotify client
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1000a2af/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/TransparentEncryption.md
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/TransparentEncryption.md b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/TransparentEncryption.md
index 3f9fbf0..3454265 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/TransparentEncryption.md
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/site/markdown/TransparentEncryption.md
@@ -177,6 +177,33 @@ Get encryption information from a file. This can be used to find out whether a f
|:---- |:---- |
| *path* | The path of the file to get encryption information. |
+### <a name="reencryptZone"></a>reencryptZone
+
+Usage: `[-reencryptZone <action> -path <zone>]`
+
+Re-encrypts an encryption zone, by iterating through the encryption zone, and calling the KeyProvider's reencryptEncryptedKeys interface to batch-re-encrypt all files' EDEKs with the latest version encryption zone key in the key provider. Requires superuser permissions.
+
+Note that re-encryption does not apply to snapshots, due to snapshots' immutable nature.
+
+| | |
+|:---- |:---- |
+| *action* | The re-encrypt action to perform. Must be either `-start` or `-cancel`. |
+| *path* | The path to the root of the encryption zone. |
+
+Re-encryption is a NameNode-only operation in HDFS, so could potentially put intensive load to the NameNode. The following configurations can be changed to control the stress on the NameNode, depending on the acceptable throughput impact to the cluster.
+
+| | |
+|:---- |:---- |
+| *dfs.namenode.reencrypt.batch.size* | The number of EDEKs in a batch to be sent to the KMS for re-encryption. Each batch is processed when holding the name system read/write lock, with throttling happening between batches. See configs below. |
+| *dfs.namenode.reencrypt.throttle.limit.handler.ratio* | Ratio of read locks to be held during re-encryption. 1.0 means no throttling. 0.5 means re-encryption can hold the readlock at most 50% of its total processing time. Negative value or 0 are invalid. |
+| *dfs.namenode.reencrypt.throttle.limit.updater.ratio* | Ratio of write locks to be held during re-encryption. 1.0 means no throttling. 0.5 means re-encryption can hold the writelock at most 50% of its total processing time. Negative value or 0 are invalid. |
+
+### <a name="listReencryptionStatus"></a>listReencryptionStatus
+
+Usage: `[-listReencryptionStatus]`
+
+List re-encryption information for all encryption zones. Requires superuser permissions.
+
<a name="Example_usage"></a>Example usage
-------------
@@ -282,4 +309,20 @@ These exploits assume that the attacker has compromised HDFS, but does not have
### <a name="Rogue_user_exploits"></a>Rogue user exploits
-A rogue user can collect keys of files they have access to, and use them later to decrypt the encrypted data of those files. As the user had access to those files, they already had access to the file contents. This can be mitigated through periodic key rolling policies.
+A rogue user can collect keys of files they have access to, and use them later to decrypt the encrypted data of those files. As the user had access to those files, they already had access to the file contents. This can be mitigated through periodic key rolling policies. The [reencryptZone](#reencryptZone) command is usually required after key rolling, to make sure the EDEKs on existing files use the new version key.
+
+Manual steps to a complete key rolling and re-encryption are listed below. These instructions assume that you are running as the key admin or HDFS superuser as is appropriate.
+
+ # As the key admin, roll the key to a new version
+ hadoop key roll exposedKey
+
+ # As the super user, re-encrypt the encryption zone. Possibly list zones first.
+ hdfs crypto -listZones
+ hdfs crypto -reencryptZone -start -path /zone
+
+ # As the super user, periodically check the status of re-encryption
+ hdfs crypto -listReencryptionStatus
+
+ # As the super user, get encryption information from the file and double check it's encryption key version
+ hdfs crypto -getFileEncryptionInfo -path /zone/helloWorld
+ # console output: {cipherSuite: {name: AES/CTR/NoPadding, algorithmBlockSize: 16}, cryptoProtocolVersion: CryptoProtocolVersion{description='Encryption zones', version=2, unknownValue=null}, edek: 2010d301afbd43b58f10737ce4e93b39, iv: ade2293db2bab1a2e337f91361304cb3, keyName: exposedKey, ezKeyVersionName: exposedKey@1}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1000a2af/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReencryption.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReencryption.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReencryption.java
new file mode 100644
index 0000000..7ba3f91
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReencryption.java
@@ -0,0 +1,1847 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.namenode;
+
+import java.io.File;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.EnumSet;
+import java.util.HashSet;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
+
+import com.google.common.base.Supplier;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.crypto.key.JavaKeyStoreProvider;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.fs.FileContext;
+import org.apache.hadoop.fs.FileContextTestWrapper;
+import org.apache.hadoop.fs.FileEncryptionInfo;
+import org.apache.hadoop.fs.FileSystemTestHelper;
+import org.apache.hadoop.fs.FileSystemTestWrapper;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RemoteIterator;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.DFSTestUtil;
+import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.hadoop.hdfs.HdfsConfiguration;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.client.CreateEncryptionZoneFlag;
+import org.apache.hadoop.hdfs.client.HdfsAdmin;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants.ReencryptAction;
+import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
+import org.apache.hadoop.hdfs.protocol.ReencryptionStatus;
+import org.apache.hadoop.hdfs.protocol.SnapshotAccessControlException;
+import org.apache.hadoop.hdfs.protocol.ZoneReencryptionStatus;
+import org.apache.hadoop.hdfs.server.namenode.ReencryptionUpdater.ZoneSubmissionTracker;
+import org.apache.hadoop.ipc.RemoteException;
+import org.apache.hadoop.ipc.RetriableException;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+
+import static org.apache.hadoop.test.GenericTestUtils.assertExceptionContains;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import org.junit.rules.Timeout;
+import org.mockito.internal.util.reflection.Whitebox;
+import org.slf4j.LoggerFactory;
+import org.slf4j.event.Level;
+
+/**
+ * Test class for re-encryption.
+ */
+public class TestReencryption {
+
+ protected static final org.slf4j.Logger LOG =
+ LoggerFactory.getLogger(TestReencryption.class);
+
+ private Configuration conf;
+ private FileSystemTestHelper fsHelper;
+
+ private MiniDFSCluster cluster;
+ private HdfsAdmin dfsAdmin;
+ private DistributedFileSystem fs;
+ private FSNamesystem fsn;
+ private File testRootDir;
+ private static final String TEST_KEY = "test_key";
+
+ private FileSystemTestWrapper fsWrapper;
+ private FileContextTestWrapper fcWrapper;
+
+ private static final EnumSet<CreateEncryptionZoneFlag> NO_TRASH =
+ EnumSet.of(CreateEncryptionZoneFlag.NO_TRASH);
+
+ private String getKeyProviderURI() {
+ return JavaKeyStoreProvider.SCHEME_NAME + "://file" + new Path(
+ testRootDir.toString(), "test.jks").toUri();
+ }
+
+ @Rule
+ public Timeout globalTimeout = new Timeout(180 * 1000);
+
+ @Before
+ public void setup() throws Exception {
+ conf = new HdfsConfiguration();
+ fsHelper = new FileSystemTestHelper();
+ // Set up java key store
+ String testRoot = fsHelper.getTestRootDir();
+ testRootDir = new File(testRoot).getAbsoluteFile();
+ conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH,
+ getKeyProviderURI());
+ conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY,
+ true);
+ // Lower the batch size for testing
+ conf.setInt(DFSConfigKeys.DFS_NAMENODE_LIST_ENCRYPTION_ZONES_NUM_RESPONSES,
+ 2);
+ // Lower the listing limit for testing
+ conf.setInt(DFSConfigKeys.DFS_LIST_LIMIT, 3);
+ // Adjust configs for re-encrypt test cases
+ conf.setInt(DFSConfigKeys.DFS_NAMENODE_REENCRYPT_BATCH_SIZE_KEY, 5);
+ conf.setTimeDuration(
+ DFSConfigKeys.DFS_NAMENODE_REENCRYPT_SLEEP_INTERVAL_KEY, 1,
+ TimeUnit.SECONDS);
+ cluster = new MiniDFSCluster.Builder(conf).numDataNodes(1).build();
+ cluster.waitActive();
+ fs = cluster.getFileSystem();
+ fsn = cluster.getNamesystem();
+ fsWrapper = new FileSystemTestWrapper(fs);
+ fcWrapper = new FileContextTestWrapper(
+ FileContext.getFileContext(cluster.getURI(), conf));
+ dfsAdmin = new HdfsAdmin(cluster.getURI(), conf);
+ setProvider();
+ // Create a test key
+ DFSTestUtil.createKey(TEST_KEY, cluster, conf);
+ GenericTestUtils.setLogLevel(EncryptionZoneManager.LOG, Level.TRACE);
+ GenericTestUtils.setLogLevel(ReencryptionHandler.LOG, Level.TRACE);
+ GenericTestUtils.setLogLevel(ReencryptionStatus.LOG, Level.TRACE);
+ GenericTestUtils.setLogLevel(ReencryptionUpdater.LOG, Level.TRACE);
+ }
+
+ private void setProvider() {
+ // Need to set the client's KeyProvider to the NN's for JKS,
+ // else the updates do not get flushed properly
+ fs.getClient()
+ .setKeyProvider(cluster.getNameNode().getNamesystem().getProvider());
+ }
+
+ @After
+ public void teardown() {
+ if (cluster != null) {
+ cluster.shutdown();
+ cluster = null;
+ }
+ EncryptionFaultInjector.instance = new EncryptionFaultInjector();
+ }
+
+ private FileEncryptionInfo getFileEncryptionInfo(Path path) throws Exception {
+ return fsn.getFileInfo(path.toString(), false).getFileEncryptionInfo();
+ }
+
+ @Test
+ public void testReencryptionBasic() throws Exception {
+ /* Setup test dir:
+ * /zones/zone/[0-9]
+ * /dir/f
+ */
+ final int len = 8196;
+ final Path zoneParent = new Path("/zones");
+ final Path zone = new Path(zoneParent, "zone");
+ fsWrapper.mkdir(zone, FsPermission.getDirDefault(), true);
+ dfsAdmin.createEncryptionZone(zone, TEST_KEY, NO_TRASH);
+ for (int i = 0; i < 10; ++i) {
+ DFSTestUtil
+ .createFile(fs, new Path(zone, Integer.toString(i)), len, (short) 1,
+ 0xFEED);
+ }
+ final Path subdir = new Path("/dir");
+ fsWrapper.mkdir(subdir, FsPermission.getDirDefault(), true);
+ DFSTestUtil.createFile(fs, new Path(subdir, "f"), len, (short) 1, 0xFEED);
+
+ // test re-encrypt without keyroll
+ final Path encFile1 = new Path(zone, "0");
+ final FileEncryptionInfo fei0 = getFileEncryptionInfo(encFile1);
+ dfsAdmin.reencryptEncryptionZone(zone, ReencryptAction.START);
+ waitForReencryptedZones(1);
+ assertKeyVersionEquals(encFile1, fei0);
+ // key not rolled, so no edeks need to be updated.
+ verifyZoneStatus(zone, null, 0);
+
+ // test re-encrypt after keyroll
+ fsn.getProvider().rollNewVersion(TEST_KEY);
+ fsn.getProvider().flush();
+ dfsAdmin.reencryptEncryptionZone(zone, ReencryptAction.START);
+ waitForReencryptedZones(2);
+ FileEncryptionInfo fei1 = getFileEncryptionInfo(encFile1);
+ assertKeyVersionChanged(encFile1, fei0);
+
+ // test listReencryptionStatus
+ RemoteIterator<ZoneReencryptionStatus> it =
+ dfsAdmin.listReencryptionStatus();
+ assertTrue(it.hasNext());
+ ZoneReencryptionStatus zs = it.next();
+ assertEquals(zone.toString(), zs.getZoneName());
+ assertEquals(ZoneReencryptionStatus.State.Completed, zs.getState());
+ assertTrue(zs.getCompletionTime() > 0);
+ assertTrue(zs.getCompletionTime() > zs.getSubmissionTime());
+ assertNotEquals(fei0.getEzKeyVersionName(), zs.getEzKeyVersionName());
+ assertEquals(fei1.getEzKeyVersionName(), zs.getEzKeyVersionName());
+ assertEquals(10, zs.getFilesReencrypted());
+
+ // test re-encrypt on same zone again
+ dfsAdmin.reencryptEncryptionZone(zone, ReencryptAction.START);
+ waitForReencryptedZones(3);
+ assertKeyVersionEquals(encFile1, fei1);
+
+ // test non-EZ submission
+ try {
+ dfsAdmin.reencryptEncryptionZone(subdir, ReencryptAction.START);
+ fail("Re-encrypting non-EZ should fail");
+ } catch (RemoteException expected) {
+ LOG.info("Expected exception caught.", expected);
+ assertExceptionContains("not the root of an encryption zone", expected);
+ }
+
+ // test non-existing dir
+ try {
+ dfsAdmin.reencryptEncryptionZone(new Path(zone, "notexist"),
+ ReencryptAction.START);
+ fail("Re-encrypting non-existing dir should fail");
+ } catch (RemoteException expected) {
+ LOG.info("Expected exception caught.", expected);
+ assertTrue(
+ expected.unwrapRemoteException() instanceof FileNotFoundException);
+ }
+
+ // test directly on a EZ file
+ try {
+ dfsAdmin.reencryptEncryptionZone(encFile1, ReencryptAction.START);
+ fail("Re-encrypting on a file should fail");
+ } catch (RemoteException expected) {
+ LOG.info("Expected exception caught.", expected);
+ assertExceptionContains("not the root of an encryption zone", expected);
+ }
+
+ // test same command resubmission
+ getEzManager().pauseReencryptForTesting();
+ dfsAdmin.reencryptEncryptionZone(zone, ReencryptAction.START);
+ waitForQueuedZones(1);
+ try {
+ dfsAdmin.reencryptEncryptionZone(zone, ReencryptAction.START);
+ } catch (RemoteException expected) {
+ LOG.info("Expected exception caught.", expected);
+ assertExceptionContains("already submitted", expected);
+ }
+ getEzManager().resumeReencryptForTesting();
+ waitForReencryptedZones(4);
+
+ // test empty EZ
+ final Path emptyZone = new Path("/emptyZone");
+ fsWrapper.mkdir(emptyZone, FsPermission.getDirDefault(), true);
+ dfsAdmin.createEncryptionZone(emptyZone, TEST_KEY, NO_TRASH);
+
+ dfsAdmin.reencryptEncryptionZone(emptyZone, ReencryptAction.START);
+ waitForReencryptedZones(5);
+
+ dfsAdmin.reencryptEncryptionZone(emptyZone, ReencryptAction.START);
+ waitForReencryptedZones(6);
+
+ // test rename ez and listReencryptionStatus
+ final Path renamedZone = new Path("/renamedZone");
+ fsWrapper.rename(zone, renamedZone);
+ it = dfsAdmin.listReencryptionStatus();
+ assertTrue(it.hasNext());
+ zs = it.next();
+ assertEquals(renamedZone.toString(), zs.getZoneName());
+ }
+
+ @Test
+ public void testReencryptOrdering() throws Exception {
+ /* Setup dir as follows:
+ * /zones/zone/[0-3]
+ * /zones/zone/dir/f
+ * /zones/zone/f[0-4]
+ */
+ final int len = 8196;
+ final Path zoneParent = new Path("/zones");
+ final Path zone = new Path(zoneParent, "zone");
+ fsWrapper.mkdir(zone, FsPermission.getDirDefault(), true);
+ dfsAdmin.createEncryptionZone(zone, TEST_KEY, NO_TRASH);
+ Path subdir = new Path(zone, "dir");
+ fsWrapper.mkdir(subdir, FsPermission.getDirDefault(), true);
+ DFSTestUtil.createFile(fs, new Path(subdir, "f"), len, (short) 1, 0xFEED);
+ for (int i = 0; i < 4; ++i) {
+ DFSTestUtil
+ .createFile(fs, new Path(zone, Integer.toString(i)), len, (short) 1,
+ 0xFEED);
+ }
+ for (int i = 0; i < 5; ++i) {
+ DFSTestUtil.createFile(fs, new Path(zone, "f" + Integer.toString(i)), len,
+ (short) 1, 0xFEED);
+ }
+
+ // /zones/zone/f[0-4] should be re-encrypted after /zones/zone/dir/f
+ final Path lastReencryptedFile = new Path(subdir, "f");
+ final Path notReencrypted = new Path(zone, "f0");
+ final FileEncryptionInfo fei = getFileEncryptionInfo(lastReencryptedFile);
+ final FileEncryptionInfo feiLast = getFileEncryptionInfo(notReencrypted);
+ fsn.getProvider().rollNewVersion(TEST_KEY);
+ fsn.getProvider().flush();
+ // mark pause after first checkpoint (5 files)
+ getEzManager().pauseForTestingAfterNthSubmission(1);
+ dfsAdmin.reencryptEncryptionZone(zone, ReencryptAction.START);
+ waitForReencryptedFiles(zone.toString(), 5);
+ assertKeyVersionChanged(lastReencryptedFile, fei);
+ assertKeyVersionEquals(notReencrypted, feiLast);
+ }
+
+ @Test
+ public void testDeleteDuringReencrypt() throws Exception {
+ final int len = 8196;
+ final Path zoneParent = new Path("/zones");
+ final Path zone = new Path(zoneParent, "zone");
+ fsWrapper.mkdir(zone, FsPermission.getDirDefault(), true);
+ dfsAdmin.createEncryptionZone(zone, TEST_KEY, NO_TRASH);
+ for (int i = 0; i < 10; ++i) {
+ DFSTestUtil
+ .createFile(fs, new Path(zone, Integer.toString(i)), len, (short) 1,
+ 0xFEED);
+ }
+ // test zone deleted during re-encrypt
+ getEzManager().pauseReencryptForTesting();
+ getEzManager().resetMetricsForTesting();
+ dfsAdmin.reencryptEncryptionZone(zone, ReencryptAction.START);
+ waitForQueuedZones(1);
+
+ fs.delete(zone, true);
+ getEzManager().resumeReencryptForTesting();
+ waitForTotalZones(0);
+ assertNull(getZoneStatus(zone.toString()));
+ }
+
+ @Test
+ public void testZoneDeleteDuringReencrypt() throws Exception {
+ final int len = 8196;
+ final Path zoneParent = new Path("/zones");
+ final Path zone = new Path(zoneParent, "zone");
+ fsWrapper.mkdir(zone, FsPermission.getDirDefault(), true);
+ dfsAdmin.createEncryptionZone(zone, TEST_KEY, NO_TRASH);
+ for (int i = 0; i < 10; ++i) {
+ DFSTestUtil
+ .createFile(fs, new Path(zone, Integer.toString(i)), len, (short) 1,
+ 0xFEED);
+ }
+
+ fsn.getProvider().rollNewVersion(TEST_KEY);
+ fsn.getProvider().flush();
+ // test zone deleted during re-encrypt's checkpointing
+ getEzManager().pauseForTestingAfterNthSubmission(1);
+ getEzManager().resetMetricsForTesting();
+ dfsAdmin.reencryptEncryptionZone(zone, ReencryptAction.START);
+ waitForReencryptedFiles(zone.toString(), 5);
+
+ fs.delete(zoneParent, true);
+ getEzManager().resumeReencryptForTesting();
+ waitForTotalZones(0);
+ assertNull(getEzManager().getZoneStatus(zone.toString()));
+
+ // verify zone is cleared
+ RemoteIterator<ZoneReencryptionStatus> it =
+ dfsAdmin.listReencryptionStatus();
+ assertFalse(it.hasNext());
+ }
+
+ @Test
+ public void testRestartAfterReencrypt() throws Exception {
+ /* Setup dir as follows:
+ * /zones
+ * /zones/zone
+ * /zones/zone/[0-9]
+ * /zones/zone/dir
+ * /zones/zone/dir/f
+ */
+ final int len = 8196;
+ final Path zoneParent = new Path("/zones");
+ final Path zone = new Path(zoneParent, "zone");
+ fsWrapper.mkdir(zone, FsPermission.getDirDefault(), true);
+ dfsAdmin.createEncryptionZone(zone, TEST_KEY, NO_TRASH);
+ for (int i = 0; i < 10; ++i) {
+ DFSTestUtil
+ .createFile(fs, new Path(zone, Integer.toString(i)), len, (short) 1,
+ 0xFEED);
+ }
+ final Path subdir = new Path(zone, "dir");
+ fsWrapper.mkdir(subdir, FsPermission.getDirDefault(), true);
+ DFSTestUtil.createFile(fs, new Path(subdir, "f"), len, (short) 1, 0xFEED);
+
+ final Path encFile0 = new Path(zone, "0");
+ final Path encFile9 = new Path(zone, "9");
+ final FileEncryptionInfo fei0 = getFileEncryptionInfo(encFile0);
+ final FileEncryptionInfo fei9 = getFileEncryptionInfo(encFile9);
+ fsn.getProvider().rollNewVersion(TEST_KEY);
+ fsn.getProvider().flush();
+ dfsAdmin.reencryptEncryptionZone(zone, ReencryptAction.START);
+ waitForReencryptedZones(1);
+
+ assertKeyVersionChanged(encFile0, fei0);
+ assertKeyVersionChanged(encFile9, fei9);
+
+ final FileEncryptionInfo fei0new = getFileEncryptionInfo(encFile0);
+ final FileEncryptionInfo fei9new = getFileEncryptionInfo(encFile9);
+ restartClusterDisableReencrypt();
+
+ assertKeyVersionEquals(encFile0, fei0new);
+ assertKeyVersionEquals(encFile9, fei9new);
+ assertNull("Re-encrypt queue should be empty after restart",
+ getReencryptionStatus().getNextUnprocessedZone());
+ }
+
+ @Test
+ public void testRestartWithRenames() throws Exception {
+ /* Setup dir as follows:
+ * /zones
+ * /zones/zone
+ * /zones/zone/f --> renamed to f1
+ */
+ final int len = 8196;
+ final Path zoneParent = new Path("/zones");
+ final Path zone = new Path(zoneParent, "zone");
+ fsWrapper.mkdir(zone, FsPermission.getDirDefault(), true);
+ dfsAdmin.createEncryptionZone(zone, TEST_KEY, NO_TRASH);
+ DFSTestUtil.createFile(fs, new Path(zone, "f"), len, (short) 1, 0xFEED);
+ fsWrapper.rename(new Path(zone, "f"), new Path(zone, "f1"));
+
+ // re-encrypt
+ fsn.getProvider().rollNewVersion(TEST_KEY);
+ fsn.getProvider().flush();
+ dfsAdmin.reencryptEncryptionZone(zone, ReencryptAction.START);
+ waitForReencryptedZones(1);
+
+ // make sure NN can successfully restart (rename can load ok with
+ // re-encrypt since they're in correct order)
+ cluster.restartNameNodes();
+ cluster.waitActive();
+
+ waitForReencryptedZones(1);
+ }
+
+ @Test
+ public void testRestartDuringReencrypt() throws Exception {
+ /* Setup dir as follows:
+ * /zones
+ * /zones/zone
+ * /zones/zone/dir_empty
+ * /zones/zone/dir1/[0-9]
+ * /zones/zone/dir1/dir_empty1
+ * /zones/zone/dir2
+ * /zones/zone/dir2/dir_empty2
+ * /zones/zone/dir2/f
+ */
+ final int len = 8196;
+ final Path zoneParent = new Path("/zones");
+ final Path zone = new Path(zoneParent, "zone");
+ fsWrapper.mkdir(zone, FsPermission.getDirDefault(), true);
+ dfsAdmin.createEncryptionZone(zone, TEST_KEY, NO_TRASH);
+ fsWrapper
+ .mkdir(new Path(zone, "dir_empty"), FsPermission.getDirDefault(), true);
+ Path subdir = new Path(zone, "dir2");
+ fsWrapper.mkdir(subdir, FsPermission.getDirDefault(), true);
+ fsWrapper
+ .mkdir(new Path(subdir, "dir_empty2"), FsPermission.getDirDefault(),
+ true);
+ DFSTestUtil.createFile(fs, new Path(subdir, "f"), len, (short) 1, 0xFEED);
+ subdir = new Path(zone, "dir1");
+ for (int i = 0; i < 10; ++i) {
+ DFSTestUtil
+ .createFile(fs, new Path(subdir, Integer.toString(i)), len, (short) 1,
+ 0xFEED);
+ }
+ fsWrapper
+ .mkdir(new Path(subdir, "dir_empty1"), FsPermission.getDirDefault(),
+ true);
+
+ final Path encFile0 = new Path(subdir, "0");
+ final Path encFile9 = new Path(subdir, "9");
+ final FileEncryptionInfo fei0 = getFileEncryptionInfo(encFile0);
+ final FileEncryptionInfo fei9 = getFileEncryptionInfo(encFile9);
+ fsn.getProvider().rollNewVersion(TEST_KEY);
+ fsn.getProvider().flush();
+ // mark pause after first checkpoint (5 files)
+ getEzManager().pauseForTestingAfterNthSubmission(1);
+ dfsAdmin.reencryptEncryptionZone(zone, ReencryptAction.START);
+ waitForReencryptedFiles(zone.toString(), 5);
+
+ restartClusterDisableReencrypt();
+
+ final Long zoneId = fsn.getFSDirectory().getINode(zone.toString()).getId();
+ assertEquals("Re-encrypt should restore to the last checkpoint zone",
+ zoneId, getReencryptionStatus().getNextUnprocessedZone());
+ assertEquals("Re-encrypt should restore to the last checkpoint file",
+ new Path(subdir, "4").toString(),
+ getEzManager().getZoneStatus(zone.toString()).getLastCheckpointFile());
+
+ getEzManager().resumeReencryptForTesting();
+ waitForReencryptedZones(1);
+ assertKeyVersionChanged(encFile0, fei0);
+ assertKeyVersionChanged(encFile9, fei9);
+ assertNull("Re-encrypt queue should be empty after restart",
+ getReencryptionStatus().getNextUnprocessedZone());
+ assertEquals(11, getZoneStatus(zone.toString()).getFilesReencrypted());
+ }
+
+ @Test
+ public void testRestartAfterReencryptAndCheckpoint() throws Exception {
+ final int len = 8196;
+ final Path zoneParent = new Path("/zones");
+ final Path zone = new Path(zoneParent, "zone");
+ fsWrapper.mkdir(zone, FsPermission.getDirDefault(), true);
+ dfsAdmin.createEncryptionZone(zone, TEST_KEY, NO_TRASH);
+ for (int i = 0; i < 10; ++i) {
+ DFSTestUtil
+ .createFile(fs, new Path(zone, Integer.toString(i)), len, (short) 1,
+ 0xFEED);
+ }
+ final Path subdir = new Path(zone, "dir");
+ fsWrapper.mkdir(subdir, FsPermission.getDirDefault(), true);
+ DFSTestUtil.createFile(fs, new Path(subdir, "f"), len, (short) 1, 0xFEED);
+
+ final Path encFile0 = new Path(zone, "0");
+ final Path encFile9 = new Path(zone, "9");
+ final FileEncryptionInfo fei0 = getFileEncryptionInfo(encFile0);
+ final FileEncryptionInfo fei9 = getFileEncryptionInfo(encFile9);
+ fsn.getProvider().rollNewVersion(TEST_KEY);
+ fsn.getProvider().flush();
+ dfsAdmin.reencryptEncryptionZone(zone, ReencryptAction.START);
+ waitForReencryptedZones(1);
+
+ assertKeyVersionChanged(encFile0, fei0);
+ assertKeyVersionChanged(encFile9, fei9);
+
+ final FileEncryptionInfo fei0new = getFileEncryptionInfo(encFile0);
+ final FileEncryptionInfo fei9new = getFileEncryptionInfo(encFile9);
+ fs.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
+ fs.saveNamespace();
+ fs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE);
+ restartClusterDisableReencrypt();
+
+ assertKeyVersionEquals(encFile0, fei0new);
+ assertKeyVersionEquals(encFile9, fei9new);
+ assertNull("Re-encrypt queue should be empty after restart",
+ getReencryptionStatus().getNextUnprocessedZone());
+ }
+
+ @Test
+ public void testReencryptLoadedFromEdits() throws Exception {
+ /*
+ * /zones/zone/[0-9]
+ * /zones/zone/dir/f
+ */
+ final int len = 8196;
+ final Path zoneParent = new Path("/zones");
+ final Path zone = new Path(zoneParent, "zone");
+ fsWrapper.mkdir(zone, FsPermission.getDirDefault(), true);
+ dfsAdmin.createEncryptionZone(zone, TEST_KEY, NO_TRASH);
+ for (int i = 0; i < 10; ++i) {
+ DFSTestUtil
+ .createFile(fs, new Path(zone, Integer.toString(i)), len, (short) 1,
+ 0xFEED);
+ }
+ final Path subdir = new Path(zone, "dir");
+ fsWrapper.mkdir(subdir, FsPermission.getDirDefault(), true);
+ DFSTestUtil.createFile(fs, new Path(subdir, "f"), len, (short) 1, 0xFEED);
+
+ final Path encFile0 = new Path(zone, "0");
+ final Path encFile9 = new Path(zone, "9");
+ final FileEncryptionInfo fei0 = getFileEncryptionInfo(encFile0);
+ final FileEncryptionInfo fei9 = getFileEncryptionInfo(encFile9);
+ fsn.getProvider().rollNewVersion(TEST_KEY);
+ fsn.getProvider().flush();
+ // disable re-encrypt for testing, and issue a command
+ getEzManager().pauseReencryptForTesting();
+ dfsAdmin.reencryptEncryptionZone(zone, ReencryptAction.START);
+
+ // verify after restart the command is loaded
+ restartClusterDisableReencrypt();
+ waitForQueuedZones(1);
+
+ // Let the re-encrypt to start running.
+ getEzManager().resumeReencryptForTesting();
+ waitForReencryptedZones(1);
+ assertKeyVersionChanged(encFile0, fei0);
+ assertKeyVersionChanged(encFile9, fei9);
+
+ // verify status
+ verifyZoneStatus(zone, fei0, 11);
+ }
+
+ private void verifyZoneStatus(final Path zone, final FileEncryptionInfo fei,
+ final long expectedFiles) throws IOException {
+ RemoteIterator<ZoneReencryptionStatus> it =
+ dfsAdmin.listReencryptionStatus();
+ assertTrue(it.hasNext());
+ final ZoneReencryptionStatus zs = it.next();
+ assertEquals(zone.toString(), zs.getZoneName());
+ assertEquals(ZoneReencryptionStatus.State.Completed, zs.getState());
+ assertTrue(zs.getCompletionTime() > 0);
+ assertTrue(zs.getCompletionTime() > zs.getSubmissionTime());
+ if (fei != null) {
+ assertNotEquals(fei.getEzKeyVersionName(), zs.getEzKeyVersionName());
+ }
+ assertEquals(expectedFiles, zs.getFilesReencrypted());
+ }
+
+ @Test
+ public void testReencryptLoadedFromFsimage() throws Exception {
+ /*
+ * /zones/zone/[0-9]
+ * /zones/zone/dir/f
+ */
+ final int len = 8196;
+ final Path zoneParent = new Path("/zones");
+ final Path zone = new Path(zoneParent, "zone");
+ fsWrapper.mkdir(zone, FsPermission.getDirDefault(), true);
+ dfsAdmin.createEncryptionZone(zone, TEST_KEY, NO_TRASH);
+ for (int i = 0; i < 10; ++i) {
+ DFSTestUtil
+ .createFile(fs, new Path(zone, Integer.toString(i)), len, (short) 1,
+ 0xFEED);
+ }
+ final Path subdir = new Path(zone, "dir");
+ fsWrapper.mkdir(subdir, FsPermission.getDirDefault(), true);
+ DFSTestUtil.createFile(fs, new Path(subdir, "f"), len, (short) 1, 0xFEED);
+
+ final Path encFile0 = new Path(zone, "0");
+ final Path encFile9 = new Path(zone, "9");
+ final FileEncryptionInfo fei0 = getFileEncryptionInfo(encFile0);
+ final FileEncryptionInfo fei9 = getFileEncryptionInfo(encFile9);
+ fsn.getProvider().rollNewVersion(TEST_KEY);
+ fsn.getProvider().flush();
+ // disable re-encrypt for testing, and issue a command
+ getEzManager().pauseReencryptForTesting();
+ dfsAdmin.reencryptEncryptionZone(zone, ReencryptAction.START);
+ waitForQueuedZones(1);
+
+ fs.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
+ fs.saveNamespace();
+ fs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE);
+
+ // verify after loading from fsimage the command is loaded
+ restartClusterDisableReencrypt();
+ waitForQueuedZones(1);
+
+ // Let the re-encrypt to start running.
+ getEzManager().resumeReencryptForTesting();
+ waitForReencryptedZones(1);
+ assertKeyVersionChanged(encFile0, fei0);
+ assertKeyVersionChanged(encFile9, fei9);
+
+ // verify status
+ verifyZoneStatus(zone, fei0, 11);
+ }
+
+ @Test
+ public void testReencryptCommandsQueuedOrdering() throws Exception {
+ final Path zoneParent = new Path("/zones");
+ final String zoneBaseName = zoneParent.toString() + "/zone";
+ final int numZones = 10;
+ for (int i = 0; i < numZones; ++i) {
+ final Path zone = new Path(zoneBaseName + i);
+ fsWrapper.mkdir(zone, FsPermission.getDirDefault(), true);
+ dfsAdmin.createEncryptionZone(zone, TEST_KEY, NO_TRASH);
+ }
+
+ // Disable re-encrypt for testing, and issue commands
+ getEzManager().pauseReencryptForTesting();
+ for (int i = 0; i < numZones; ++i) {
+ dfsAdmin.reencryptEncryptionZone(new Path(zoneBaseName + i),
+ ReencryptAction.START);
+ }
+ waitForQueuedZones(numZones);
+
+ // Verify commands are queued in the same order submitted
+ ReencryptionStatus rzs = new ReencryptionStatus(getReencryptionStatus());
+ for (int i = 0; i < numZones; ++i) {
+ Long zoneId = fsn.getFSDirectory().getINode(zoneBaseName + i).getId();
+ assertEquals(zoneId, rzs.getNextUnprocessedZone());
+ rzs.removeZone(zoneId);
+ }
+
+ // Cancel some zones
+ Set<Integer> cancelled = new HashSet<>(Arrays.asList(0, 3, 4));
+ for (int cancel : cancelled) {
+ dfsAdmin.reencryptEncryptionZone(new Path(zoneBaseName + cancel),
+ ReencryptAction.CANCEL);
+ }
+
+ restartClusterDisableReencrypt();
+ waitForQueuedZones(numZones - cancelled.size());
+ rzs = new ReencryptionStatus(getReencryptionStatus());
+ for (int i = 0; i < numZones; ++i) {
+ if (cancelled.contains(i)) {
+ continue;
+ }
+ Long zoneId = fsn.getFSDirectory().getINode(zoneBaseName + i).getId();
+ assertEquals(zoneId, rzs.getNextUnprocessedZone());
+ rzs.removeZone(zoneId);
+ }
+
+ // Verify the same is true after loading from FSImage
+ fs.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
+ fs.saveNamespace();
+ fs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE);
+
+ restartClusterDisableReencrypt();
+ waitForQueuedZones(numZones - cancelled.size());
+ rzs = new ReencryptionStatus(getReencryptionStatus());
+ for (int i = 0; i < 10; ++i) {
+ if (cancelled.contains(i)) {
+ continue;
+ }
+ Long zoneId = fsn.getFSDirectory().getINode(zoneBaseName + i).getId();
+ assertEquals(zoneId, rzs.getNextUnprocessedZone());
+ rzs.removeZone(zoneId);
+ }
+ }
+
+ @Test
+ public void testReencryptNestedZones() throws Exception {
+ /* Setup dir as follows:
+ * / <- EZ
+ * /file
+ * /dir/dfile
+ * /level1 <- nested EZ
+ * /level1/fileL1-[0~2]
+ * /level1/level2/ <- nested EZ
+ * /level1/level2/fileL2-[0~3]
+ */
+ final int len = 8196;
+ final Path zoneRoot = new Path("/");
+ final Path zoneL1 = new Path(zoneRoot, "level1");
+ final Path zoneL2 = new Path(zoneL1, "level2");
+ final Path nonzoneDir = new Path(zoneRoot, "dir");
+ dfsAdmin.createEncryptionZone(zoneRoot, TEST_KEY, NO_TRASH);
+ DFSTestUtil
+ .createFile(fs, new Path(zoneRoot, "file"), len, (short) 1, 0xFEED);
+ DFSTestUtil
+ .createFile(fs, new Path(nonzoneDir, "dfile"), len, (short) 1, 0xFEED);
+ fsWrapper.mkdir(zoneL1, FsPermission.getDirDefault(), true);
+ dfsAdmin.createEncryptionZone(zoneL1, TEST_KEY, NO_TRASH);
+ for (int i = 0; i < 3; ++i) {
+ DFSTestUtil
+ .createFile(fs, new Path(zoneL1, "fileL1-" + i), len, (short) 1,
+ 0xFEED);
+ }
+ fsWrapper.mkdir(zoneL2, FsPermission.getDirDefault(), true);
+ dfsAdmin.createEncryptionZone(zoneL2, TEST_KEY, NO_TRASH);
+ for (int i = 0; i < 4; ++i) {
+ DFSTestUtil
+ .createFile(fs, new Path(zoneL2, "fileL2-" + i), len, (short) 1,
+ 0xFEED);
+ }
+
+ fsn.getProvider().rollNewVersion(TEST_KEY);
+ fsn.getProvider().flush();
+ // Disable re-encrypt, send re-encrypt on '/', verify queue
+ getEzManager().pauseReencryptForTesting();
+ dfsAdmin.reencryptEncryptionZone(zoneRoot, ReencryptAction.START);
+ waitForQueuedZones(1);
+ ReencryptionStatus rzs = getReencryptionStatus();
+ assertEquals(
+ (Long) fsn.getFSDirectory().getINode(zoneRoot.toString()).getId(),
+ rzs.getNextUnprocessedZone());
+
+ // Resume re-encrypt, verify files re-encrypted
+ getEzManager().resumeReencryptForTesting();
+ waitForZoneCompletes(zoneRoot.toString());
+ assertEquals(2, getZoneStatus(zoneRoot.toString()).getFilesReencrypted());
+
+ // Same tests on a child EZ.
+ getEzManager().resetMetricsForTesting();
+ getEzManager().pauseReencryptForTesting();
+ dfsAdmin.reencryptEncryptionZone(zoneL1, ReencryptAction.START);
+ waitForQueuedZones(1);
+ rzs = getReencryptionStatus();
+ assertEquals(
+ (Long) fsn.getFSDirectory().getINode(zoneL1.toString()).getId(),
+ rzs.getNextUnprocessedZone());
+
+ getEzManager().resumeReencryptForTesting();
+ waitForZoneCompletes(zoneL1.toString());
+ assertEquals(3, getZoneStatus(zoneL1.toString()).getFilesReencrypted());
+ }
+
+ @Test
+ public void testRaceCreateHandler() throws Exception {
+ /* Setup dir as follows:
+ * /dir/file[0~9]
+ */
+ final int len = 8196;
+ final Path zone = new Path("/dir");
+ fsWrapper.mkdir(zone, FsPermission.getDirDefault(), true);
+ dfsAdmin.createEncryptionZone(zone, TEST_KEY, NO_TRASH);
+ int expected = 10;
+ for (int i = 0; i < 10; ++i) {
+ DFSTestUtil
+ .createFile(fs, new Path(zone, "file" + i), len, (short) 1, 0xFEED);
+ }
+
+ fsn.getProvider().rollNewVersion(TEST_KEY);
+ fsn.getProvider().flush();
+ // Issue the command re-encrypt and pause it
+ getEzManager().pauseReencryptForTesting();
+ dfsAdmin.reencryptEncryptionZone(zone, ReencryptAction.START);
+ waitForQueuedZones(1);
+
+ // mark pause after first checkpoint (5 files)
+ getEzManager().pauseForTestingAfterNthSubmission(1);
+ // Resume the re-encrypt thread
+ getEzManager().resumeReencryptForTesting();
+ waitForReencryptedFiles(zone.toString(), 5);
+
+ /* creates the following:
+ * /dir/file8[0~5]
+ * /dir/dirsub/file[10-14]
+ * /dir/sub/file[15-19]
+ */
+ for (int i = 0; i < 6; ++i) {
+ DFSTestUtil
+ .createFile(fs, new Path(zone, "file8" + i), len, (short) 1, 0xFEED);
+ }
+ // we don't care newly created files since they should already use new edek.
+ // so naturally processes the listing from last checkpoint
+ final Path subdir = new Path(zone, "dirsub");
+ fsWrapper.mkdir(subdir, FsPermission.getDirDefault(), true);
+ for (int i = 10; i < 15; ++i) {
+ DFSTestUtil
+ .createFile(fs, new Path(subdir, "file" + i), len, (short) 1, 0xFEED);
+ }
+ // the above are created before checkpoint position, so not re-encrypted.
+ final Path sub = new Path(zone, "sub");
+ fsWrapper.mkdir(sub, FsPermission.getDirDefault(), true);
+ for (int i = 15; i < 20; ++i) {
+ DFSTestUtil
+ .createFile(fs, new Path(sub, "file" + i), len, (short) 1, 0xFEED);
+ }
+
+ // resume re-encrypt thread which was paused after first checkpoint
+ getEzManager().resumeReencryptForTesting();
+ waitForZoneCompletes(zone.toString());
+ assertEquals(expected,
+ getZoneStatus(zone.toString()).getFilesReencrypted());
+ }
+
+ @Test
+ public void testRaceDeleteHandler() throws Exception {
+ /* Setup dir as follows:
+ * /dir/file[0~9]
+ * /dir/subdir/file[10-14]
+ */
+ final int len = 8196;
+ final Path zone = new Path("/dir");
+ fsWrapper.mkdir(zone, FsPermission.getDirDefault(), true);
+ dfsAdmin.createEncryptionZone(zone, TEST_KEY, NO_TRASH);
+ int expected = 15;
+ for (int i = 0; i < 10; ++i) {
+ DFSTestUtil
+ .createFile(fs, new Path(zone, "file" + i), len, (short) 1, 0xFEED);
+ }
+ final Path subdir = new Path(zone, "subdir");
+ fsWrapper.mkdir(subdir, FsPermission.getDirDefault(), true);
+ for (int i = 10; i < 15; ++i) {
+ DFSTestUtil
+ .createFile(fs, new Path(subdir, "file" + i), len, (short) 1, 0xFEED);
+ }
+
+ fsn.getProvider().rollNewVersion(TEST_KEY);
+ fsn.getProvider().flush();
+ // Issue the command re-encrypt and pause it
+ getEzManager().pauseReencryptForTesting();
+ dfsAdmin.reencryptEncryptionZone(zone, ReencryptAction.START);
+ waitForQueuedZones(1);
+
+ // proceed to first checkpoint (5 files), delete files/subdir, then resume
+ getEzManager().pauseForTestingAfterNthSubmission(1);
+ getEzManager().resumeReencryptForTesting();
+ waitForReencryptedFiles(zone.toString(), 5);
+
+ fsWrapper.delete(new Path(zone, "file5"), true);
+ fsWrapper.delete(new Path(zone, "file8"), true);
+ expected -= 2;
+ fsWrapper.delete(subdir, true);
+ expected -= 5;
+
+ // resume re-encrypt thread which was paused after first checkpoint
+ getEzManager().resumeReencryptForTesting();
+ waitForZoneCompletes(zone.toString());
+ assertEquals(expected,
+ getZoneStatus(zone.toString()).getFilesReencrypted());
+ }
+
+ @Test
+ public void testRaceDeleteUpdater() throws Exception {
+ /* Setup dir as follows:
+ * /dir/file[0~9]
+ * /dir/subdir/file[10-14]
+ */
+ final int len = 8196;
+ final Path zone = new Path("/dir");
+ fsWrapper.mkdir(zone, FsPermission.getDirDefault(), true);
+ dfsAdmin.createEncryptionZone(zone, TEST_KEY, NO_TRASH);
+ int expected = 15;
+ for (int i = 0; i < 10; ++i) {
+ DFSTestUtil
+ .createFile(fs, new Path(zone, "file" + i), len, (short) 1, 0xFEED);
+ }
+ final Path subdir = new Path(zone, "subdir");
+ fsWrapper.mkdir(subdir, FsPermission.getDirDefault(), true);
+ for (int i = 10; i < 15; ++i) {
+ DFSTestUtil
+ .createFile(fs, new Path(subdir, "file" + i), len, (short) 1, 0xFEED);
+ }
+
+ fsn.getProvider().rollNewVersion(TEST_KEY);
+ fsn.getProvider().flush();
+ // Issue the command re-encrypt and pause it
+ getEzManager().pauseReencryptForTesting();
+ dfsAdmin.reencryptEncryptionZone(zone, ReencryptAction.START);
+ waitForQueuedZones(1);
+
+ // proceed to first checkpoint (5 files), delete files/subdir, then resume
+ getEzManager().pauseForTestingAfterNthCheckpoint(zone.toString(), 1);
+ getEzManager().pauseForTestingAfterNthSubmission(1);
+ getEzManager().resumeReencryptForTesting();
+
+ waitForReencryptedFiles(zone.toString(), 5);
+ getEzManager().resumeReencryptForTesting();
+
+ // give handler thread some time to process the files before deletion.
+ Thread.sleep(3000);
+ fsWrapper.delete(new Path(zone, "file5"), true);
+ fsWrapper.delete(new Path(zone, "file8"), true);
+ expected -= 2;
+ fsWrapper.delete(subdir, true);
+ expected -= 5;
+
+ // resume updater thread which was paused after first checkpoint, verify
+ // deleted files are skipped.
+ getEzManager().resumeReencryptUpdaterForTesting();
+ waitForZoneCompletes(zone.toString());
+ assertEquals(expected,
+ getZoneStatus(zone.toString()).getFilesReencrypted());
+ }
+
+ @Test
+ public void testRaceDeleteCurrentDirHandler() throws Exception {
+ /* Setup dir as follows:
+ * /dir/subdir/file[0~9]
+ * /dir/subdir2/file[10-14]
+ */
+ final int len = 8196;
+ final Path zone = new Path("/dir");
+ fsWrapper.mkdir(zone, FsPermission.getDirDefault(), true);
+ dfsAdmin.createEncryptionZone(zone, TEST_KEY, NO_TRASH);
+ final Path subdir = new Path(zone, "subdir");
+ int expected = 15;
+ for (int i = 0; i < 10; ++i) {
+ DFSTestUtil
+ .createFile(fs, new Path(subdir, "file" + i), len, (short) 1, 0xFEED);
+ }
+ final Path subdir2 = new Path(zone, "subdir2");
+ fsWrapper.mkdir(subdir, FsPermission.getDirDefault(), true);
+ for (int i = 10; i < 15; ++i) {
+ DFSTestUtil.createFile(fs, new Path(subdir2, "file" + i), len, (short) 1,
+ 0xFEED);
+ }
+
+ fsn.getProvider().rollNewVersion(TEST_KEY);
+ fsn.getProvider().flush();
+ // Issue the command re-encrypt and pause it
+ getEzManager().pauseReencryptForTesting();
+ dfsAdmin.reencryptEncryptionZone(zone, ReencryptAction.START);
+ waitForQueuedZones(1);
+
+ // proceed to first checkpoint (5 files), delete subdir, then resume
+ getEzManager().pauseForTestingAfterNthSubmission(1);
+ getEzManager().resumeReencryptForTesting();
+ waitForReencryptedFiles(zone.toString(), 5);
+
+ fsWrapper.delete(subdir, true);
+ expected -= 5;
+
+ // resume re-encrypt thread which was paused after first checkpoint
+ getEzManager().resumeReencryptForTesting();
+ waitForZoneCompletes(zone.toString());
+ assertEquals(expected,
+ getZoneStatus(zone.toString()).getFilesReencrypted());
+ }
+
+ @Test
+ public void testRaceDeleteCurrentDirUpdater() throws Exception {
+ /* Setup dir as follows:
+ * /dir/subdir/file[0~9]
+ * /dir/subdir2/file[10-14]
+ */
+ final int len = 8196;
+ final Path zone = new Path("/dir");
+ fsWrapper.mkdir(zone, FsPermission.getDirDefault(), true);
+ dfsAdmin.createEncryptionZone(zone, TEST_KEY, NO_TRASH);
+ final Path subdir = new Path(zone, "subdir");
+ int expected = 15;
+ for (int i = 0; i < 10; ++i) {
+ DFSTestUtil
+ .createFile(fs, new Path(subdir, "file" + i), len, (short) 1, 0xFEED);
+ }
+ final Path subdir2 = new Path(zone, "subdir2");
+ fsWrapper.mkdir(subdir, FsPermission.getDirDefault(), true);
+ for (int i = 10; i < 15; ++i) {
+ DFSTestUtil.createFile(fs, new Path(subdir2, "file" + i), len, (short) 1,
+ 0xFEED);
+ }
+
+ fsn.getProvider().rollNewVersion(TEST_KEY);
+ fsn.getProvider().flush();
+ // Issue the command re-encrypt and pause it
+ getEzManager().pauseReencryptForTesting();
+ dfsAdmin.reencryptEncryptionZone(zone, ReencryptAction.START);
+ waitForQueuedZones(1);
+
+ // proceed to first checkpoint (5 files), delete subdir, then resume
+ getEzManager().pauseForTestingAfterNthCheckpoint(zone.toString(), 1);
+ getEzManager().pauseForTestingAfterNthSubmission(1);
+ getEzManager().resumeReencryptForTesting();
+
+ waitForReencryptedFiles(zone.toString(), 5);
+ getEzManager().resumeReencryptForTesting();
+
+ // give handler thread some time to process the files before deletion.
+ Thread.sleep(3000);
+ fsWrapper.delete(subdir, true);
+ expected -= 5;
+
+ // resume updater thread which was paused after first checkpoint, verify
+ // deleted files are skipped.
+ getEzManager().resumeReencryptUpdaterForTesting();
+ waitForZoneCompletes(zone.toString());
+ assertEquals(expected,
+ getZoneStatus(zone.toString()).getFilesReencrypted());
+ }
+
+ @Test
+ public void testRaceDeleteZoneHandler() throws Exception {
+ /* Setup dir as follows:
+ * /dir/file[0~10]
+ */
+ final int len = 8196;
+ final Path zone = new Path("/dir");
+ fsWrapper.mkdir(zone, FsPermission.getDirDefault(), true);
+ dfsAdmin.createEncryptionZone(zone, TEST_KEY, NO_TRASH);
+ for (int i = 0; i < 11; ++i) {
+ DFSTestUtil
+ .createFile(fs, new Path(zone, "file" + i), len, (short) 1, 0xFEED);
+ }
+
+ fsn.getProvider().rollNewVersion(TEST_KEY);
+ fsn.getProvider().flush();
+ // Issue the command re-encrypt and pause it
+ getEzManager().pauseReencryptForTesting();
+ dfsAdmin.reencryptEncryptionZone(zone, ReencryptAction.START);
+ waitForQueuedZones(1);
+
+ // let both handler and updater pause, then delete zone.
+ getEzManager().pauseForTestingAfterNthSubmission(1);
+ getEzManager().pauseForTestingAfterNthCheckpoint(zone.toString(), 1);
+ getEzManager().resumeReencryptForTesting();
+ waitForReencryptedFiles(zone.toString(), 5);
+ getEzManager().pauseForTestingAfterNthSubmission(1);
+ getEzManager().resumeReencryptForTesting();
+
+ Thread.sleep(3000);
+ EncryptionZoneManager ezm = getEzManager();
+ ReencryptionHandler handler = (ReencryptionHandler) Whitebox
+ .getInternalState(ezm, "reencryptionHandler");
+ Map<Long, ZoneSubmissionTracker> tasks =
+ (Map<Long, ZoneSubmissionTracker>) Whitebox
+ .getInternalState(handler, "submissions");
+ List<Future> futures = new LinkedList<>();
+ for (ZoneSubmissionTracker zst : tasks.values()) {
+ for (Future f : zst.getTasks()) {
+ futures.add(f);
+ }
+ }
+ fsWrapper.delete(zone, true);
+ getEzManager().resumeReencryptForTesting();
+
+ // verify no running tasks
+ for (Future f : futures) {
+ assertTrue(f.isDone());
+ }
+
+ waitForTotalZones(0);
+ }
+
+ @Test
+ public void testRaceDeleteCreateHandler() throws Exception {
+ /* Setup dir as follows:
+ * /dir/file[0~9]
+ */
+ final int len = 8196;
+ final Path zone = new Path("/dir");
+ fsWrapper.mkdir(zone, FsPermission.getDirDefault(), true);
+ dfsAdmin.createEncryptionZone(zone, TEST_KEY, NO_TRASH);
+ int expected = 10;
+ for (int i = 0; i < 10; ++i) {
+ DFSTestUtil
+ .createFile(fs, new Path(zone, "file" + i), len, (short) 1, 0xFEED);
+ }
+
+ fsn.getProvider().rollNewVersion(TEST_KEY);
+ fsn.getProvider().flush();
+ // Issue the command re-encrypt and pause it
+ getEzManager().pauseReencryptForTesting();
+ dfsAdmin.reencryptEncryptionZone(zone, ReencryptAction.START);
+ waitForQueuedZones(1);
+
+ // mark pause after first checkpoint (5 files)
+ getEzManager().pauseForTestingAfterNthSubmission(1);
+ // Resume the re-encrypt thread
+ getEzManager().resumeReencryptForTesting();
+ waitForReencryptedFiles(zone.toString(), 5);
+
+ final Path recreated = new Path(zone, "file9");
+ fsWrapper.delete(recreated, true);
+ DFSTestUtil.createFile(fs, recreated, len, (short) 2, 0xFEED);
+ expected -= 1; // newly created files use new edek, no need to re-encrypt
+
+ // resume re-encrypt thread which was paused after first checkpoint
+ getEzManager().resumeReencryptForTesting();
+ waitForZoneCompletes(zone.toString());
+ assertEquals(expected,
+ getZoneStatus(zone.toString()).getFilesReencrypted());
+ }
+
+ @Test
+ public void testRaceDeleteCreateUpdater() throws Exception {
+ /* Setup dir as follows:
+ * /dir/file[0~9]
+ */
+ final int len = 8196;
+ final Path zone = new Path("/dir");
+ fsWrapper.mkdir(zone, FsPermission.getDirDefault(), true);
+ dfsAdmin.createEncryptionZone(zone, TEST_KEY, NO_TRASH);
+ int expected = 10;
+ for (int i = 0; i < 10; ++i) {
+ DFSTestUtil
+ .createFile(fs, new Path(zone, "file" + i), len, (short) 1, 0xFEED);
+ }
+
+ fsn.getProvider().rollNewVersion(TEST_KEY);
+ fsn.getProvider().flush();
+ // Issue the command re-encrypt and pause it
+ getEzManager().pauseReencryptForTesting();
+ dfsAdmin.reencryptEncryptionZone(zone, ReencryptAction.START);
+ waitForQueuedZones(1);
+
+ // mark pause after first checkpoint (5 files)
+ getEzManager().pauseForTestingAfterNthCheckpoint(zone.toString(), 1);
+ getEzManager().pauseForTestingAfterNthSubmission(1);
+ getEzManager().resumeReencryptForTesting();
+ waitForReencryptedFiles(zone.toString(), 5);
+ getEzManager().resumeReencryptForTesting();
+
+ // give handler thread some time to process the files before deletion.
+ Thread.sleep(3000);
+ final Path recreated = new Path(zone, "file9");
+ final FileEncryptionInfo feiOrig = getFileEncryptionInfo(recreated);
+ final String contentOrig = DFSTestUtil.readFile(fs, recreated);
+ fsWrapper.delete(recreated, true);
+ DFSTestUtil.createFile(fs, recreated, len, (short) 2, 0xFEED);
+ expected -= 1;
+
+ // resume updater thread which was paused after first checkpoint
+ getEzManager().resumeReencryptUpdaterForTesting();
+ waitForZoneCompletes(zone.toString());
+ assertEquals(expected,
+ getZoneStatus(zone.toString()).getFilesReencrypted());
+
+ // verify new file is using it's own edeks, with new keyversions,
+ // and can be decrypted correctly.
+ assertKeyVersionChanged(recreated, feiOrig);
+ final String content = DFSTestUtil.readFile(fs, recreated);
+ assertEquals(contentOrig, content);
+ }
+
+ // TODO: update test once HDFS-11203 is implemented.
+ @Test
+ public void testReencryptRaceRename() throws Exception {
+ /* Setup dir as follows:
+ * /dir/file[0~9]
+ * /dir/subdir/file[10-14]
+ */
+ final int len = 8196;
+ final Path zone = new Path("/dir");
+ fsWrapper.mkdir(zone, FsPermission.getDirDefault(), true);
+ dfsAdmin.createEncryptionZone(zone, TEST_KEY, NO_TRASH);
+ for (int i = 0; i < 10; ++i) {
+ DFSTestUtil
+ .createFile(fs, new Path(zone, "file" + i), len, (short) 1, 0xFEED);
+ }
+ final Path subdir = new Path(zone, "subdir");
+ fsWrapper.mkdir(subdir, FsPermission.getDirDefault(), true);
+ for (int i = 10; i < 15; ++i) {
+ DFSTestUtil
+ .createFile(fs, new Path(subdir, "file" + i), len, (short) 1, 0xFEED);
+ }
+
+ fsn.getProvider().rollNewVersion(TEST_KEY);
+ fsn.getProvider().flush();
+ // Issue the command re-encrypt and pause it
+ getEzManager().pauseReencryptForTesting();
+ dfsAdmin.reencryptEncryptionZone(zone, ReencryptAction.START);
+ waitForQueuedZones(1);
+
+ // mark pause after first checkpoint (5 files)
+ getEzManager().pauseForTestingAfterNthSubmission(1);
+ // Resume the re-encrypt thread
+ getEzManager().resumeReencryptForTesting();
+ waitForReencryptedFiles(zone.toString(), 5);
+
+ try {
+ fsWrapper.rename(new Path(zone, "file8"), new Path(zone, "file08"));
+ fail("rename a file in an EZ should be disabled");
+ } catch (IOException e) {
+ assertExceptionContains("under re-encryption", e);
+ }
+
+ // resume handler and pause updater, test again.
+ getEzManager().pauseReencryptUpdaterForTesting();
+ getEzManager().resumeReencryptForTesting();
+ try {
+ fsWrapper.rename(new Path(zone, "file8"), new Path(zone, "file08"));
+ fail("rename a file in an EZ should be disabled");
+ } catch (IOException e) {
+ assertExceptionContains("under re-encryption", e);
+ }
+ }
+
+ @Test
+ public void testReencryptSnapshots() throws Exception {
+ /* Setup test dir:
+ * /zones/zone/[0-9]
+ * /dir/f
+ *
+ * /zones/zone is snapshottable, and rename file 5 to 5new,
+ * 6 to 6new then delete (so the file is only referred from a snapshot).
+ */
+ final int len = 8196;
+ final Path zoneParent = new Path("/zones");
+ final Path zone = new Path(zoneParent, "zone");
+ fsWrapper.mkdir(zone, FsPermission.getDirDefault(), true);
+ dfsAdmin.allowSnapshot(zone);
+ dfsAdmin.createEncryptionZone(zone, TEST_KEY, NO_TRASH);
+ for (int i = 0; i < 10; ++i) {
+ DFSTestUtil
+ .createFile(fs, new Path(zone, Integer.toString(i)), len, (short) 1,
+ 0xFEED);
+ }
+ final Path subdir = new Path("/dir");
+ fsWrapper.mkdir(subdir, FsPermission.getDirDefault(), true);
+ DFSTestUtil.createFile(fs, new Path(subdir, "f"), len, (short) 1, 0xFEED);
+ // create a snapshot and rename a file, so INodeReference is created.
+ final Path zoneSnap = fs.createSnapshot(zone);
+ fsWrapper.rename(new Path(zone, "5"), new Path(zone, "5new"));
+ fsWrapper.rename(new Path(zone, "6"), new Path(zone, "6new"));
+ fsWrapper.delete(new Path(zone, "6new"), true);
+
+ // test re-encrypt on snapshot dir
+ final Path encFile1 = new Path(zone, "0");
+ final FileEncryptionInfo fei0 = getFileEncryptionInfo(encFile1);
+ fsn.getProvider().rollNewVersion(TEST_KEY);
+ fsn.getProvider().flush();
+ try {
+ dfsAdmin.reencryptEncryptionZone(zoneSnap, ReencryptAction.START);
+ fail("Reencrypt command on snapshot path should fail.");
+ } catch (RemoteException expected) {
+ LOG.info("Expected exception", expected);
+ assertTrue(expected
+ .unwrapRemoteException() instanceof SnapshotAccessControlException);
+ }
+ dfsAdmin.reencryptEncryptionZone(zone, ReencryptAction.START);
+ waitForReencryptedZones(1);
+ waitForReencryptedFiles(zone.toString(), 9);
+ assertKeyVersionChanged(encFile1, fei0);
+ }
+
+ private void restartClusterDisableReencrypt() throws Exception {
+ cluster.restartNameNode(false);
+ fsn = cluster.getNamesystem();
+ getEzManager().pauseReencryptForTesting();
+ cluster.waitActive();
+ }
+
+ private void waitForReencryptedZones(final int expected)
+ throws TimeoutException, InterruptedException {
+ LOG.info("Waiting for re-encrypted zones to be {}", expected);
+ try {
+ GenericTestUtils.waitFor(new Supplier<Boolean>() {
+ @Override
+ public Boolean get() {
+ return getReencryptionStatus().getNumZonesReencrypted() == expected;
+ }
+ }, 100, 10000);
+ } finally {
+ LOG.info("Re-encrypted zones = {} ",
+ getReencryptionStatus().getNumZonesReencrypted());
+ }
+ }
+
+ private void waitForQueuedZones(final int expected)
+ throws TimeoutException, InterruptedException {
+ LOG.info("Waiting for queued zones for re-encryption to be {}", expected);
+ GenericTestUtils.waitFor(new Supplier<Boolean>() {
+ @Override
+ public Boolean get() {
+ return getReencryptionStatus().zonesQueued() == expected;
+ }
+ }, 100, 10000);
+ }
+
+ private void waitForTotalZones(final int expected)
+ throws TimeoutException, InterruptedException {
+ LOG.info("Waiting for queued zones for re-encryption to be {}", expected);
+ GenericTestUtils.waitFor(new Supplier<Boolean>() {
+ @Override
+ public Boolean get() {
+ return getReencryptionStatus().zonesTotal() == expected;
+ }
+ }, 100, 10000);
+ }
+
+ private void waitForZoneCompletes(final String zone)
+ throws TimeoutException, InterruptedException {
+ LOG.info("Waiting for re-encryption zone {} to complete.", zone);
+ GenericTestUtils.waitFor(new Supplier<Boolean>() {
+ @Override
+ public Boolean get() {
+ try {
+ return getZoneStatus(zone).getState()
+ == ZoneReencryptionStatus.State.Completed;
+ } catch (Exception ex) {
+ LOG.error("Exception caught", ex);
+ return false;
+ }
+ }
+ }, 100, 10000);
+ }
+
+ private EncryptionZoneManager getEzManager() {
+ return fsn.getFSDirectory().ezManager;
+ }
+
+ private ReencryptionStatus getReencryptionStatus() {
+ return getEzManager().getReencryptionStatus();
+ }
+
+ private ZoneReencryptionStatus getZoneStatus(final String zone)
+ throws IOException {
+ return getEzManager().getZoneStatus(zone);
+ }
+
+ private void waitForReencryptedFiles(final String zone, final int expected)
+ throws TimeoutException, InterruptedException {
+ LOG.info("Waiting for total re-encrypted file count to be {}", expected);
+ GenericTestUtils.waitFor(new Supplier<Boolean>() {
+ @Override
+ public Boolean get() {
+ try {
+ return getZoneStatus(zone).getFilesReencrypted() == expected;
+ } catch (IOException e) {
+ return false;
+ }
+ }
+ }, 100, 10000);
+ }
+
+ private void assertKeyVersionChanged(final Path file,
+ final FileEncryptionInfo original) throws Exception {
+ final FileEncryptionInfo actual = getFileEncryptionInfo(file);
+ assertNotEquals("KeyVersion should be different",
+ original.getEzKeyVersionName(), actual.getEzKeyVersionName());
+ }
+
+ private void assertKeyVersionEquals(final Path file,
+ final FileEncryptionInfo expected) throws Exception {
+ final FileEncryptionInfo actual = getFileEncryptionInfo(file);
+ assertEquals("KeyVersion should be the same",
+ expected.getEzKeyVersionName(), actual.getEzKeyVersionName());
+ }
+
+ @Test
+ public void testReencryptCancel() throws Exception {
+ /* Setup test dir:
+ * /zones/zone/[0-9]
+ * /dir/f
+ */
+ final int len = 8196;
+ final Path zoneParent = new Path("/zones");
+ final Path zone = new Path(zoneParent, "zone");
+ fsWrapper.mkdir(zone, FsPermission.getDirDefault(), true);
+ dfsAdmin.createEncryptionZone(zone, TEST_KEY, NO_TRASH);
+ for (int i = 0; i < 10; ++i) {
+ DFSTestUtil
+ .createFile(fs, new Path(zone, Integer.toString(i)), len, (short) 1,
+ 0xFEED);
+ }
+ final Path subdir = new Path("/dir");
+ fsWrapper.mkdir(subdir, FsPermission.getDirDefault(), true);
+ DFSTestUtil.createFile(fs, new Path(subdir, "f"), len, (short) 1, 0xFEED);
+
+ fsn.getProvider().rollNewVersion(TEST_KEY);
+ fsn.getProvider().flush();
+ // disable, test basic
+ getEzManager().pauseReencryptForTesting();
+ dfsAdmin.reencryptEncryptionZone(zone, ReencryptAction.START);
+ waitForQueuedZones(1);
+
+ dfsAdmin.reencryptEncryptionZone(zone, ReencryptAction.CANCEL);
+ getEzManager().resumeReencryptForTesting();
+ waitForZoneCompletes(zone.toString());
+ assertEquals(0, getZoneStatus(zone.toString()).getFilesReencrypted());
+
+ // test same command resubmission
+ try {
+ dfsAdmin.reencryptEncryptionZone(zone, ReencryptAction.CANCEL);
+ } catch (RemoteException expected) {
+ assertExceptionContains("not under re-encryption", expected);
+ }
+
+ fsn.getProvider().rollNewVersion(TEST_KEY);
+ fsn.getProvider().flush();
+ // test cancelling half-way
+ getEzManager().pauseForTestingAfterNthSubmission(1);
+ getEzManager().resumeReencryptForTesting();
+ dfsAdmin.reencryptEncryptionZone(zone, ReencryptAction.START);
+ waitForReencryptedFiles(zone.toString(), 5);
+ dfsAdmin.reencryptEncryptionZone(zone, ReencryptAction.CANCEL);
+ getEzManager().resumeReencryptForTesting();
+ waitForZoneCompletes(zone.toString());
+ assertEquals(5, getZoneStatus(zone.toString()).getFilesReencrypted());
+ assertNull(
+ getEzManager().getZoneStatus(zone.toString()).getLastCheckpointFile());
+ assertNull(getReencryptionStatus().getNextUnprocessedZone());
+
+ // test cancelling non-EZ dir
+ try {
+ dfsAdmin.reencryptEncryptionZone(subdir, ReencryptAction.CANCEL);
+ fail("Re-encrypting non-EZ should fail");
+ } catch (RemoteException expected) {
+ LOG.info("Expected exception caught.", expected);
+ assertExceptionContains("not the root of an encryption zone", expected);
+ }
+
+ // test cancelling non-existing dir
+ try {
+ dfsAdmin.reencryptEncryptionZone(new Path(zone, "notexist"),
+ ReencryptAction.CANCEL);
+ fail("Re-encrypting non-existing dir should fail");
+ } catch (RemoteException expected) {
+ LOG.info("Expected exception caught.", expected);
+ assertTrue(
+ expected.unwrapRemoteException() instanceof FileNotFoundException);
+ }
+
+ // test cancelling directly on a EZ file
+ final Path encFile = new Path(zone, "0");
+ try {
+ dfsAdmin.reencryptEncryptionZone(encFile, ReencryptAction.CANCEL);
+ fail("Re-encrypting on a file should fail");
+ } catch (RemoteException expected) {
+ LOG.info("Expected exception caught.", expected);
+ assertExceptionContains("not the root of an encryption zone", expected);
+ }
+
+ // final check - should only had 5 files re-encrypted overall.
+ assertEquals(5, getZoneStatus(zone.toString()).getFilesReencrypted());
+ }
+
+ @Test
+ public void testReencryptCancelForUpdater() throws Exception {
+ /* Setup test dir:
+ * /zones/zone/[0-9]
+ * /dir/f
+ */
+ final int len = 8196;
+ final Path zoneParent = new Path("/zones");
+ final Path zone = new Path(zoneParent, "zone");
+ fsWrapper.mkdir(zone, FsPermission.getDirDefault(), true);
+ dfsAdmin.createEncryptionZone(zone, TEST_KEY, NO_TRASH);
+ for (int i = 0; i < 10; ++i) {
+ DFSTestUtil
+ .createFile(fs, new Path(zone, Integer.toString(i)), len, (short) 1,
+ 0xFEED);
+ }
+ final Path subdir = new Path("/dir");
+ fsWrapper.mkdir(subdir, FsPermission.getDirDefault(), true);
+ DFSTestUtil.createFile(fs, new Path(subdir, "f"), len, (short) 1, 0xFEED);
+
+ fsn.getProvider().rollNewVersion(TEST_KEY);
+ fsn.getProvider().flush();
+ // disable, test basic
+ getEzManager().pauseReencryptUpdaterForTesting();
+ dfsAdmin.reencryptEncryptionZone(zone, ReencryptAction.START);
+ Thread.sleep(3000);
+
+ dfsAdmin.reencryptEncryptionZone(zone, ReencryptAction.CANCEL);
+ getEzManager().resumeReencryptUpdaterForTesting();
+ waitForZoneCompletes(zone.toString());
+ Thread.sleep(3000);
+ assertEquals(0, getZoneStatus(zone.toString()).getFilesReencrypted());
+
+ }
+
+ @Test
+ public void testReencryptionWithoutProvider() throws Exception {
+ /* Setup test dir:
+ * /zones/zone/[0-9]
+ */
+ final int len = 8196;
+ final Path zoneParent = new Path("/zones");
+ final Path zone = new Path(zoneParent, "zone");
+ fsWrapper.mkdir(zone, FsPermission.getDirDefault(), true);
+ dfsAdmin.createEncryptionZone(zone, TEST_KEY, NO_TRASH);
+ for (int i = 0; i < 10; ++i) {
+ DFSTestUtil
+ .createFile(fs, new Path(zone, Integer.toString(i)), len, (short) 1,
+ 0xFEED);
+ }
+
+ // re-encrypt the zone
+ fsn.getProvider().rollNewVersion(TEST_KEY);
+ fsn.getProvider().flush();
+ dfsAdmin.reencryptEncryptionZone(zone, ReencryptAction.START);
+ waitForReencryptedZones(1);
+
+ // start NN without providers
+ cluster.getConfiguration(0)
+ .unset(CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH);
+ cluster.restartNameNodes();
+ cluster.waitActive();
+
+ // test re-encrypt should fail
+ try {
+ dfsAdmin.reencryptEncryptionZone(zone, ReencryptAction.START);
+ fail("should not be able to re-encrypt");
+ } catch (RemoteException expected) {
+ assertExceptionContains("rejected", expected.unwrapRemoteException());
+ }
+ try {
+ dfsAdmin.reencryptEncryptionZone(zone, ReencryptAction.CANCEL);
+ fail("should not be able to cancel re-encrypt");
+ } catch (RemoteException expected) {
+ assertExceptionContains("rejected", expected.unwrapRemoteException());
+ }
+
+ // test listReencryptionStatus should still work
+ RemoteIterator<ZoneReencryptionStatus> it =
+ dfsAdmin.listReencryptionStatus();
+ assertTrue(it.hasNext());
+ ZoneReencryptionStatus zs = it.next();
+ assertEquals(zone.toString(), zs.getZoneName());
+ assertEquals(ZoneReencryptionStatus.State.Completed, zs.getState());
+ assertTrue(zs.getCompletionTime() > 0);
+ assertTrue(zs.getCompletionTime() > zs.getSubmissionTime());
+ assertEquals(10, zs.getFilesReencrypted());
+ }
+
+ @Test
+ public void testReencryptionNNSafeMode() throws Exception {
+ /* Setup test dir:
+ * /zones/zone/[0-9]
+ */
+ final int len = 8196;
+ final Path zoneParent = new Path("/zones");
+ final Path zone = new Path(zoneParent, "zone");
+ fsWrapper.mkdir(zone, FsPermission.getDirDefault(), true);
+ dfsAdmin.createEncryptionZone(zone, TEST_KEY, NO_TRASH);
+ for (int i = 0; i < 10; ++i) {
+ DFSTestUtil
+ .createFile(fs, new Path(zone, Integer.toString(i)), len, (short) 1,
+ 0xFEED);
+ }
+
+ fsn.getProvider().rollNewVersion(TEST_KEY);
+ fsn.getProvider().flush();
+ // mark pause after first checkpoint (5 files)
+ getEzManager().pauseForTestingAfterNthSubmission(1);
+ dfsAdmin.reencryptEncryptionZone(zone, ReencryptAction.START);
+ waitForReencryptedFiles(zone.toString(), 5);
+
+ fs.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
+ getEzManager().resumeReencryptForTesting();
+ for (int i = 0; i < 3; ++i) {
+ Thread.sleep(1000);
+ RemoteIterator<ZoneReencryptionStatus> it =
+ dfsAdmin.listReencryptionStatus();
+ assertTrue(it.hasNext());
+ ZoneReencryptionStatus zs = it.next();
+ assertEquals(zone.toString(), zs.getZoneName());
+ assertEquals(0, zs.getCompletionTime());
+ assertEquals(5, zs.getFilesReencrypted());
+ }
+
+ fs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE);
+ waitForReencryptedFiles(zone.toString(), 10);
+ }
+
+ @Test
+ public void testReencryptionKMSDown() throws Exception {
+ class MyInjector extends EncryptionFaultInjector {
+ private volatile int exceptionCount = 0;
+
+ MyInjector(int numFailures) {
+ exceptionCount = numFailures;
+ }
+
+ @Override
+ public void reencryptEncryptedKeys() throws IOException {
+ if (exceptionCount > 0) {
+ --exceptionCount;
+ throw new IOException("Injected KMS failure");
+ }
+ }
+ }
+ final MyInjector injector = new MyInjector(1);
+ EncryptionFaultInjector.instance = injector;
+ /* Setup test dir:
+ * /zones/zone/[0-9]
+ */
+ final int len = 8196;
+ final Path zoneParent = new Path("/zones");
+ final Path zone = new Path(zoneParent, "zone");
+ fsWrapper.mkdir(zone, FsPermission.getDirDefault(), true);
+ dfsAdmin.createEncryptionZone(zone, TEST_KEY, NO_TRASH);
+ for (int i = 0; i < 10; ++i) {
+ DFSTestUtil
+ .createFile(fs, new Path(zone, Integer.toString(i)), len, (short) 1,
+ 0xFEED);
+ }
+
+ // re-encrypt the zone
+ fsn.getProvider().rollNewVersion(TEST_KEY);
+ fsn.getProvider().flush();
+
+ dfsAdmin.reencryptEncryptionZone(zone, ReencryptAction.START);
+ waitForReencryptedZones(1);
+ assertEquals(0, injector.exceptionCount);
+
+ // test listReencryptionStatus should still work
+ RemoteIterator<ZoneReencryptionStatus> it =
+ dfsAdmin.listReencryptionStatus();
+ assertTrue(it.hasNext());
+ ZoneReencryptionStatus zs = it.next();
+ assertEquals(zone.toString(), zs.getZoneName());
+ assertEquals(ZoneReencryptionStatus.State.Completed, zs.getState());
+ assertTrue(zs.getCompletionTime() > 0);
+ assertTrue(zs.getCompletionTime() > zs.getSubmissionTime());
+ assertEquals(5, zs.getFilesReencrypted());
+ assertEquals(5, zs.getNumReencryptionFailures());
+ }
+
+ @Test
+ public void testReencryptionUpdaterFaultOneTask() throws Exception {
+ class MyInjector extends EncryptionFaultInjector {
+ private volatile int exceptionCount = 0;
+
+ MyInjector(int numFailures) {
+ exceptionCount = numFailures;
+ }
+
+ @Override
+ public void reencryptUpdaterProcessOneTask() throws IOException {
+ if (exceptionCount > 0) {
+ --exceptionCount;
+ throw new IOException("Injected process task failure");
+ }
+ }
+ }
+ final MyInjector injector = new MyInjector(1);
+ EncryptionFaultInjector.instance = injector;
+ /* Setup test dir:
+ * /zones/zone/[0-9]
+ */
+ final int len = 8196;
+ final Path zoneParent = new Path("/zones");
+ final Path zone = new Path(zoneParent, "zone");
+ fsWrapper.mkdir(zone, FsPermission.getDirDefault(), true);
+ dfsAdmin.createEncryptionZone(zone, TEST_KEY, NO_TRASH);
+ for (int i = 0; i < 10; ++i) {
+ DFSTestUtil
+ .createFile(fs, new Path(zone, Integer.toString(i)), len, (short) 1,
+ 0xFEED);
+ }
+
+ // re-encrypt the zone
+ fsn.getProvider().rollNewVersion(TEST_KEY);
+ fsn.getProvider().flush();
+
+ dfsAdmin.reencryptEncryptionZone(zone, ReencryptAction.START);
+ waitForReencryptedZones(1);
+ assertEquals(0, injector.exceptionCount);
+
+ // test listReencryptionStatus should still work
+ RemoteIterator<ZoneReencryptionStatus> it =
+ dfsAdmin.listReencryptionStatus();
+ assertTrue(it.hasNext());
+ ZoneReencryptionStatus zs = it.next();
+ assertEquals(zone.toString(), zs.getZoneName());
+ assertEquals(ZoneReencryptionStatus.State.Completed, zs.getState());
+ assertTrue(zs.getCompletionTime() > 0);
+ assertTrue(zs.getCompletionTime() > zs.getSubmissionTime());
+ assertEquals(5, zs.getFilesReencrypted());
+ assertEquals(1, zs.getNumReencryptionFailures());
+ }
+
+
+ @Test
+ public void testReencryptionUpdaterFaultCkpt() throws Exception {
+ class MyInjector extends EncryptionFaultInjector {
+ private volatile int exceptionCount = 0;
+
+ MyInjector(int numFailures) {
+ exceptionCount = numFailures;
+ }
+
+ @Override
+ public void reencryptUpdaterProcessCheckpoint() throws IOException {
+ if (exceptionCount > 0) {
+ --exceptionCount;
+ throw new IOException("Injected process checkpoint failure");
+ }
+ }
+ }
+ final MyInjector injector = new MyInjector(1);
+ EncryptionFaultInjector.instance = injector;
+ /* Setup test dir:
+ * /zones/zone/[0-9]
+ */
+ final int len = 8196;
+ final Path zoneParent = new Path("/zones");
+ final Path zone = new Path(zoneParent, "zone");
+ fsWrapper.mkdir(zone, FsPermission.getDirDefault(), true);
+ dfsAdmin.createEncryptionZone(zone, TEST_KEY, NO_TRASH);
+ for (int i = 0; i < 10; ++i) {
+ DFSTestUtil
+ .createFile(fs, new Path(zone, Integer.toString(i)), len, (short) 1,
+ 0xFEED);
+ }
+
+ // re-encrypt the zone
+ fsn.getProvider().rollNewVersion(TEST_KEY);
+ fsn.getProvider().flush();
+
+ dfsAdmin.reencryptEncryptionZone(zone, ReencryptAction.START);
+ waitForReencryptedZones(1);
+ assertEquals(0, injector.exceptionCount);
+
+ // test listReencryptionStatus should still work
+ RemoteIterator<ZoneReencryptionStatus> it =
+ dfsAdmin.listReencryptionStatus();
+ assertTrue(it.hasNext());
+ ZoneReencryptionStatus zs = it.next();
+ assertEquals(zone.toString(), zs.getZoneName());
+ assertEquals(ZoneReencryptionStatus.State.Completed, zs.getState());
+ assertTrue(zs.getCompletionTime() > 0);
+ assertTrue(zs.getCompletionTime() > zs.getSubmissionTime());
+ assertEquals(10, zs.getFilesReencrypted());
+ assertEquals(1, zs.getNumReencryptionFailures());
+ }
+
+ @Test
+ public void testReencryptionUpdaterFaultRecover() throws Exception {
+ class MyInjector extends EncryptionFaultInjector {
+ private volatile int exceptionCount = 0;
+
+ MyInjector(int oneTask) {
+ exceptionCount = oneTask;
+ }
+
+ @Override
+ public void reencryptUpdaterProcessOneTask() throws IOException {
+ if (exceptionCount > 0) {
+ --exceptionCount;
+ throw new RetriableException("Injected process task failure");
+ }
+ }
+ }
+ final MyInjector injector = new MyInjector(10);
+ EncryptionFaultInjector.instance = injector;
+ /* Setup test dir:
+ * /zones/zone/[0-9]
+ */
+ final int len = 8196;
+ final Path zoneParent = new Path("/zones");
+ final Path zone = new Path(zoneParent, "zone");
+ fsWrapper.mkdir(zone, FsPermission.getDirDefault(), true);
+ dfsAdmin.createEncryptionZone(zone, TEST_KEY, NO_TRASH);
+ for (int i = 0; i < 10; ++i) {
+ DFSTestUtil
+ .createFile(fs, new Path(zone, Integer.toString(i)), len, (short) 1,
+ 0xFEED);
+ }
+
+ // re-encrypt the zone
+ fsn.getProvider().rollNewVersion(TEST_KEY);
+ fsn.getProvider().flush();
+
+ final EncryptionZoneManager ezm = getEzManager();
+ final ReencryptionHandler handler = (ReencryptionHandler) Whitebox
+ .getInternalState(ezm, "reencryptionHandler");
+ final ReencryptionUpdater updater = (ReencryptionUpdater) Whitebox
+ .getInternalState(handler, "reencryptionUpdater");
+ Whitebox.setInternalState(updater, "faultRetryInterval", 50);
+ dfsAdmin.reencryptEncryptionZone(zone, ReencryptAction.START);
+ waitForReencryptedZones(1);
+ assertEquals(0, injector.exceptionCount);
+
+ // test listReencryptionStatus should still work
+ RemoteIterator<ZoneReencryptionStatus> it =
+ dfsAdmin.listReencryptionStatus();
+ assertTrue(it.hasNext());
+ ZoneReencryptionStatus zs = it.next();
+ assertEquals(zone.toString(), zs.getZoneName());
+ assertEquals(ZoneReencryptionStatus.State.Completed, zs.getState());
+ assertTrue(zs.getCompletionTime() > 0);
+ assertTrue(zs.getCompletionTime() > zs.getSubmissionTime());
+ assertEquals(10, zs.getFilesReencrypted());
+ assertEquals(0, zs.getNumReencryptionFailures());
+ }
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[04/50] [abbrv] hadoop git commit: HADOOP-14773. Extend ZKCuratorManager API for more reusability. (Íñigo Goiri via Subru).
Posted by as...@apache.org.
HADOOP-14773. Extend ZKCuratorManager API for more reusability. (Íñigo Goiri via Subru).
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/75dd866b
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/75dd866b
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/75dd866b
Branch: refs/heads/YARN-5972
Commit: 75dd866bfb8b63cb9f13179d4365b05c48e0907d
Parents: f34646d
Author: Subru Krishnan <su...@apache.org>
Authored: Tue Aug 15 16:53:59 2017 -0700
Committer: Subru Krishnan <su...@apache.org>
Committed: Tue Aug 15 16:53:59 2017 -0700
----------------------------------------------------------------------
.../hadoop/util/curator/ZKCuratorManager.java | 54 ++++++++++++++++++--
.../util/curator/TestZKCuratorManager.java | 2 +-
.../recovery/ZKRMStateStore.java | 19 +------
3 files changed, 52 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/75dd866b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/curator/ZKCuratorManager.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/curator/ZKCuratorManager.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/curator/ZKCuratorManager.java
index 3adf028..9a031af 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/curator/ZKCuratorManager.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/util/curator/ZKCuratorManager.java
@@ -33,9 +33,12 @@ import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.util.ZKUtil;
import org.apache.zookeeper.CreateMode;
import org.apache.zookeeper.data.ACL;
+import org.apache.zookeeper.data.Stat;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+import com.google.common.base.Preconditions;
+
/**
* Helper class that provides utility methods specific to ZK operations.
*/
@@ -179,7 +182,6 @@ public final class ZKCuratorManager {
/**
* Get the data in a ZNode.
* @param path Path of the ZNode.
- * @param stat Output statistics of the ZNode.
* @return The data in the ZNode.
* @throws Exception If it cannot contact Zookeeper.
*/
@@ -190,16 +192,38 @@ public final class ZKCuratorManager {
/**
* Get the data in a ZNode.
* @param path Path of the ZNode.
- * @param stat Output statistics of the ZNode.
+ * @param stat
+ * @return The data in the ZNode.
+ * @throws Exception If it cannot contact Zookeeper.
+ */
+ public byte[] getData(final String path, Stat stat) throws Exception {
+ return curator.getData().storingStatIn(stat).forPath(path);
+ }
+
+ /**
+ * Get the data in a ZNode.
+ * @param path Path of the ZNode.
* @return The data in the ZNode.
* @throws Exception If it cannot contact Zookeeper.
*/
- public String getSringData(final String path) throws Exception {
+ public String getStringData(final String path) throws Exception {
byte[] bytes = getData(path);
return new String(bytes, Charset.forName("UTF-8"));
}
/**
+ * Get the data in a ZNode.
+ * @param path Path of the ZNode.
+ * @param stat Output statistics of the ZNode.
+ * @return The data in the ZNode.
+ * @throws Exception If it cannot contact Zookeeper.
+ */
+ public String getStringData(final String path, Stat stat) throws Exception {
+ byte[] bytes = getData(path, stat);
+ return new String(bytes, Charset.forName("UTF-8"));
+ }
+
+ /**
* Set data into a ZNode.
* @param path Path of the ZNode.
* @param data Data to set.
@@ -272,14 +296,36 @@ public final class ZKCuratorManager {
}
/**
+ * Utility function to ensure that the configured base znode exists.
+ * This recursively creates the znode as well as all of its parents.
+ * @param path Path of the znode to create.
+ * @throws Exception If it cannot create the file.
+ */
+ public void createRootDirRecursively(String path) throws Exception {
+ String[] pathParts = path.split("/");
+ Preconditions.checkArgument(
+ pathParts.length >= 1 && pathParts[0].isEmpty(),
+ "Invalid path: %s", path);
+ StringBuilder sb = new StringBuilder();
+
+ for (int i = 1; i < pathParts.length; i++) {
+ sb.append("/").append(pathParts[i]);
+ create(sb.toString());
+ }
+ }
+
+ /**
* Delete a ZNode.
* @param path Path of the ZNode.
+ * @return If the znode was deleted.
* @throws Exception If it cannot contact ZooKeeper.
*/
- public void delete(final String path) throws Exception {
+ public boolean delete(final String path) throws Exception {
if (exists(path)) {
curator.delete().deletingChildrenIfNeeded().forPath(path);
+ return true;
}
+ return false;
}
/**
http://git-wip-us.apache.org/repos/asf/hadoop/blob/75dd866b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/curator/TestZKCuratorManager.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/curator/TestZKCuratorManager.java b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/curator/TestZKCuratorManager.java
index 2bcf508..3e78a44 100644
--- a/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/curator/TestZKCuratorManager.java
+++ b/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/util/curator/TestZKCuratorManager.java
@@ -67,7 +67,7 @@ public class TestZKCuratorManager {
curator.create(testZNode);
assertTrue(curator.exists(testZNode));
curator.setData(testZNode, expectedString, -1);
- String testString = curator.getSringData("/test");
+ String testString = curator.getStringData("/test");
assertEquals(expectedString, testString);
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/75dd866b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
index 4b6e82c..a445e75 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/ZKRMStateStore.java
@@ -19,7 +19,6 @@
package org.apache.hadoop.yarn.server.resourcemanager.recovery;
import com.google.common.annotations.VisibleForTesting;
-import com.google.common.base.Preconditions;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.curator.framework.CuratorFramework;
@@ -338,7 +337,7 @@ public class ZKRMStateStore extends RMStateStore {
@Override
public synchronized void startInternal() throws Exception {
// ensure root dirs exist
- createRootDirRecursively(znodeWorkingPath);
+ zkManager.createRootDirRecursively(znodeWorkingPath);
create(zkRootNodePath);
setRootNodeAcls();
delete(fencingNodePath);
@@ -1147,22 +1146,6 @@ public class ZKRMStateStore extends RMStateStore {
}
/**
- * Utility function to ensure that the configured base znode exists.
- * This recursively creates the znode as well as all of its parents.
- */
- private void createRootDirRecursively(String path) throws Exception {
- String pathParts[] = path.split("/");
- Preconditions.checkArgument(pathParts.length >= 1 && pathParts[0].isEmpty(),
- "Invalid path: %s", path);
- StringBuilder sb = new StringBuilder();
-
- for (int i = 1; i < pathParts.length; i++) {
- sb.append("/").append(pathParts[i]);
- create(sb.toString());
- }
- }
-
- /**
* Get alternate path for app id if path according to configured split index
* does not exist. We look for path based on all possible split indices.
* @param appId
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[02/50] [abbrv] hadoop git commit: YARN-7014. Fix off-by-one error
causing heap corruption (Jason Lowe via nroberts)
Posted by as...@apache.org.
YARN-7014. Fix off-by-one error causing heap corruption (Jason Lowe via nroberts)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/d2654590
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/d2654590
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/d2654590
Branch: refs/heads/YARN-5972
Commit: d265459024b8e5f5eccf421627f684ca8f162112
Parents: dadb0c2
Author: Nathan Roberts <nr...@apache.org>
Authored: Tue Aug 15 15:52:48 2017 -0500
Committer: Nathan Roberts <nr...@apache.org>
Committed: Tue Aug 15 15:52:48 2017 -0500
----------------------------------------------------------------------
.../src/main/native/container-executor/impl/utils/string-utils.c | 3 +--
1 file changed, 1 insertion(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d2654590/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/string-utils.c
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/string-utils.c b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/string-utils.c
index 703d484..063df7e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/string-utils.c
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/native/container-executor/impl/utils/string-utils.c
@@ -44,8 +44,7 @@ int validate_container_id(const char* input) {
* container_e17_1410901177871_0001_01_000005
* container_1410901177871_0001_01_000005
*/
- char* input_cpy = malloc(strlen(input));
- strcpy(input_cpy, input);
+ char* input_cpy = strdup(input);
char* p = strtok(input_cpy, "_");
int idx = 0;
while (p != NULL) {
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[19/50] [abbrv] hadoop git commit: HADOOP-14769. WASB: delete
recursive should not fail if a file is deleted. Contributed by Thomas
Marquardt
Posted by as...@apache.org.
HADOOP-14769. WASB: delete recursive should not fail if a file is deleted.
Contributed by Thomas Marquardt
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/c6b4e656
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/c6b4e656
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/c6b4e656
Branch: refs/heads/YARN-5972
Commit: c6b4e656b76b68cc1d0dbcc15a5aa5ea23335b7b
Parents: 99e558b
Author: Steve Loughran <st...@apache.org>
Authored: Fri Aug 18 14:13:40 2017 +0100
Committer: Steve Loughran <st...@apache.org>
Committed: Fri Aug 18 14:13:40 2017 +0100
----------------------------------------------------------------------
.../fs/azure/AzureNativeFileSystemStore.java | 21 ++++---
.../hadoop/fs/azure/NativeAzureFileSystem.java | 47 ++++++++-------
.../TestFileSystemOperationsWithThreads.java | 61 ++++++++++++++++----
3 files changed, 86 insertions(+), 43 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c6b4e656/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
index 554027b..b0cd701 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/AzureNativeFileSystemStore.java
@@ -2459,8 +2459,11 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore {
try {
blob.delete(operationContext, lease);
} catch (StorageException e) {
- LOG.error("Encountered Storage Exception for delete on Blob: {}, Exception Details: {} Error Code: {}",
- blob.getUri(), e.getMessage(), e.getErrorCode());
+ if (!NativeAzureFileSystemHelper.isFileNotFoundException(e)) {
+ LOG.error("Encountered Storage Exception for delete on Blob: {}"
+ + ", Exception Details: {} Error Code: {}",
+ blob.getUri(), e.getMessage(), e.getErrorCode());
+ }
// On exception, check that if:
// 1. It's a BlobNotFound exception AND
// 2. It got there after one-or-more retries THEN
@@ -2491,17 +2494,17 @@ public class AzureNativeFileSystemStore implements NativeFileSystemStore {
// Container doesn't exist, no need to do anything
return true;
}
-
// Get the blob reference and delete it.
CloudBlobWrapper blob = getBlobReference(key);
- if (blob.exists(getInstrumentedContext())) {
- safeDelete(blob, lease);
- return true;
- } else {
+ safeDelete(blob, lease);
+ return true;
+ } catch (Exception e) {
+ if (e instanceof StorageException
+ && NativeAzureFileSystemHelper.isFileNotFoundException(
+ (StorageException) e)) {
+ // the file or directory does not exist
return false;
}
- } catch (Exception e) {
- // Re-throw as an Azure storage exception.
throw new AzureException(e);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c6b4e656/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
index a7558a3..2abc6c6 100644
--- a/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
+++ b/hadoop-tools/hadoop-azure/src/main/java/org/apache/hadoop/fs/azure/NativeAzureFileSystem.java
@@ -2043,7 +2043,12 @@ public class NativeAzureFileSystem extends FileSystem {
AzureFileSystemThreadTask task = new AzureFileSystemThreadTask() {
@Override
public boolean execute(FileMetadata file) throws IOException{
- return deleteFile(file.getKey(), file.isDir());
+ if (!deleteFile(file.getKey(), file.isDir())) {
+ LOG.warn("Attempt to delete non-existent {} {}",
+ file.isDir() ? "directory" : "file",
+ file.getKey());
+ }
+ return true;
}
};
@@ -2080,30 +2085,28 @@ public class NativeAzureFileSystem extends FileSystem {
return new AzureFileSystemThreadPoolExecutor(threadCount, threadNamePrefix, operation, key, config);
}
- // Delete single file / directory from key.
+ /**
+ * Delete the specified file or directory and increment metrics.
+ * If the file or directory does not exist, the operation returns false.
+ * @param path the path to a file or directory.
+ * @param isDir true if the path is a directory; otherwise false.
+ * @return true if delete is successful; otherwise false.
+ * @throws IOException if an IO error occurs while attempting to delete the
+ * path.
+ *
+ */
@VisibleForTesting
- boolean deleteFile(String key, boolean isDir) throws IOException {
- try {
- if (store.delete(key)) {
- if (isDir) {
- instrumentation.directoryDeleted();
- } else {
- instrumentation.fileDeleted();
- }
- return true;
- } else {
- return false;
- }
- } catch(IOException e) {
- Throwable innerException = NativeAzureFileSystemHelper.checkForAzureStorageException(e);
-
- if (innerException instanceof StorageException
- && NativeAzureFileSystemHelper.isFileNotFoundException((StorageException) innerException)) {
- return false;
- }
+ boolean deleteFile(String path, boolean isDir) throws IOException {
+ if (!store.delete(path)) {
+ return false;
+ }
- throw e;
+ if (isDir) {
+ instrumentation.directoryDeleted();
+ } else {
+ instrumentation.fileDeleted();
}
+ return true;
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/c6b4e656/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestFileSystemOperationsWithThreads.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestFileSystemOperationsWithThreads.java b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestFileSystemOperationsWithThreads.java
index ce3cdee..fd3690c 100644
--- a/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestFileSystemOperationsWithThreads.java
+++ b/hadoop-tools/hadoop-azure/src/test/java/org/apache/hadoop/fs/azure/TestFileSystemOperationsWithThreads.java
@@ -39,6 +39,8 @@ import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.ExpectedException;
import org.mockito.Mockito;
+import org.mockito.invocation.InvocationOnMock;
+import org.mockito.stubbing.Answer;
/**
* Tests the Native Azure file system (WASB) using parallel threads for rename and delete operations.
@@ -529,30 +531,65 @@ public class TestFileSystemOperationsWithThreads extends AbstractWasbTestBase {
}
/*
- * Test case for delete operation with multiple threads and flat listing enabled.
+ * Validate that when a directory is deleted recursively, the operation succeeds
+ * even if a child directory delete fails because the directory does not exist.
+ * This can happen if a child directory is deleted by an external agent while
+ * the parent is in progress of being deleted recursively.
+ */
+ @Test
+ public void testRecursiveDirectoryDeleteWhenChildDirectoryDeleted()
+ throws Exception {
+ testRecusiveDirectoryDelete(true);
+ }
+
+ /*
+ * Validate that when a directory is deleted recursively, the operation succeeds
+ * even if a file delete fails because it does not exist.
+ * This can happen if a file is deleted by an external agent while
+ * the parent directory is in progress of being deleted.
*/
@Test
- public void testDeleteSingleDeleteFailure() throws Exception {
+ public void testRecursiveDirectoryDeleteWhenDeletingChildFileReturnsFalse()
+ throws Exception {
+ testRecusiveDirectoryDelete(false);
+ }
+ private void testRecusiveDirectoryDelete(boolean useDir) throws Exception {
+ String childPathToBeDeletedByExternalAgent = (useDir)
+ ? "root/0"
+ : "root/0/fileToRename";
// Spy azure file system object and return false for deleting one file
- LOG.info("testDeleteSingleDeleteFailure");
NativeAzureFileSystem mockFs = Mockito.spy((NativeAzureFileSystem) fs);
- String path = mockFs.pathToKey(mockFs.makeAbsolute(new Path("root/0")));
- Mockito.when(mockFs.deleteFile(path, true)).thenReturn(false);
+ String path = mockFs.pathToKey(mockFs.makeAbsolute(new Path(
+ childPathToBeDeletedByExternalAgent)));
+
+ Answer<Boolean> answer = new Answer<Boolean>() {
+ public Boolean answer(InvocationOnMock invocation) throws Throwable {
+ String path = (String) invocation.getArguments()[0];
+ boolean isDir = (boolean) invocation.getArguments()[1];
+ boolean realResult = fs.deleteFile(path, isDir);
+ assertTrue(realResult);
+ boolean fakeResult = false;
+ return fakeResult;
+ }
+ };
+
+ Mockito.when(mockFs.deleteFile(path, useDir)).thenAnswer(answer);
createFolder(mockFs, "root");
Path sourceFolder = new Path("root");
- assertFalse(mockFs.delete(sourceFolder, true));
- assertTrue(mockFs.exists(sourceFolder));
- // Validate from logs that threads are enabled and delete operation failed.
+ assertTrue(mockFs.delete(sourceFolder, true));
+ assertFalse(mockFs.exists(sourceFolder));
+
+ // Validate from logs that threads are enabled, that a child directory was
+ // deleted by an external caller, and the parent delete operation still
+ // succeeds.
String content = logs.getOutput();
assertInLog(content,
"Using thread pool for Delete operation with threads");
- assertInLog(content, "Delete operation failed for file " + path);
- assertInLog(content,
- "Terminating execution of Delete operation now as some other thread already got exception or operation failed");
- assertInLog(content, "Failed to delete files / subfolders in blob");
+ assertInLog(content, String.format("Attempt to delete non-existent %s %s",
+ useDir ? "directory" : "file", path));
}
/*
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[33/50] [abbrv] hadoop git commit: YARN-7047. Moving logging APIs
over to slf4j in hadoop-yarn-server-nodemanager. Contributed by Yeliang Cang.
Posted by as...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5ff57a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerCommandExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerCommandExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerCommandExecutor.java
index 9026d22..5739912 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerCommandExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/docker/DockerCommandExecutor.java
@@ -16,13 +16,13 @@
*/
package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime.docker;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperation;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationException;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged.PrivilegedOperationExecutor;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.runtime.ContainerExecutionException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.util.Map;
@@ -30,7 +30,8 @@ import java.util.Map;
* Utility class for executing common docker operations.
*/
public final class DockerCommandExecutor {
- private static final Log LOG = LogFactory.getLog(DockerCommandExecutor.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(DockerCommandExecutor.class);
/**
* Potential states that the docker status can return.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5ff57a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java
index bb4b7f3..2378c45 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java
@@ -18,6 +18,8 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer;
import static org.apache.hadoop.util.Shell.getAllShells;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.io.DataInputStream;
import java.io.File;
@@ -44,8 +46,6 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileContext;
@@ -86,7 +86,8 @@ import com.google.common.util.concurrent.ThreadFactoryBuilder;
public class ContainerLocalizer {
- static final Log LOG = LogFactory.getLog(ContainerLocalizer.class);
+ static final Logger LOG =
+ LoggerFactory.getLogger(ContainerLocalizer.class);
public static final String FILECACHE = "filecache";
public static final String APPCACHE = "appcache";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5ff57a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java
index 47e6a55..dd31543 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java
@@ -26,9 +26,9 @@ import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.atomic.AtomicLong;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.api.records.ApplicationId;
@@ -58,7 +58,8 @@ import com.google.common.annotations.VisibleForTesting;
class LocalResourcesTrackerImpl implements LocalResourcesTracker {
- static final Log LOG = LogFactory.getLog(LocalResourcesTrackerImpl.class);
+ static final Logger LOG =
+ LoggerFactory.getLogger(LocalResourcesTrackerImpl.class);
private static final String RANDOM_DIR_REGEX = "-?\\d+";
private static final Pattern RANDOM_DIR_PATTERN = Pattern
.compile(RANDOM_DIR_REGEX);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5ff57a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalizedResource.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalizedResource.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalizedResource.java
index 98f4e21..bd9602e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalizedResource.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalizedResource.java
@@ -24,9 +24,9 @@ import java.util.concurrent.atomic.AtomicLong;
import java.util.concurrent.locks.Lock;
import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.event.Dispatcher;
@@ -53,7 +53,8 @@ import org.apache.hadoop.yarn.state.StateMachineFactory;
*/
public class LocalizedResource implements EventHandler<ResourceEvent> {
- private static final Log LOG = LogFactory.getLog(LocalizedResource.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(LocalizedResource.class);
volatile Path localPath;
volatile long size = -1;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5ff57a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
index 5bc0da7..c37f2e3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
@@ -19,6 +19,8 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer;
import static org.apache.hadoop.fs.CreateFlag.CREATE;
import static org.apache.hadoop.fs.CreateFlag.OVERWRITE;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.io.DataOutputStream;
import java.io.File;
@@ -50,8 +52,6 @@ import java.util.concurrent.ThreadFactory;
import java.util.concurrent.TimeUnit;
import org.apache.commons.codec.digest.DigestUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
@@ -148,7 +148,8 @@ import com.google.common.util.concurrent.ThreadFactoryBuilder;
public class ResourceLocalizationService extends CompositeService
implements EventHandler<LocalizationEvent>, LocalizationProtocol {
- private static final Log LOG = LogFactory.getLog(ResourceLocalizationService.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ResourceLocalizationService.class);
public static final String NM_PRIVATE_DIR = "nmPrivate";
public static final FsPermission NM_PRIVATE_PERM = new FsPermission((short) 0700);
@@ -956,7 +957,7 @@ public class ResourceLocalizationService extends CompositeService
}
}
} catch(Throwable t) {
- LOG.fatal("Error: Shutting down", t);
+ LOG.error("Error: Shutting down", t);
} finally {
LOG.info("Public cache exiting");
threadPool.shutdownNow();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5ff57a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceSet.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceSet.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceSet.java
index 5da3abc..6df1073 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceSet.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceSet.java
@@ -18,11 +18,11 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.api.records.LocalResource;
import org.apache.hadoop.yarn.api.records.LocalResourceVisibility;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.net.URISyntaxException;
import java.util.ArrayList;
@@ -40,7 +40,8 @@ import java.util.concurrent.ConcurrentHashMap;
*/
public class ResourceSet {
- private static final Log LOG = LogFactory.getLog(ResourceSet.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ResourceSet.class);
// resources by localization state (localized, pending, failed)
private Map<String, Path> localizedResources =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5ff57a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/security/LocalizerSecurityInfo.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/security/LocalizerSecurityInfo.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/security/LocalizerSecurityInfo.java
index bd204c4..423b528 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/security/LocalizerSecurityInfo.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/security/LocalizerSecurityInfo.java
@@ -19,9 +19,9 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.security;
import java.lang.annotation.Annotation;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.KerberosInfo;
import org.apache.hadoop.security.SecurityInfo;
@@ -32,7 +32,8 @@ import org.apache.hadoop.yarn.server.nodemanager.api.LocalizationProtocolPB;
public class LocalizerSecurityInfo extends SecurityInfo {
- private static final Log LOG = LogFactory.getLog(LocalizerSecurityInfo.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(LocalizerSecurityInfo.class);
@Override
public KerberosInfo getKerberosInfo(Class<?> protocol, Configuration conf) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5ff57a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/security/LocalizerTokenSelector.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/security/LocalizerTokenSelector.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/security/LocalizerTokenSelector.java
index 7c4f982..89787af 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/security/LocalizerTokenSelector.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/security/LocalizerTokenSelector.java
@@ -19,9 +19,9 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.security;
import java.util.Collection;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.io.Text;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.security.token.TokenIdentifier;
@@ -30,8 +30,8 @@ import org.apache.hadoop.security.token.TokenSelector;
public class LocalizerTokenSelector implements
TokenSelector<LocalizerTokenIdentifier> {
- private static final Log LOG = LogFactory
- .getLog(LocalizerTokenSelector.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(LocalizerTokenSelector.class);
@SuppressWarnings("unchecked")
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5ff57a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/sharedcache/SharedCacheUploadService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/sharedcache/SharedCacheUploadService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/sharedcache/SharedCacheUploadService.java
index 16c36eb..9afbf3f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/sharedcache/SharedCacheUploadService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/sharedcache/SharedCacheUploadService.java
@@ -22,9 +22,9 @@ import java.io.IOException;
import java.net.InetSocketAddress;
import java.util.Map;
import java.util.concurrent.ExecutorService;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.conf.Configuration;
@@ -50,8 +50,8 @@ import com.google.common.util.concurrent.ThreadFactoryBuilder;
*/
public class SharedCacheUploadService extends AbstractService implements
EventHandler<SharedCacheUploadEvent> {
- private static final Log LOG =
- LogFactory.getLog(SharedCacheUploadService.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(SharedCacheUploadService.class);
private boolean enabled;
private FileSystem fs;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5ff57a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/sharedcache/SharedCacheUploader.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/sharedcache/SharedCacheUploader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/sharedcache/SharedCacheUploader.java
index e077f89..23aa5b3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/sharedcache/SharedCacheUploader.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/sharedcache/SharedCacheUploader.java
@@ -24,9 +24,9 @@ import java.lang.reflect.UndeclaredThrowableException;
import java.net.URISyntaxException;
import java.util.concurrent.Callable;
import java.util.concurrent.ThreadLocalRandom;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
@@ -60,7 +60,8 @@ class SharedCacheUploader implements Callable<Boolean> {
static final FsPermission FILE_PERMISSION =
new FsPermission((short)00555);
- private static final Log LOG = LogFactory.getLog(SharedCacheUploader.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(SharedCacheUploader.class);
private final LocalResource resource;
private final Path localPath;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5ff57a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
index 0d9e686..1601c3f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/AppLogAggregatorImpl.java
@@ -32,9 +32,9 @@ import java.util.Set;
import java.util.concurrent.BlockingQueue;
import java.util.concurrent.LinkedBlockingQueue;
import java.util.concurrent.atomic.AtomicBoolean;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileContext;
@@ -83,8 +83,8 @@ import com.google.common.collect.Sets;
public class AppLogAggregatorImpl implements AppLogAggregator {
- private static final Log LOG = LogFactory
- .getLog(AppLogAggregatorImpl.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(AppLogAggregatorImpl.class);
private static final int THREAD_SLEEP_TIME = 1000;
// This is temporary solution. The configuration will be deleted once
// we find a more scalable method to only write a single log file per LRS.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5ff57a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java
index 38315fd..aafd7d8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/LogAggregationService.java
@@ -26,9 +26,9 @@ import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.TimeUnit;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileContext;
@@ -69,7 +69,8 @@ import com.google.common.util.concurrent.ThreadFactoryBuilder;
public class LogAggregationService extends AbstractService implements
LogHandler {
- private static final Log LOG = LogFactory.getLog(LogAggregationService.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(LogAggregationService.class);
private static final long MIN_LOG_ROLLING_INTERVAL = 3600;
// This configuration is for debug and test purpose. By setting
// this configuration as true. We can break the lower bound of
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5ff57a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/SampleContainerLogAggregationPolicy.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/SampleContainerLogAggregationPolicy.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/SampleContainerLogAggregationPolicy.java
index 56c760b..03f548c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/SampleContainerLogAggregationPolicy.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/SampleContainerLogAggregationPolicy.java
@@ -19,8 +19,8 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager.logaggregation;
import java.util.Collection;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.api.records.ContainerId;
@@ -41,8 +41,8 @@ import org.apache.hadoop.yarn.server.api.ContainerType;
@Private
public class SampleContainerLogAggregationPolicy implements
ContainerLogAggregationPolicy {
- private static final Log LOG =
- LogFactory.getLog(SampleContainerLogAggregationPolicy.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(SampleContainerLogAggregationPolicy.class);
static String SAMPLE_RATE = "SR";
public static final float DEFAULT_SAMPLE_RATE = 0.2f;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5ff57a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/NonAggregatingLogHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/NonAggregatingLogHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/NonAggregatingLogHandler.java
index 9961748..9c43dde 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/NonAggregatingLogHandler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/loghandler/NonAggregatingLogHandler.java
@@ -26,9 +26,9 @@ import java.util.concurrent.ScheduledThreadPoolExecutor;
import java.util.concurrent.ThreadFactory;
import java.util.concurrent.TimeUnit;
import java.util.concurrent.RejectedExecutionException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.Path;
@@ -60,8 +60,8 @@ import com.google.common.util.concurrent.ThreadFactoryBuilder;
public class NonAggregatingLogHandler extends AbstractService implements
LogHandler {
- private static final Log LOG = LogFactory
- .getLog(NonAggregatingLogHandler.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(NonAggregatingLogHandler.class);
private final Dispatcher dispatcher;
private final DeletionService delService;
private final Map<ApplicationId, String> appOwners;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5ff57a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
index 13e7491..d764e1d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/ContainersMonitorImpl.java
@@ -20,8 +20,8 @@ package org.apache.hadoop.yarn.server.nodemanager.containermanager.monitor;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.service.AbstractService;
@@ -55,8 +55,8 @@ import java.util.concurrent.ConcurrentHashMap;
public class ContainersMonitorImpl extends AbstractService implements
ContainersMonitor {
- private final static Log LOG = LogFactory
- .getLog(ContainersMonitorImpl.class);
+ private final static Logger LOG =
+ LoggerFactory.getLogger(ContainersMonitorImpl.class);
private long monitoringInterval;
private MonitoringThread monitoringThread;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5ff57a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/ConfigurationNodeLabelsProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/ConfigurationNodeLabelsProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/ConfigurationNodeLabelsProvider.java
index 8a385905..7490cc2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/ConfigurationNodeLabelsProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/ConfigurationNodeLabelsProvider.java
@@ -20,9 +20,9 @@ package org.apache.hadoop.yarn.server.nodemanager.nodelabels;
import java.io.IOException;
import java.util.TimerTask;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -31,8 +31,8 @@ import org.apache.hadoop.yarn.conf.YarnConfiguration;
*/
public class ConfigurationNodeLabelsProvider extends AbstractNodeLabelsProvider {
- private static final Log LOG = LogFactory
- .getLog(ConfigurationNodeLabelsProvider.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ConfigurationNodeLabelsProvider.class);
public ConfigurationNodeLabelsProvider() {
super("Configuration Based NodeLabels Provider");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5ff57a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/ScriptBasedNodeLabelsProvider.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/ScriptBasedNodeLabelsProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/ScriptBasedNodeLabelsProvider.java
index 79d1eb9..32f180a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/ScriptBasedNodeLabelsProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/nodelabels/ScriptBasedNodeLabelsProvider.java
@@ -25,9 +25,9 @@ import java.util.Arrays;
import java.util.Set;
import java.util.Timer;
import java.util.TimerTask;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.util.Shell.ShellCommandExecutor;
@@ -132,7 +132,8 @@ public class ScriptBasedNodeLabelsProvider extends AbstractNodeLabelsProvider {
*/
private class NodeLabelsScriptRunner extends TimerTask {
- private final Log LOG = LogFactory.getLog(NodeLabelsScriptRunner.class);
+ private final Logger LOG =
+ LoggerFactory.getLogger(NodeLabelsScriptRunner.class);
public NodeLabelsScriptRunner() {
ArrayList<String> execScript = new ArrayList<String>();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5ff57a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java
index c556b39..b9f84e3 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java
@@ -20,6 +20,7 @@ package org.apache.hadoop.yarn.server.nodemanager.recovery;
import static org.fusesource.leveldbjni.JniDBFactory.asString;
import static org.fusesource.leveldbjni.JniDBFactory.bytes;
+import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
@@ -34,8 +35,6 @@ import java.util.Timer;
import java.util.TimerTask;
import java.util.Set;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -69,7 +68,6 @@ import org.fusesource.leveldbjni.JniDBFactory;
import org.fusesource.leveldbjni.internal.NativeDB;
import org.iq80.leveldb.DB;
import org.iq80.leveldb.DBException;
-import org.iq80.leveldb.Logger;
import org.iq80.leveldb.Options;
import org.iq80.leveldb.WriteBatch;
@@ -79,8 +77,8 @@ import com.google.common.collect.ListMultimap;
public class NMLeveldbStateStoreService extends NMStateStoreService {
- public static final Log LOG =
- LogFactory.getLog(NMLeveldbStateStoreService.class);
+ public static final org.slf4j.Logger LOG =
+ LoggerFactory.getLogger(NMLeveldbStateStoreService.class);
private static final String DB_NAME = "yarn-nm-state";
private static final String DB_SCHEMA_VERSION_KEY = "nm-schema-version";
@@ -1382,8 +1380,9 @@ public class NMLeveldbStateStoreService extends NMStateStoreService {
}
}
- private static class LeveldbLogger implements Logger {
- private static final Log LOG = LogFactory.getLog(LeveldbLogger.class);
+ private static class LeveldbLogger implements org.iq80.leveldb.Logger {
+ private static final org.slf4j.Logger LOG =
+ LoggerFactory.getLogger(LeveldbLogger.class);
@Override
public void log(String message) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5ff57a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/security/NMContainerTokenSecretManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/security/NMContainerTokenSecretManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/security/NMContainerTokenSecretManager.java
index 2a92d40..256f649 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/security/NMContainerTokenSecretManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/security/NMContainerTokenSecretManager.java
@@ -24,9 +24,9 @@ import java.util.Iterator;
import java.util.List;
import java.util.Map.Entry;
import java.util.TreeMap;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.token.SecretManager;
@@ -48,8 +48,8 @@ import org.apache.hadoop.yarn.server.security.MasterKeyData;
public class NMContainerTokenSecretManager extends
BaseContainerTokenSecretManager {
- private static final Log LOG = LogFactory
- .getLog(NMContainerTokenSecretManager.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(NMContainerTokenSecretManager.class);
private MasterKeyData previousMasterKey;
private final TreeMap<Long, List<ContainerId>> recentlyStartedContainerTracker;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5ff57a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/security/NMTokenSecretManagerInNM.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/security/NMTokenSecretManagerInNM.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/security/NMTokenSecretManagerInNM.java
index 4ed6118..0956e77 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/security/NMTokenSecretManagerInNM.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/security/NMTokenSecretManagerInNM.java
@@ -23,9 +23,9 @@ import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
@@ -45,8 +45,8 @@ import com.google.common.annotations.VisibleForTesting;
public class NMTokenSecretManagerInNM extends BaseNMTokenSecretManager {
- private static final Log LOG = LogFactory
- .getLog(NMTokenSecretManagerInNM.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(NMTokenSecretManagerInNM.class);
private MasterKeyData previousMasterKey;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5ff57a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java
index 8aaae79..a459958 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/timelineservice/NMTimelinePublisher.java
@@ -22,9 +22,9 @@ import java.io.IOException;
import java.util.HashMap;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.service.CompositeService;
import org.apache.hadoop.yarn.api.records.ApplicationId;
@@ -66,7 +66,8 @@ import com.google.common.annotations.VisibleForTesting;
*/
public class NMTimelinePublisher extends CompositeService {
- private static final Log LOG = LogFactory.getLog(NMTimelinePublisher.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(NMTimelinePublisher.class);
private Dispatcher dispatcher;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5ff57a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java
index 7a89285..54b6e1c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/CgroupsLCEResourcesHandler.java
@@ -37,11 +37,11 @@ import java.util.Map.Entry;
import java.util.Set;
import java.util.regex.Matcher;
import java.util.regex.Pattern;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import com.google.common.annotations.VisibleForTesting;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.io.IOUtils;
@@ -68,8 +68,8 @@ import org.apache.hadoop.yarn.util.SystemClock;
@Deprecated
public class CgroupsLCEResourcesHandler implements LCEResourcesHandler {
- final static Log LOG = LogFactory
- .getLog(CgroupsLCEResourcesHandler.class);
+ final static Logger LOG =
+ LoggerFactory.getLogger(CgroupsLCEResourcesHandler.class);
private Configuration conf;
private String cgroupPrefix;
@@ -435,7 +435,7 @@ public class CgroupsLCEResourcesHandler implements LCEResourcesHandler {
} catch (IOException e) {
throw new IOException("Error while reading " + getMtabFileName(), e);
} finally {
- IOUtils.cleanup(LOG, in);
+ IOUtils.cleanupWithLogger(LOG, in);
}
return ret;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5ff57a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/DefaultLCEResourcesHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/DefaultLCEResourcesHandler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/DefaultLCEResourcesHandler.java
index df2cc52..70c96af 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/DefaultLCEResourcesHandler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/DefaultLCEResourcesHandler.java
@@ -18,18 +18,18 @@
package org.apache.hadoop.yarn.server.nodemanager.util;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.server.nodemanager.LinuxContainerExecutor;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
@Deprecated
public class DefaultLCEResourcesHandler implements LCEResourcesHandler {
- final static Log LOG = LogFactory
- .getLog(DefaultLCEResourcesHandler.class);
+ final static Logger LOG =
+ LoggerFactory.getLogger(DefaultLCEResourcesHandler.class);
private Configuration conf;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5ff57a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/NodeManagerHardwareUtils.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/NodeManagerHardwareUtils.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/NodeManagerHardwareUtils.java
index 2726a41..32f73c8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/NodeManagerHardwareUtils.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/NodeManagerHardwareUtils.java
@@ -18,13 +18,13 @@
package org.apache.hadoop.yarn.server.nodemanager.util;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.util.ResourceCalculatorPlugin;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* Helper class to determine hardware related characteristics such as the
@@ -34,8 +34,8 @@ import org.apache.hadoop.yarn.util.ResourceCalculatorPlugin;
@InterfaceStability.Unstable
public class NodeManagerHardwareUtils {
- private static final Log LOG = LogFactory
- .getLog(NodeManagerHardwareUtils.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(NodeManagerHardwareUtils.class);
private static boolean isHardwareDetectionEnabled(Configuration conf) {
return conf.getBoolean(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5ff57a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/ProcessIdFileReader.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/ProcessIdFileReader.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/ProcessIdFileReader.java
index 41d299e..a4c04ab 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/ProcessIdFileReader.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/util/ProcessIdFileReader.java
@@ -22,9 +22,9 @@ import java.io.File;
import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStreamReader;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.util.Shell;
import org.apache.hadoop.yarn.api.records.ContainerId;
@@ -35,7 +35,8 @@ import org.apache.hadoop.yarn.util.ConverterUtils;
*/
public class ProcessIdFileReader {
- private static final Log LOG = LogFactory.getLog(ProcessIdFileReader.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(ProcessIdFileReader.class);
/**
* Get the process id from specified file path.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5ff57a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java
index 04a889f..7a6aa0f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NMWebServices.java
@@ -25,6 +25,8 @@ import java.nio.charset.Charset;
import java.util.ArrayList;
import java.util.List;
import java.util.Map.Entry;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import javax.servlet.http.HttpServletRequest;
import javax.servlet.http.HttpServletResponse;
@@ -43,8 +45,6 @@ import javax.ws.rs.core.StreamingOutput;
import javax.ws.rs.core.UriInfo;
import org.apache.commons.io.IOUtils;
import org.apache.commons.lang.StringUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
import org.apache.hadoop.http.JettyUtils;
@@ -82,7 +82,8 @@ import com.google.inject.Singleton;
@Singleton
@Path("/ws/v1/node")
public class NMWebServices {
- private static final Log LOG = LogFactory.getLog(NMWebServices.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(NMWebServices.class);
private Context nmContext;
private ResourceView rview;
private WebApp webapp;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5ff57a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NavBlock.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NavBlock.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NavBlock.java
index 0a2731e..5cbcff5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NavBlock.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/NavBlock.java
@@ -18,9 +18,6 @@
package org.apache.hadoop.yarn.server.nodemanager.webapp;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.commons.logging.impl.Log4JLogger;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.util.Log4jWarningErrorMetricsAppender;
import org.apache.hadoop.yarn.webapp.YarnWebParams;
@@ -30,6 +27,8 @@ import org.apache.hadoop.yarn.webapp.view.HtmlBlock;
import com.google.inject.Inject;
+import static org.apache.hadoop.util.GenericsUtil.isLog4jLogger;
+
public class NavBlock extends HtmlBlock implements YarnWebParams {
private Configuration conf;
@@ -43,8 +42,7 @@ public class NavBlock extends HtmlBlock implements YarnWebParams {
protected void render(Block html) {
boolean addErrorsAndWarningsLink = false;
- Log log = LogFactory.getLog(NMErrorsAndWarningsPage.class);
- if (log instanceof Log4JLogger) {
+ if (isLog4jLogger(NMErrorsAndWarningsPage.class)) {
Log4jWarningErrorMetricsAppender appender = Log4jWarningErrorMetricsAppender.findAppender();
if (appender != null) {
addErrorsAndWarningsLink = true;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5ff57a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/WebServer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/WebServer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/WebServer.java
index 53e529b..813ba14 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/WebServer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/WebServer.java
@@ -19,9 +19,9 @@
package org.apache.hadoop.yarn.server.nodemanager.webapp;
import static org.apache.hadoop.yarn.util.StringHelper.pajoin;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.AuthenticationFilterInitializer;
import org.apache.hadoop.security.HttpCrossOriginFilterInitializer;
@@ -45,7 +45,8 @@ import java.util.List;
public class WebServer extends AbstractService {
- private static final Log LOG = LogFactory.getLog(WebServer.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(WebServer.class);
private final Context nmContext;
private final NMWebApp nmWebApp;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5ff57a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java
index e520a31..b5cb43b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java
@@ -19,12 +19,12 @@
package org.apache.hadoop.yarn.server.nodemanager;
import static org.junit.Assert.fail;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.util.Collection;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.security.UserGroupInformation;
@@ -60,8 +60,8 @@ import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
public class DummyContainerManager extends ContainerManagerImpl {
- private static final Log LOG = LogFactory
- .getLog(DummyContainerManager.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(DummyContainerManager.class);
public DummyContainerManager(Context context, ContainerExecutor exec,
DeletionService deletionContext, NodeStatusUpdater nodeStatusUpdater,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5ff57a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/MockNodeStatusUpdater.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/MockNodeStatusUpdater.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/MockNodeStatusUpdater.java
index 50487c8..2e80259 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/MockNodeStatusUpdater.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/MockNodeStatusUpdater.java
@@ -19,11 +19,11 @@
package org.apache.hadoop.yarn.server.nodemanager;
import java.io.IOException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.nio.ByteBuffer;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.exceptions.YarnException;
import org.apache.hadoop.yarn.factories.RecordFactory;
@@ -46,7 +46,8 @@ import org.apache.hadoop.yarn.server.utils.YarnServerBuilderUtils;
* real RM.
*/
public class MockNodeStatusUpdater extends NodeStatusUpdaterImpl {
- static final Log LOG = LogFactory.getLog(MockNodeStatusUpdater.class);
+ static final Logger LOG =
+ LoggerFactory.getLogger(MockNodeStatusUpdater.class);
private static final RecordFactory recordFactory = RecordFactoryProvider
.getRecordFactory(null);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5ff57a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestContainerManagerWithLCE.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestContainerManagerWithLCE.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestContainerManagerWithLCE.java
index 028db6a..7bf042f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestContainerManagerWithLCE.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestContainerManagerWithLCE.java
@@ -20,9 +20,9 @@ package org.apache.hadoop.yarn.server.nodemanager;
import java.io.File;
import java.io.IOException;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.UnsupportedFileSystemException;
import org.apache.hadoop.fs.permission.FsPermission;
@@ -34,8 +34,8 @@ import org.junit.Assume;
public class TestContainerManagerWithLCE extends TestContainerManager {
- private static final Log LOG = LogFactory
- .getLog(TestContainerManagerWithLCE.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TestContainerManagerWithLCE.class);
public TestContainerManagerWithLCE() throws UnsupportedFileSystemException {
super();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5ff57a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutor.java
index d67bd76..d4db6b0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutor.java
@@ -27,6 +27,8 @@ import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.FileOutputStream;
@@ -40,8 +42,6 @@ import java.util.HashSet;
import java.util.List;
import java.util.Set;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.FileContext;
@@ -137,8 +137,8 @@ import org.junit.Test;
* </ol>
*/
public class TestLinuxContainerExecutor {
- private static final Log LOG = LogFactory
- .getLog(TestLinuxContainerExecutor.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TestLinuxContainerExecutor.class);
private static File workSpace;
static {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5ff57a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java
index cfd0e36..6ca96db 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestLinuxContainerExecutorWithMocks.java
@@ -29,6 +29,8 @@ import static org.mockito.Mockito.doThrow;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.when;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.FileReader;
@@ -44,8 +46,6 @@ import java.util.LinkedList;
import java.util.List;
import java.util.Map;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
@@ -79,8 +79,8 @@ import static java.nio.file.StandardCopyOption.REPLACE_EXISTING;
public class TestLinuxContainerExecutorWithMocks {
- private static final Log LOG = LogFactory
- .getLog(TestLinuxContainerExecutorWithMocks.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TestLinuxContainerExecutorWithMocks.class);
private static final String MOCK_EXECUTOR =
"./src/test/resources/mock-container-executor";
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5ff57a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeHealthService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeHealthService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeHealthService.java
index c564c01..8083a56 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeHealthService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeHealthService.java
@@ -22,11 +22,11 @@ import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.io.PrintWriter;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import com.google.common.base.Joiner;
import com.google.common.base.Strings;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.FileUtil;
@@ -47,8 +47,8 @@ import static org.mockito.Mockito.spy;
public class TestNodeHealthService {
- private static volatile Log LOG = LogFactory
- .getLog(TestNodeHealthService.class);
+ private static volatile Logger LOG =
+ LoggerFactory.getLogger(TestNodeHealthService.class);
protected static File testRootDir = new File("target",
TestNodeHealthService.class.getName() + "-localDir").getAbsoluteFile();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5ff57a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerReboot.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerReboot.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerReboot.java
index c1df562..f661cf5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerReboot.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerReboot.java
@@ -22,6 +22,8 @@ import static org.mockito.Matchers.argThat;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
@@ -32,8 +34,6 @@ import java.util.HashMap;
import java.util.List;
import java.util.Map;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
@@ -82,7 +82,8 @@ public class TestNodeManagerReboot {
private FileContext localFS;
private MyNodeManager nm;
private DeletionService delService;
- static final Log LOG = LogFactory.getLog(TestNodeManagerReboot.class);
+ static final Logger LOG =
+ LoggerFactory.getLogger(TestNodeManagerReboot.class);
@Before
public void setup() throws UnsupportedFileSystemException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5ff57a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerResync.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerResync.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerResync.java
index b8cd7dd..97e9922 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerResync.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeManagerResync.java
@@ -21,6 +21,8 @@ package org.apache.hadoop.yarn.server.nodemanager;
import static org.junit.Assert.assertEquals;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
@@ -36,8 +38,6 @@ import java.util.concurrent.ConcurrentMap;
import java.util.concurrent.CyclicBarrier;
import java.util.concurrent.atomic.AtomicBoolean;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.UnsupportedFileSystemException;
@@ -113,8 +113,8 @@ public class TestNodeManagerResync {
new NodeManagerEvent(NodeManagerEventType.RESYNC);
private final long DUMMY_RM_IDENTIFIER = 1234;
- protected static Log LOG = LogFactory
- .getLog(TestNodeManagerResync.class);
+ protected static final Logger LOG =
+ LoggerFactory.getLogger(TestNodeManagerResync.class);
@Before
public void setup() throws UnsupportedFileSystemException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5ff57a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java
index ca029be..11c3c35 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java
@@ -21,6 +21,8 @@ package org.apache.hadoop.yarn.server.nodemanager;
import static org.apache.hadoop.yarn.server.utils.YarnServerBuilderUtils.newNodeHeartbeatResponse;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.when;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.io.EOFException;
import java.io.File;
@@ -45,8 +47,6 @@ import java.util.concurrent.TimeUnit;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.Path;
@@ -125,7 +125,8 @@ public class TestNodeStatusUpdater {
DefaultMetricsSystem.setMiniClusterMode(true);
}
- static final Log LOG = LogFactory.getLog(TestNodeStatusUpdater.class);
+ static final Logger LOG =
+ LoggerFactory.getLogger(TestNodeStatusUpdater.class);
static final File basedir =
new File("target", TestNodeStatusUpdater.class.getName());
static final File nmLocalDir = new File(basedir, "nm0");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5ff57a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java
index a24c83b..26e4085 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/BaseAMRMProxyTest.java
@@ -33,9 +33,9 @@ import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.Credentials;
import org.apache.hadoop.security.UserGroupInformation;
@@ -87,8 +87,8 @@ import org.junit.Before;
*
*/
public abstract class BaseAMRMProxyTest {
- private static final Log LOG = LogFactory
- .getLog(BaseAMRMProxyTest.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(BaseAMRMProxyTest.class);
// The AMRMProxyService instance that will be used by all the test cases
private MockAMRMProxyService amrmProxyService;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5ff57a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestAMRMProxyService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestAMRMProxyService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestAMRMProxyService.java
index 72e5f53..937ede5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestAMRMProxyService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/amrmproxy/TestAMRMProxyService.java
@@ -22,9 +22,9 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
@@ -50,8 +50,8 @@ import org.junit.Test;
public class TestAMRMProxyService extends BaseAMRMProxyTest {
- private static final Log LOG = LogFactory
- .getLog(TestAMRMProxyService.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TestAMRMProxyService.class);
private static MockResourceManagerFacade mockRM;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5ff57a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java
index 6c96a47..3cafcbd 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java
@@ -19,6 +19,8 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager;
import static org.mockito.Mockito.spy;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
@@ -30,8 +32,6 @@ import java.util.HashSet;
import java.util.List;
import java.util.Map;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.Path;
@@ -114,8 +114,8 @@ public abstract class BaseContainerManagerTest {
tmpDir = new File("target", this.getClass().getSimpleName() + "-tmpDir");
}
- protected static Log LOG = LogFactory
- .getLog(BaseContainerManagerTest.class);
+ protected static Logger LOG =
+ LoggerFactory.getLogger(BaseContainerManagerTest.class);
protected static final int HTTP_PORT = 5412;
protected Configuration conf = new YarnConfiguration();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5ff57a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java
index f075713..bfe87c6 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestAuxServices.java
@@ -26,6 +26,8 @@ import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import static org.mockito.Mockito.mock;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import com.google.common.base.Charsets;
import com.google.common.collect.Sets;
@@ -42,8 +44,6 @@ import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileUtil;
@@ -73,7 +73,8 @@ import org.junit.Assert;
import org.junit.Test;
public class TestAuxServices {
- private static final Log LOG = LogFactory.getLog(TestAuxServices.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TestAuxServices.class);
private static final File TEST_DIR = new File(
System.getProperty("test.build.data",
System.getProperty("java.io.tmpdir")),
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5ff57a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java
index 9844225..f379c08 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java
@@ -40,7 +40,6 @@ import java.util.List;
import java.util.Map;
import com.google.common.base.Supplier;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.UnsupportedFileSystemException;
@@ -110,6 +109,7 @@ import org.junit.Before;
import org.junit.Test;
import org.mockito.ArgumentCaptor;
import org.mockito.Mockito;
+import org.slf4j.LoggerFactory;
public class TestContainerManager extends BaseContainerManagerTest {
@@ -118,7 +118,7 @@ public class TestContainerManager extends BaseContainerManagerTest {
}
static {
- LOG = LogFactory.getLog(TestContainerManager.class);
+ LOG = LoggerFactory.getLogger(TestContainerManager.class);
}
private boolean delayContainers = false;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5ff57a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/privileged/TestPrivilegedOperationExecutor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/privileged/TestPrivilegedOperationExecutor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/privileged/TestPrivilegedOperationExecutor.java
index 7146412..c5b2e97 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/privileged/TestPrivilegedOperationExecutor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/privileged/TestPrivilegedOperationExecutor.java
@@ -20,13 +20,13 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.privileged;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.io.File;
import java.util.ArrayList;
@@ -34,8 +34,8 @@ import java.util.Arrays;
import java.util.List;
public class TestPrivilegedOperationExecutor {
- private static final Log LOG = LogFactory
- .getLog(TestPrivilegedOperationExecutor.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TestPrivilegedOperationExecutor.class);
private String localDataDir;
private String customExecutorPath;
private Configuration nullConf = null;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[21/50] [abbrv] hadoop git commit: Revert "HADOOP-14732.
ProtobufRpcEngine should use Time.monotonicNow to measure durations.
Contributed by Hanisha Koneru."
Posted by as...@apache.org.
Revert "HADOOP-14732. ProtobufRpcEngine should use Time.monotonicNow to measure durations. Contributed by Hanisha Koneru."
This reverts commit 8bef4eca28a3466707cc4ea0de0330449319a5eb.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2d105a20
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2d105a20
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2d105a20
Branch: refs/heads/YARN-5972
Commit: 2d105a206884b62ccdba61f2de3e2fe65fc43074
Parents: e05fa34
Author: Arpit Agarwal <ar...@apache.org>
Authored: Fri Aug 18 10:15:52 2017 -0700
Committer: Arpit Agarwal <ar...@apache.org>
Committed: Fri Aug 18 10:15:52 2017 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/ipc/ProtobufRpcEngine.java | 14 +++++++-------
1 file changed, 7 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2d105a20/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java
----------------------------------------------------------------------
diff --git a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java
index 2c0cfe5..639bbad 100644
--- a/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java
+++ b/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/ipc/ProtobufRpcEngine.java
@@ -190,7 +190,7 @@ public class ProtobufRpcEngine implements RpcEngine {
throws ServiceException {
long startTime = 0;
if (LOG.isDebugEnabled()) {
- startTime = Time.monotonicNow();
+ startTime = Time.now();
}
if (args.length != 2) { // RpcController + Message
@@ -245,7 +245,7 @@ public class ProtobufRpcEngine implements RpcEngine {
}
if (LOG.isDebugEnabled()) {
- long callTime = Time.monotonicNow() - startTime;
+ long callTime = Time.now() - startTime;
LOG.debug("Call: " + method.getName() + " took " + callTime + "ms");
}
@@ -373,19 +373,19 @@ public class ProtobufRpcEngine implements RpcEngine {
this.server = currentCallInfo.get().server;
this.call = Server.getCurCall().get();
this.methodName = currentCallInfo.get().methodName;
- this.setupTime = Time.monotonicNow();
+ this.setupTime = Time.now();
}
@Override
public void setResponse(Message message) {
- long processingTime = Time.monotonicNow() - setupTime;
+ long processingTime = Time.now() - setupTime;
call.setDeferredResponse(RpcWritable.wrap(message));
server.updateDeferredMetrics(methodName, processingTime);
}
@Override
public void error(Throwable t) {
- long processingTime = Time.monotonicNow() - setupTime;
+ long processingTime = Time.now() - setupTime;
String detailedMetricsName = t.getClass().getSimpleName();
server.updateDeferredMetrics(detailedMetricsName, processingTime);
call.setDeferredError(t);
@@ -513,7 +513,7 @@ public class ProtobufRpcEngine implements RpcEngine {
Message param = request.getValue(prototype);
Message result;
- long startTime = Time.monotonicNow();
+ long startTime = Time.now();
int qTime = (int) (startTime - receiveTime);
Exception exception = null;
boolean isDeferred = false;
@@ -537,7 +537,7 @@ public class ProtobufRpcEngine implements RpcEngine {
throw e;
} finally {
currentCallInfo.set(null);
- int processingTime = (int) (Time.monotonicNow() - startTime);
+ int processingTime = (int) (Time.now() - startTime);
if (LOG.isDebugEnabled()) {
String msg =
"Served: " + methodName + (isDeferred ? ", deferred" : "") +
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[13/50] [abbrv] hadoop git commit: YARN-3254. HealthReport should
include disk full information. Contributed by Suma Shivaprasad.
Posted by as...@apache.org.
YARN-3254. HealthReport should include disk full information. Contributed by Suma Shivaprasad.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f9a0e233
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f9a0e233
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f9a0e233
Branch: refs/heads/YARN-5972
Commit: f9a0e2338150f1bd3ba2c29f76979183fd3ed80c
Parents: 1f04cb4
Author: Sunil G <su...@apache.org>
Authored: Thu Aug 17 15:07:15 2017 +0530
Committer: Sunil G <su...@apache.org>
Committed: Thu Aug 17 15:07:15 2017 +0530
----------------------------------------------------------------------
.../server/nodemanager/DirectoryCollection.java | 61 +++++++++++++++++++-
.../nodemanager/LocalDirsHandlerService.java | 59 +++++++++++++++----
.../nodemanager/TestDirectoryCollection.java | 23 ++++++++
3 files changed, 130 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9a0e233/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DirectoryCollection.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DirectoryCollection.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DirectoryCollection.java
index ae2a4ef..502485f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DirectoryCollection.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DirectoryCollection.java
@@ -38,6 +38,7 @@ import org.apache.commons.io.FileUtils;
import org.apache.commons.lang.RandomStringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.fs.FileAlreadyExistsException;
import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.fs.Path;
@@ -99,6 +100,7 @@ public class DirectoryCollection {
private List<String> localDirs;
private List<String> errorDirs;
private List<String> fullDirs;
+ private Map<String, DiskErrorInformation> directoryErrorInfo;
// read/write lock for accessing above directories.
private final ReadLock readLock;
@@ -192,6 +194,7 @@ public class DirectoryCollection {
localDirs = new CopyOnWriteArrayList<>(dirs);
errorDirs = new CopyOnWriteArrayList<>();
fullDirs = new CopyOnWriteArrayList<>();
+ directoryErrorInfo = new ConcurrentHashMap<>();
ReentrantReadWriteLock lock = new ReentrantReadWriteLock();
this.readLock = lock.readLock();
@@ -248,11 +251,25 @@ public class DirectoryCollection {
/**
* @return the directories that have used all disk space
*/
-
List<String> getFullDirs() {
this.readLock.lock();
try {
- return fullDirs;
+ return Collections.unmodifiableList(fullDirs);
+ } finally {
+ this.readLock.unlock();
+ }
+ }
+
+ /**
+ * @return the directories that have errors - many not have appropriate permissions
+ * or other disk validation checks might have failed in {@link DiskValidator}
+ *
+ */
+ @InterfaceStability.Evolving
+ List<String> getErroredDirs() {
+ this.readLock.lock();
+ try {
+ return Collections.unmodifiableList(errorDirs);
} finally {
this.readLock.unlock();
}
@@ -271,6 +288,39 @@ public class DirectoryCollection {
}
/**
+ *
+ * @param dirName Absolute path of Directory for which error diagnostics are needed
+ * @return DiskErrorInformation - disk error diagnostics for the specified directory
+ * null - the disk associated with the directory has passed disk utilization checks
+ * /error validations in {@link DiskValidator}
+ *
+ */
+ @InterfaceStability.Evolving
+ DiskErrorInformation getDirectoryErrorInfo(String dirName) {
+ this.readLock.lock();
+ try {
+ return directoryErrorInfo.get(dirName);
+ } finally {
+ this.readLock.unlock();
+ }
+ }
+
+ /**
+ *
+ * @param dirName Absolute path of Directory for which the disk has been marked as unhealthy
+ * @return Check if disk associated with the directory is unhealthy
+ */
+ @InterfaceStability.Evolving
+ boolean isDiskUnHealthy(String dirName) {
+ this.readLock.lock();
+ try {
+ return directoryErrorInfo.containsKey(dirName);
+ } finally {
+ this.readLock.unlock();
+ }
+ }
+
+ /**
* Create any non-existent directories and parent directories, updating the
* list of valid directories if necessary.
* @param localFs local file system to use
@@ -297,6 +347,9 @@ public class DirectoryCollection {
try {
localDirs.remove(dir);
errorDirs.add(dir);
+ directoryErrorInfo.put(dir,
+ new DiskErrorInformation(DiskErrorCause.OTHER,
+ "Cannot create directory : " + dir + ", error " + e.getMessage()));
numFailures++;
} finally {
this.writeLock.unlock();
@@ -343,11 +396,13 @@ public class DirectoryCollection {
localDirs.clear();
errorDirs.clear();
fullDirs.clear();
+ directoryErrorInfo.clear();
for (Map.Entry<String, DiskErrorInformation> entry : dirsFailedCheck
.entrySet()) {
String dir = entry.getKey();
DiskErrorInformation errorInformation = entry.getValue();
+
switch (entry.getValue().cause) {
case DISK_FULL:
fullDirs.add(entry.getKey());
@@ -359,6 +414,8 @@ public class DirectoryCollection {
LOG.warn(entry.getValue().cause + " is unknown for disk error.");
break;
}
+ directoryErrorInfo.put(entry.getKey(), errorInformation);
+
if (preCheckGoodDirs.contains(dir)) {
LOG.warn("Directory " + dir + " error, " + errorInformation.message
+ ", removing from list of valid directories");
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9a0e233/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java
index f8cb4ee..6e00808 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LocalDirsHandlerService.java
@@ -53,6 +53,8 @@ public class LocalDirsHandlerService extends AbstractService {
private static Log LOG = LogFactory.getLog(LocalDirsHandlerService.class);
+ private static final String diskCapacityExceededErrorMsg = "usable space is below configured utilization percentage/no more usable space";
+
/**
* Good local directories, use internally,
* initial value is the same as NM_LOCAL_DIRS.
@@ -344,21 +346,36 @@ public class LocalDirsHandlerService extends AbstractService {
}
StringBuilder report = new StringBuilder();
- List<String> failedLocalDirsList = localDirs.getFailedDirs();
- List<String> failedLogDirsList = logDirs.getFailedDirs();
+ List<String> erroredLocalDirsList = localDirs.getErroredDirs();
+ List<String> erroredLogDirsList = logDirs.getErroredDirs();
+ List<String> diskFullLocalDirsList = localDirs.getFullDirs();
+ List<String> diskFullLogDirsList = logDirs.getFullDirs();
List<String> goodLocalDirsList = localDirs.getGoodDirs();
List<String> goodLogDirsList = logDirs.getGoodDirs();
- int numLocalDirs = goodLocalDirsList.size() + failedLocalDirsList.size();
- int numLogDirs = goodLogDirsList.size() + failedLogDirsList.size();
+
+ int numLocalDirs = goodLocalDirsList.size() + erroredLocalDirsList.size() + diskFullLocalDirsList.size();
+ int numLogDirs = goodLogDirsList.size() + erroredLogDirsList.size() + diskFullLogDirsList.size();
if (!listGoodDirs) {
- if (!failedLocalDirsList.isEmpty()) {
- report.append(failedLocalDirsList.size() + "/" + numLocalDirs
- + " local-dirs are bad: "
- + StringUtils.join(",", failedLocalDirsList) + "; ");
+ if (!erroredLocalDirsList.isEmpty()) {
+ report.append(erroredLocalDirsList.size() + "/" + numLocalDirs
+ + " local-dirs have errors: "
+ + buildDiskErrorReport(erroredLocalDirsList, localDirs));
+ }
+ if (!diskFullLocalDirsList.isEmpty()) {
+ report.append(diskFullLocalDirsList.size() + "/" + numLocalDirs
+ + " local-dirs " + diskCapacityExceededErrorMsg
+ + buildDiskErrorReport(diskFullLocalDirsList, localDirs) + "; ");
}
- if (!failedLogDirsList.isEmpty()) {
- report.append(failedLogDirsList.size() + "/" + numLogDirs
- + " log-dirs are bad: " + StringUtils.join(",", failedLogDirsList));
+
+ if (!erroredLogDirsList.isEmpty()) {
+ report.append(erroredLogDirsList.size() + "/" + numLogDirs
+ + " log-dirs have errors: "
+ + buildDiskErrorReport(erroredLogDirsList, logDirs));
+ }
+ if (!diskFullLogDirsList.isEmpty()) {
+ report.append(diskFullLogDirsList.size() + "/" + numLogDirs
+ + " log-dirs " + diskCapacityExceededErrorMsg
+ + buildDiskErrorReport(diskFullLogDirsList, logDirs));
}
} else {
report.append(goodLocalDirsList.size() + "/" + numLocalDirs
@@ -620,4 +637,24 @@ public class LocalDirsHandlerService extends AbstractService {
logDirs.getGoodDirsDiskUtilizationPercentage());
}
}
+
+ private String buildDiskErrorReport(List<String> dirs, DirectoryCollection directoryCollection) {
+ StringBuilder sb = new StringBuilder();
+
+ sb.append(" [ ");
+ for (int i = 0; i < dirs.size(); i++) {
+ final String dirName = dirs.get(i);
+ if ( directoryCollection.isDiskUnHealthy(dirName)) {
+ sb.append(dirName + " : " + directoryCollection.getDirectoryErrorInfo(dirName).message);
+ } else {
+ sb.append(dirName + " : " + "Unknown cause for disk error");
+ }
+
+ if ( i != (dirs.size() - 1)) {
+ sb.append(" , ");
+ }
+ }
+ sb.append(" ] ");
+ return sb.toString();
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f9a0e233/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDirectoryCollection.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDirectoryCollection.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDirectoryCollection.java
index e529628..095f21a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDirectoryCollection.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDirectoryCollection.java
@@ -128,8 +128,12 @@ public class TestDirectoryCollection {
DirectoryCollection dc = new DirectoryCollection(dirs, 0.0F);
dc.checkDirs();
Assert.assertEquals(0, dc.getGoodDirs().size());
+ Assert.assertEquals(0, dc.getErroredDirs().size());
Assert.assertEquals(1, dc.getFailedDirs().size());
Assert.assertEquals(1, dc.getFullDirs().size());
+ Assert.assertNotNull(dc.getDirectoryErrorInfo(dirA));
+ Assert.assertEquals(DirectoryCollection.DiskErrorCause.DISK_FULL, dc.getDirectoryErrorInfo(dirA).cause);
+
// no good dirs
Assert.assertEquals(0, dc.getGoodDirsDiskUtilizationPercentage());
@@ -139,16 +143,21 @@ public class TestDirectoryCollection {
testDir.getTotalSpace());
dc.checkDirs();
Assert.assertEquals(1, dc.getGoodDirs().size());
+ Assert.assertEquals(0, dc.getErroredDirs().size());
Assert.assertEquals(0, dc.getFailedDirs().size());
Assert.assertEquals(0, dc.getFullDirs().size());
+ Assert.assertNull(dc.getDirectoryErrorInfo(dirA));
+
Assert.assertEquals(utilizedSpacePerc,
dc.getGoodDirsDiskUtilizationPercentage());
dc = new DirectoryCollection(dirs, testDir.getTotalSpace() / (1024 * 1024));
dc.checkDirs();
Assert.assertEquals(0, dc.getGoodDirs().size());
+ Assert.assertEquals(0, dc.getErroredDirs().size());
Assert.assertEquals(1, dc.getFailedDirs().size());
Assert.assertEquals(1, dc.getFullDirs().size());
+ Assert.assertNotNull(dc.getDirectoryErrorInfo(dirA));
// no good dirs
Assert.assertEquals(0, dc.getGoodDirsDiskUtilizationPercentage());
@@ -158,8 +167,11 @@ public class TestDirectoryCollection {
testDir.getTotalSpace());
dc.checkDirs();
Assert.assertEquals(1, dc.getGoodDirs().size());
+ Assert.assertEquals(0, dc.getErroredDirs().size());
Assert.assertEquals(0, dc.getFailedDirs().size());
Assert.assertEquals(0, dc.getFullDirs().size());
+ Assert.assertNull(dc.getDirectoryErrorInfo(dirA));
+
Assert.assertEquals(utilizedSpacePerc,
dc.getGoodDirsDiskUtilizationPercentage());
}
@@ -209,12 +221,17 @@ public class TestDirectoryCollection {
Assert.assertEquals(0, dc.getGoodDirs().size());
Assert.assertEquals(1, dc.getFailedDirs().size());
Assert.assertEquals(1, dc.getFullDirs().size());
+ Assert.assertEquals(0, dc.getErroredDirs().size());
+ Assert.assertNotNull(dc.getDirectoryErrorInfo(dirA));
+ Assert.assertEquals(DirectoryCollection.DiskErrorCause.DISK_FULL, dc.getDirectoryErrorInfo(dirA).cause);
dc.setDiskUtilizationPercentageCutoff(100.0F, 100.0F);
dc.checkDirs();
Assert.assertEquals(1, dc.getGoodDirs().size());
Assert.assertEquals(0, dc.getFailedDirs().size());
Assert.assertEquals(0, dc.getFullDirs().size());
+ Assert.assertEquals(0, dc.getErroredDirs().size());
+ Assert.assertNull(dc.getDirectoryErrorInfo(dirA));
conf.set(CommonConfigurationKeys.FS_PERMISSIONS_UMASK_KEY, "077");
@@ -232,12 +249,18 @@ public class TestDirectoryCollection {
Assert.assertEquals(0, dc.getGoodDirs().size());
Assert.assertEquals(1, dc.getFailedDirs().size());
Assert.assertEquals(0, dc.getFullDirs().size());
+ Assert.assertEquals(1, dc.getErroredDirs().size());
+ Assert.assertNotNull(dc.getDirectoryErrorInfo(dirB));
+ Assert.assertEquals(DirectoryCollection.DiskErrorCause.OTHER, dc.getDirectoryErrorInfo(dirB).cause);
+
permDirB = new FsPermission((short) 0700);
localFs.setPermission(pathB, permDirB);
dc.checkDirs();
Assert.assertEquals(1, dc.getGoodDirs().size());
Assert.assertEquals(0, dc.getFailedDirs().size());
Assert.assertEquals(0, dc.getFullDirs().size());
+ Assert.assertEquals(0, dc.getErroredDirs().size());
+ Assert.assertNull(dc.getDirectoryErrorInfo(dirA));
}
@Test
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[26/50] [abbrv] hadoop git commit: HADOOP-14194. Aliyun OSS should
not use empty endpoint as default. Contributed by Genmao Yu
Posted by as...@apache.org.
HADOOP-14194. Aliyun OSS should not use empty endpoint as default. Contributed by Genmao Yu
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/267e19a0
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/267e19a0
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/267e19a0
Branch: refs/heads/YARN-5972
Commit: 267e19a09f366a965b30c8d4dc75e377b0d92fff
Parents: 7a82d7b
Author: Kai Zheng <ka...@intel.com>
Authored: Mon Aug 21 13:36:28 2017 +0800
Committer: Kai Zheng <ka...@intel.com>
Committed: Mon Aug 21 13:36:28 2017 +0800
----------------------------------------------------------------------
.../apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java | 4 ++++
1 file changed, 4 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/267e19a0/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java
index a944fc1..a85a739 100644
--- a/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java
+++ b/hadoop-tools/hadoop-aliyun/src/main/java/org/apache/hadoop/fs/aliyun/oss/AliyunOSSFileSystemStore.java
@@ -129,6 +129,10 @@ public class AliyunOSSFileSystemStore {
}
String endPoint = conf.getTrimmed(ENDPOINT_KEY, "");
+ if (StringUtils.isEmpty(endPoint)) {
+ throw new IllegalArgumentException("Aliyun OSS endpoint should not be " +
+ "null or empty. Please set proper endpoint with 'fs.oss.endpoint'.");
+ }
CredentialsProvider provider =
AliyunOSSUtils.getCredentialsProvider(conf);
ossClient = new OSSClient(endPoint, provider, clientConf);
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[44/50] [abbrv] hadoop git commit: HDFS-10899. Add functionality to
re-encrypt EDEKs.
Posted by as...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1000a2af/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReencryptionHandler.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReencryptionHandler.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReencryptionHandler.java
new file mode 100644
index 0000000..f0dd92c
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestReencryptionHandler.java
@@ -0,0 +1,197 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.namenode;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.crypto.key.JavaKeyStoreProvider;
+import org.apache.hadoop.crypto.key.KeyProvider;
+import org.apache.hadoop.crypto.key.KeyProviderCryptoExtension;
+import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
+import org.apache.hadoop.fs.FileSystemTestHelper;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.test.GenericTestUtils;
+import org.apache.hadoop.util.KMSUtil;
+import org.apache.hadoop.util.StopWatch;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.Timeout;
+import org.mockito.Mockito;
+import org.mockito.internal.util.reflection.Whitebox;
+import org.slf4j.LoggerFactory;
+import org.slf4j.event.Level;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.Future;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.TimeUnit;
+
+import static org.junit.Assert.assertTrue;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_NAMENODE_REENCRYPT_THROTTLE_LIMIT_HANDLER_RATIO_KEY;
+import static org.junit.Assert.fail;
+
+/**
+ * Test class for ReencryptionHandler.
+ */
+public class TestReencryptionHandler {
+
+ protected static final org.slf4j.Logger LOG =
+ LoggerFactory.getLogger(TestReencryptionHandler.class);
+
+ @Rule
+ public Timeout globalTimeout = new Timeout(180 * 1000);
+
+ @Before
+ public void setup() {
+ GenericTestUtils.setLogLevel(ReencryptionHandler.LOG, Level.TRACE);
+ }
+
+ private ReencryptionHandler mockReencryptionhandler(final Configuration conf)
+ throws IOException {
+ // mock stuff to create a mocked ReencryptionHandler
+ conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH,
+ JavaKeyStoreProvider.SCHEME_NAME + "://file" + new Path(
+ new FileSystemTestHelper().getTestRootDir(), "test.jks").toUri());
+ final EncryptionZoneManager ezm = Mockito.mock(EncryptionZoneManager.class);
+ final KeyProvider kp = KMSUtil.createKeyProvider(conf,
+ CommonConfigurationKeysPublic.HADOOP_SECURITY_KEY_PROVIDER_PATH);
+ Mockito.when(ezm.getProvider()).thenReturn(
+ KeyProviderCryptoExtension.createKeyProviderCryptoExtension(kp));
+ return new ReencryptionHandler(ezm, conf);
+ }
+
+ @Test
+ public void testThrottle() throws Exception {
+ final Configuration conf = new Configuration();
+ conf.setDouble(DFS_NAMENODE_REENCRYPT_THROTTLE_LIMIT_HANDLER_RATIO_KEY,
+ 0.5);
+ final ReencryptionHandler rh = mockReencryptionhandler(conf);
+
+ // mock StopWatches so all = 30s, locked = 20s. With ratio = .5, throttle
+ // should wait for 30 * 0.5 - 20 = 5s.
+ final StopWatch mockAll = Mockito.mock(StopWatch.class);
+ Mockito.when(mockAll.now(TimeUnit.MILLISECONDS)).thenReturn((long) 30000);
+ Mockito.when(mockAll.reset()).thenReturn(mockAll);
+ final StopWatch mockLocked = Mockito.mock(StopWatch.class);
+ Mockito.when(mockLocked.now(TimeUnit.MILLISECONDS))
+ .thenReturn((long) 20000);
+ Mockito.when(mockLocked.reset()).thenReturn(mockLocked);
+ final BlockingQueue<Runnable> queue = new LinkedBlockingQueue<>();
+ Whitebox.setInternalState(rh, "throttleTimerAll", mockAll);
+ Whitebox.setInternalState(rh, "throttleTimerLocked", mockLocked);
+ Whitebox.setInternalState(rh, "taskQueue", queue);
+ final StopWatch sw = new StopWatch().start();
+ rh.throttle();
+ sw.stop();
+ assertTrue("should have throttled for at least 4 second",
+ sw.now(TimeUnit.MILLISECONDS) > 8000);
+ assertTrue("should have throttled for at most 6 second",
+ sw.now(TimeUnit.MILLISECONDS) < 12000);
+ }
+
+ @Test
+ public void testThrottleNoOp() throws Exception {
+ final Configuration conf = new Configuration();
+ conf.setDouble(DFS_NAMENODE_REENCRYPT_THROTTLE_LIMIT_HANDLER_RATIO_KEY,
+ 0.5);
+ final ReencryptionHandler rh = mockReencryptionhandler(conf);
+
+ // mock StopWatches so all = 30s, locked = 10s. With ratio = .5, throttle
+ // should not happen.
+ StopWatch mockAll = Mockito.mock(StopWatch.class);
+ Mockito.when(mockAll.now()).thenReturn(new Long(30000));
+ Mockito.when(mockAll.reset()).thenReturn(mockAll);
+ StopWatch mockLocked = Mockito.mock(StopWatch.class);
+ Mockito.when(mockLocked.now()).thenReturn(new Long(10000));
+ Mockito.when(mockLocked.reset()).thenReturn(mockLocked);
+ final BlockingQueue<Runnable> queue = new LinkedBlockingQueue<>();
+ Whitebox.setInternalState(rh, "throttleTimerAll", mockAll);
+ Whitebox.setInternalState(rh, "throttleTimerLocked", mockLocked);
+ Whitebox.setInternalState(rh, "taskQueue", queue);
+ final Map<Long, ReencryptionUpdater.ZoneSubmissionTracker>
+ submissions = new HashMap<>();
+ Whitebox.setInternalState(rh, "submissions", submissions);
+ StopWatch sw = new StopWatch().start();
+ rh.throttle();
+ sw.stop();
+ assertTrue("should not have throttled",
+ sw.now(TimeUnit.MILLISECONDS) < 1000);
+ }
+
+ @Test
+ public void testThrottleConfigs() throws Exception {
+ final Configuration conf = new Configuration();
+ conf.setDouble(DFS_NAMENODE_REENCRYPT_THROTTLE_LIMIT_HANDLER_RATIO_KEY,
+ -1.0);
+ try {
+ mockReencryptionhandler(conf);
+ fail("Should not be able to init");
+ } catch (IllegalArgumentException e) {
+ GenericTestUtils.assertExceptionContains(" is not positive", e);
+ }
+
+ conf.setDouble(DFS_NAMENODE_REENCRYPT_THROTTLE_LIMIT_HANDLER_RATIO_KEY,
+ 0.0);
+ try {
+ mockReencryptionhandler(conf);
+ fail("Should not be able to init");
+ } catch (IllegalArgumentException e) {
+ GenericTestUtils.assertExceptionContains(" is not positive", e);
+ }
+ }
+
+ @Test
+ public void testThrottleAccumulatingTasks() throws Exception {
+ final Configuration conf = new Configuration();
+ final ReencryptionHandler rh = mockReencryptionhandler(conf);
+
+ // mock tasks piling up
+ final Map<Long, ReencryptionUpdater.ZoneSubmissionTracker>
+ submissions = new HashMap<>();
+ final ReencryptionUpdater.ZoneSubmissionTracker zst =
+ new ReencryptionUpdater.ZoneSubmissionTracker();
+ submissions.put(new Long(1), zst);
+ Future mock = Mockito.mock(Future.class);
+ for (int i = 0; i < Runtime.getRuntime().availableProcessors() * 3; ++i) {
+ zst.addTask(mock);
+ }
+
+ Thread removeTaskThread = new Thread() {
+ public void run() {
+ try {
+ Thread.sleep(3000);
+ } catch (InterruptedException ie) {
+ LOG.info("removeTaskThread interrupted.");
+ Thread.currentThread().interrupt();
+ }
+ zst.getTasks().clear();
+ }
+ };
+
+ Whitebox.setInternalState(rh, "submissions", submissions);
+ final StopWatch sw = new StopWatch().start();
+ removeTaskThread.start();
+ rh.throttle();
+ sw.stop();
+ assertTrue("should have throttled for at least 3 second",
+ sw.now(TimeUnit.MILLISECONDS) > 3000);
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/1000a2af/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testCryptoConf.xml
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testCryptoConf.xml b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testCryptoConf.xml
index f9bb29e..a6980aa 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testCryptoConf.xml
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/resources/testCryptoConf.xml
@@ -603,5 +603,85 @@
</comparator>
</comparators>
</test>
+
+ <!--More thorough test cases for re-encryption are in TestEncryptionZones-->
+ <test>
+ <description>Test success of reencrypt submission on a EZ</description>
+ <test-commands>
+ <command>-fs NAMENODE -mkdir /src</command>
+ <crypto-admin-command>-createZone -path /src -keyName myKey</crypto-admin-command>
+ <crypto-admin-command>-reencryptZone -start -path /src</crypto-admin-command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm -r -skipTrash /src</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>successfully submitted for zone: /src action: START</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+
+ <test>
+ <description>Test failure of reencrypt submission on a non-EZ</description>
+ <test-commands>
+ <command>-fs NAMENODE -mkdir /src</command>
+ <crypto-admin-command>-reencryptZone -start -path /src</crypto-admin-command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm -r -skipTrash /src</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>not the root of an encryption zone</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+
+ <!-- Cannot test successful cancel here, since the submit will finish very quickly.
+ TestReencryption covers successful cancellations. -->
+
+ <test>
+ <description>Test failure of reencrypt on cancellation a not-being-re-encrypted EZ</description>
+ <test-commands>
+ <command>-fs NAMENODE -mkdir /src</command>
+ <crypto-admin-command>-createZone -path /src -keyName myKey</crypto-admin-command>
+ <crypto-admin-command>-reencryptZone -cancel -path /src</crypto-admin-command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm -r -skipTrash /src</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>SubstringComparator</type>
+ <expected-output>Zone /src is not under re-encryption</expected-output>
+ </comparator>
+ </comparators>
+ </test>
+
+ <test>
+ <description>Test success of list reencrypt status on a EZ</description>
+ <test-commands>
+ <command>-fs NAMENODE -mkdir /src</command>
+ <command>-fs NAMENODE -mkdir /zone2</command>
+ <crypto-admin-command>-createZone -path /src -keyName myKey</crypto-admin-command>
+ <crypto-admin-command>-createZone -path /zone2 -keyName myKey</crypto-admin-command>
+ <crypto-admin-command>-reencryptZone -start -path /src</crypto-admin-command>
+ <crypto-admin-command>-reencryptZone -start -path /zone2</crypto-admin-command>
+ <crypto-admin-command>-listReencryptionStatus</crypto-admin-command>
+ </test-commands>
+ <cleanup-commands>
+ <command>-fs NAMENODE -rm -r -skipTrash /src</command>
+ <command>-fs NAMENODE -rm -r -skipTrash /zone2</command>
+ </cleanup-commands>
+ <comparators>
+ <comparator>
+ <type>TokenComparator</type>
+ <expected-output>/src,Completed,false,myKey,zone2</expected-output>
+ </comparator>
+ </comparators>
+ </test>
</tests>
</configuration>
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[32/50] [abbrv] hadoop git commit: YARN-7047. Moving logging APIs
over to slf4j in hadoop-yarn-server-nodemanager. Contributed by Yeliang Cang.
Posted by as...@apache.org.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5ff57a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsHandlerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsHandlerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsHandlerImpl.java
index ab989cf..996fff0 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsHandlerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestCGroupsHandlerImpl.java
@@ -21,8 +21,8 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources;
import org.apache.commons.io.FileUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
@@ -58,8 +58,8 @@ import static org.mockito.Mockito.verifyZeroInteractions;
* Tests for the CGroups handler implementation.
*/
public class TestCGroupsHandlerImpl {
- private static final Log LOG =
- LogFactory.getLog(TestCGroupsHandlerImpl.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TestCGroupsHandlerImpl.class);
private PrivilegedOperationExecutor privilegedOperationExecutorMock;
private String tmpPath;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5ff57a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestResourceHandlerModule.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestResourceHandlerModule.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestResourceHandlerModule.java
index 9da37dd..e5414a5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestResourceHandlerModule.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestResourceHandlerModule.java
@@ -20,19 +20,19 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.util.List;
public class TestResourceHandlerModule {
- private static final Log LOG = LogFactory.
- getLog(TestResourceHandlerModule.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TestResourceHandlerModule.class);
Configuration emptyConf;
Configuration networkEnabledConf;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5ff57a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestTrafficControlBandwidthHandlerImpl.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestTrafficControlBandwidthHandlerImpl.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestTrafficControlBandwidthHandlerImpl.java
index 13b0188..09f4f1d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestTrafficControlBandwidthHandlerImpl.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestTrafficControlBandwidthHandlerImpl.java
@@ -20,8 +20,6 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.yarn.api.records.ContainerId;
@@ -35,6 +33,8 @@ import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.mockito.ArgumentCaptor;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.io.File;
import java.util.List;
@@ -49,8 +49,8 @@ import static org.mockito.Mockito.verifyNoMoreInteractions;
import static org.mockito.Mockito.when;
public class TestTrafficControlBandwidthHandlerImpl {
- private static final Log LOG =
- LogFactory.getLog(TestTrafficControlBandwidthHandlerImpl.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TestTrafficControlBandwidthHandlerImpl.class);
private static final int ROOT_BANDWIDTH_MBIT = 100;
private static final int YARN_BANDWIDTH_MBIT = 70;
private static final int TEST_CLASSID = 100;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5ff57a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestTrafficController.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestTrafficController.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestTrafficController.java
index 7ea7135..c523663 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestTrafficController.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/TestTrafficController.java
@@ -20,8 +20,6 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
@@ -33,6 +31,8 @@ import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
import org.mockito.ArgumentCaptor;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
@@ -49,7 +49,8 @@ import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
public class TestTrafficController {
- private static final Log LOG = LogFactory.getLog(TestTrafficController.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TestTrafficController.class);
private static final int ROOT_BANDWIDTH_MBIT = 100;
private static final int YARN_BANDWIDTH_MBIT = 70;
private static final int CONTAINER_BANDWIDTH_MBIT = 10;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5ff57a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
index 9894dcd..d57d33c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/runtime/TestDockerContainerRuntime.java
@@ -20,8 +20,6 @@
package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.runtime;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
@@ -44,6 +42,8 @@ import org.junit.Before;
import org.junit.Test;
import org.mockito.ArgumentCaptor;
import org.mockito.Mockito;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
@@ -64,8 +64,8 @@ import static org.mockito.Matchers.eq;
import static org.mockito.Mockito.*;
public class TestDockerContainerRuntime {
- private static final Log LOG = LogFactory
- .getLog(TestDockerContainerRuntime.class);
+ private static final Logger LOG =
+ LoggerFactory.getLogger(TestDockerContainerRuntime.class);
private Configuration conf;
private PrivilegedOperationExecutor mockExecutor;
private CGroupsHandler mockCGroupsHandler;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5ff57a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestContainerLocalizer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestContainerLocalizer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestContainerLocalizer.java
index 6f6482f..17eee82 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestContainerLocalizer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestContainerLocalizer.java
@@ -37,6 +37,8 @@ import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
@@ -52,8 +54,6 @@ import java.util.concurrent.Future;
import java.util.concurrent.TimeUnit;
import org.apache.commons.io.FileUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.AbstractFileSystem;
import org.apache.hadoop.fs.CommonConfigurationKeys;
@@ -94,7 +94,8 @@ import com.google.common.base.Supplier;
public class TestContainerLocalizer {
- static final Log LOG = LogFactory.getLog(TestContainerLocalizer.class);
+ static final Logger LOG =
+ LoggerFactory.getLogger(TestContainerLocalizer.class);
static final Path basedir =
new Path("target", TestContainerLocalizer.class.getName());
static final FsPermission CACHE_DIR_PERM = new FsPermission((short)0710);
@@ -299,7 +300,7 @@ public class TestContainerLocalizer {
try {
localizerA.runLocalization(nmAddr);
} catch (Exception e) {
- LOG.warn(e);
+ LOG.warn(e.toString());
}
}
};
@@ -309,7 +310,7 @@ public class TestContainerLocalizer {
try {
localizerB.runLocalization(nmAddr);
} catch (Exception e) {
- LOG.warn(e);
+ LOG.warn(e.toString());
}
}
};
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5ff57a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
index 37fe77a..6383e83 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/logaggregation/TestLogAggregationService.java
@@ -60,7 +60,6 @@ import java.util.concurrent.locks.ReadWriteLock;
import java.util.concurrent.locks.ReentrantReadWriteLock;
import org.apache.commons.lang.StringUtils;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.AbstractFileSystem;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
@@ -137,6 +136,7 @@ import org.mockito.Mockito;
import org.eclipse.jetty.util.MultiException;
import com.google.common.base.Supplier;
+import org.slf4j.LoggerFactory;
//@Ignore
public class TestLogAggregationService extends BaseContainerManagerTest {
@@ -144,7 +144,7 @@ public class TestLogAggregationService extends BaseContainerManagerTest {
private Map<ApplicationAccessType, String> acls = createAppAcls();
static {
- LOG = LogFactory.getLog(TestLogAggregationService.class);
+ LOG = LoggerFactory.getLogger(TestLogAggregationService.class);
}
private static RecordFactory recordFactory = RecordFactoryProvider
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5ff57a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java
index 0f1c6f5..6f7fadf 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/monitor/TestContainersMonitor.java
@@ -36,7 +36,6 @@ import java.util.Map;
import java.util.regex.Pattern;
import com.google.common.base.Supplier;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.UnsupportedFileSystemException;
@@ -76,6 +75,7 @@ import org.apache.hadoop.yarn.util.TestProcfsBasedProcessTree;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
+import org.slf4j.LoggerFactory;
public class TestContainersMonitor extends BaseContainerManagerTest {
@@ -84,7 +84,7 @@ public class TestContainersMonitor extends BaseContainerManagerTest {
}
static {
- LOG = LogFactory.getLog(TestContainersMonitor.class);
+ LOG = LoggerFactory.getLogger(TestContainersMonitor.class);
}
@Before
public void setup() throws IOException {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/d5ff57a0/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerQueuing.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerQueuing.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerQueuing.java
index a1c247b..9676568 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerQueuing.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/scheduler/TestContainerSchedulerQueuing.java
@@ -24,7 +24,6 @@ import java.util.Arrays;
import java.util.HashMap;
import java.util.List;
-import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.fs.UnsupportedFileSystemException;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.protocolrecords.ContainerUpdateRequest;
@@ -57,6 +56,7 @@ import org.apache.hadoop.yarn.server.nodemanager.executor.ContainerStartContext;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.junit.Assert;
import org.junit.Test;
+import org.slf4j.LoggerFactory;
import static org.mockito.Mockito.spy;
@@ -70,7 +70,7 @@ public class TestContainerSchedulerQueuing extends BaseContainerManagerTest {
}
static {
- LOG = LogFactory.getLog(TestContainerSchedulerQueuing.class);
+ LOG = LoggerFactory.getLogger(TestContainerSchedulerQueuing.class);
}
private boolean delayContainers = true;
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[41/50] [abbrv] hadoop git commit: YARN-6251. Do async container
release to prevent deadlock during container updates. (Arun Suresh via
wangda)
Posted by as...@apache.org.
YARN-6251. Do async container release to prevent deadlock during container updates. (Arun Suresh via wangda)
Change-Id: I6c67d20c5dd4d22752830ebf0ed2340824976ecb
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f49843a9
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f49843a9
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f49843a9
Branch: refs/heads/YARN-5972
Commit: f49843a9888ad8fe5c1bb4c16bfb5217d693009d
Parents: 4249172
Author: Wangda Tan <wa...@apache.org>
Authored: Wed Aug 23 09:56:20 2017 -0700
Committer: Wangda Tan <wa...@apache.org>
Committed: Wed Aug 23 09:56:20 2017 -0700
----------------------------------------------------------------------
...pportunisticContainerAllocatorAMService.java | 2 +
.../scheduler/AbstractYarnScheduler.java | 12 ++++-
.../scheduler/SchedulerApplicationAttempt.java | 12 +++--
.../scheduler/capacity/CapacityScheduler.java | 12 +++++
.../distributed/NodeQueueLoadMonitor.java | 4 ++
.../scheduler/event/ReleaseContainerEvent.java | 46 ++++++++++++++++++++
.../scheduler/event/SchedulerEventType.java | 3 ++
.../scheduler/fair/FairScheduler.java | 13 ++++++
.../scheduler/fifo/FifoScheduler.java | 15 ++++++-
.../yarn/server/resourcemanager/MockNodes.java | 2 +-
...pportunisticContainerAllocatorAMService.java | 36 +++++++++------
.../capacity/TestContainerResizing.java | 31 ++++++++++++-
.../capacity/TestIncreaseAllocationExpirer.java | 27 +++++++++---
13 files changed, 188 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f49843a9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/OpportunisticContainerAllocatorAMService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/OpportunisticContainerAllocatorAMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/OpportunisticContainerAllocatorAMService.java
index 3c278de..4fc2916 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/OpportunisticContainerAllocatorAMService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/OpportunisticContainerAllocatorAMService.java
@@ -391,6 +391,8 @@ public class OpportunisticContainerAllocatorAMService
break;
case NODE_LABELS_UPDATE:
break;
+ case RELEASE_CONTAINER:
+ break;
// <-- IGNORED EVENTS : END -->
default:
LOG.error("Unknown event arrived at" +
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f49843a9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
index c3879dd..2c27017 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/AbstractYarnScheduler.java
@@ -67,7 +67,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.RMAuditLogger.AuditConstant
import org.apache.hadoop.yarn.server.resourcemanager.RMContext;
import org.apache.hadoop.yarn.server.resourcemanager.RMServerUtils;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
-import org.apache.hadoop.yarn.server.resourcemanager.nodelabels.RMNodeLabelsManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEvent;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMAppEventType;
@@ -89,6 +88,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.QueueEntit
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.ReleaseContainerEvent;
import org.apache.hadoop.yarn.server.scheduler.OpportunisticContainerContext;
import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
@@ -1273,4 +1273,14 @@ public abstract class AbstractYarnScheduler
public List<NodeId> getNodeIds(String resourceName) {
return nodeTracker.getNodeIdsByResourceName(resourceName);
}
+
+ /**
+ * To be used to release a container via a Scheduler Event rather than
+ * in the same thread.
+ * @param container Container.
+ */
+ public void asyncContainerRelease(RMContainer container) {
+ this.rmContext.getDispatcher().getEventHandler()
+ .handle(new ReleaseContainerEvent(container));
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f49843a9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
index cc14a1e..f9a7219 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/SchedulerApplicationAttempt.java
@@ -76,6 +76,7 @@ import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeCleanContainer
import org.apache.hadoop.yarn.server.resourcemanager.rmnode.RMNodeUpdateContainerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.SchedulingMode;
+
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.SchedulingPlacementSet;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.policy.SchedulableEntity;
@@ -866,10 +867,13 @@ public class SchedulerApplicationAttempt implements SchedulableEntity {
// Mark container for release (set RRs to null, so RM does not think
// it is a recoverable container)
((RMContainerImpl) c).setResourceRequests(null);
- ((AbstractYarnScheduler) rmContext.getScheduler()).completedContainer(c,
- SchedulerUtils.createAbnormalContainerStatus(c.getContainerId(),
- SchedulerUtils.UPDATED_CONTAINER),
- RMContainerEventType.KILL);
+
+ // Release this container async-ly so as to prevent
+ // 'LeafQueue::completedContainer()' from trying to acquire a lock
+ // on the app and queue which can contended for in the reverse order
+ // by the Scheduler thread.
+ ((AbstractYarnScheduler)rmContext.getScheduler())
+ .asyncContainerRelease(c);
tempIter.remove();
}
return updatedContainers;
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f49843a9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
index e4ca003..fde84c4 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
@@ -124,6 +124,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeLabelsU
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeRemovedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeResourceUpdateSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent;
+
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.ReleaseContainerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEventType;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.placement.PlacementSet;
@@ -1491,6 +1493,16 @@ public class CapacityScheduler extends
}
}
break;
+ case RELEASE_CONTAINER:
+ {
+ RMContainer container = ((ReleaseContainerEvent) event).getContainer();
+ completedContainer(container,
+ SchedulerUtils.createAbnormalContainerStatus(
+ container.getContainerId(),
+ SchedulerUtils.RELEASED_CONTAINER),
+ RMContainerEventType.RELEASED);
+ }
+ break;
case KILL_RESERVED_CONTAINER:
{
ContainerPreemptEvent killReservedContainerEvent =
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f49843a9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/distributed/NodeQueueLoadMonitor.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/distributed/NodeQueueLoadMonitor.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/distributed/NodeQueueLoadMonitor.java
index fb67270..ed0ee1e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/distributed/NodeQueueLoadMonitor.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/distributed/NodeQueueLoadMonitor.java
@@ -203,6 +203,10 @@ public class NodeQueueLoadMonitor implements ClusterMonitor {
LOG.debug("Node update event from: " + rmNode.getNodeID());
OpportunisticContainersStatus opportunisticContainersStatus =
rmNode.getOpportunisticContainersStatus();
+ if (opportunisticContainersStatus == null) {
+ opportunisticContainersStatus =
+ OpportunisticContainersStatus.newInstance();
+ }
int estimatedQueueWaitTime =
opportunisticContainersStatus.getEstimatedQueueWaitTime();
int waitQueueLength = opportunisticContainersStatus.getWaitQueueLength();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f49843a9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/ReleaseContainerEvent.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/ReleaseContainerEvent.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/ReleaseContainerEvent.java
new file mode 100644
index 0000000..4f31684
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/ReleaseContainerEvent.java
@@ -0,0 +1,46 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.event;
+
+import org.apache.hadoop.yarn.server.resourcemanager.rmcontainer.RMContainer;
+
+/**
+ * Event used to release a container.
+ */
+public class ReleaseContainerEvent extends SchedulerEvent {
+
+ private final RMContainer container;
+
+ /**
+ * Create Event.
+ * @param rmContainer RMContainer.
+ */
+ public ReleaseContainerEvent(RMContainer rmContainer) {
+ super(SchedulerEventType.RELEASE_CONTAINER);
+ this.container = rmContainer;
+ }
+
+ /**
+ * Get RMContainer.
+ * @return RMContainer.
+ */
+ public RMContainer getContainer() {
+ return container;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f49843a9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/SchedulerEventType.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/SchedulerEventType.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/SchedulerEventType.java
index 35b7c14..229e0bb 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/SchedulerEventType.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/event/SchedulerEventType.java
@@ -38,6 +38,9 @@ public enum SchedulerEventType {
// Source: ContainerAllocationExpirer
CONTAINER_EXPIRED,
+ // Source: SchedulerAppAttempt::pullNewlyUpdatedContainer.
+ RELEASE_CONTAINER,
+
/* Source: SchedulingEditPolicy */
KILL_RESERVED_CONTAINER,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f49843a9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
index 0f417c3..c521250 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fair/FairScheduler.java
@@ -83,6 +83,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSc
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeRemovedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeResourceUpdateSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent;
+
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.ReleaseContainerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.security.RMContainerTokenSecretManager;
import org.apache.hadoop.yarn.util.resource.DefaultResourceCalculator;
@@ -1195,6 +1197,17 @@ public class FairScheduler extends
appAttemptRemovedEvent.getFinalAttemptState(),
appAttemptRemovedEvent.getKeepContainersAcrossAppAttempts());
break;
+ case RELEASE_CONTAINER:
+ if (!(event instanceof ReleaseContainerEvent)) {
+ throw new RuntimeException("Unexpected event type: " + event);
+ }
+ RMContainer container = ((ReleaseContainerEvent) event).getContainer();
+ completedContainer(container,
+ SchedulerUtils.createAbnormalContainerStatus(
+ container.getContainerId(),
+ SchedulerUtils.RELEASED_CONTAINER),
+ RMContainerEventType.RELEASED);
+ break;
case CONTAINER_EXPIRED:
if (!(event instanceof ContainerExpiredSchedulerEvent)) {
throw new RuntimeException("Unexpected event type: " + event);
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f49843a9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
index 92a88b9..94c7e16 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
@@ -65,7 +65,6 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.ContainerUpdates;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.NodeType;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.Queue;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.QueueMetrics;
-import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedContainerChangeRequest;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerAppUtils;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplication;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.SchedulerApplicationAttempt;
@@ -80,6 +79,8 @@ import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeAddedSc
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeRemovedSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeResourceUpdateSchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.NodeUpdateSchedulerEvent;
+
+import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.ReleaseContainerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.event.SchedulerEvent;
import org.apache.hadoop.yarn.server.resourcemanager.scheduler.common.PendingAsk;
import org.apache.hadoop.yarn.server.scheduler.SchedulerRequestKey;
@@ -820,6 +821,18 @@ public class FifoScheduler extends
RMContainerEventType.EXPIRE);
}
break;
+ case RELEASE_CONTAINER: {
+ if (!(event instanceof ReleaseContainerEvent)) {
+ throw new RuntimeException("Unexpected event type: " + event);
+ }
+ RMContainer container = ((ReleaseContainerEvent) event).getContainer();
+ completedContainer(container,
+ SchedulerUtils.createAbnormalContainerStatus(
+ container.getContainerId(),
+ SchedulerUtils.RELEASED_CONTAINER),
+ RMContainerEventType.RELEASED);
+ }
+ break;
default:
LOG.error("Invalid eventtype " + event.getType() + ". Ignoring!");
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f49843a9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
index 7f58711..611c7f2 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
@@ -263,7 +263,7 @@ public class MockNodes {
}
public OpportunisticContainersStatus getOpportunisticContainersStatus() {
- return null;
+ return OpportunisticContainersStatus.newInstance();
}
@Override
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f49843a9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestOpportunisticContainerAllocatorAMService.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestOpportunisticContainerAllocatorAMService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestOpportunisticContainerAllocatorAMService.java
index b885118..9b9eb3c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestOpportunisticContainerAllocatorAMService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestOpportunisticContainerAllocatorAMService.java
@@ -44,6 +44,8 @@ import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.api.records.UpdateContainerRequest;
import org.apache.hadoop.yarn.api.records.UpdatedContainer;
+import org.apache.hadoop.yarn.event.Dispatcher;
+import org.apache.hadoop.yarn.event.DrainDispatcher;
import org.apache.hadoop.yarn.server.api.DistributedSchedulingAMProtocolPB;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
@@ -108,6 +110,7 @@ public class TestOpportunisticContainerAllocatorAMService {
private static final int GB = 1024;
private MockRM rm;
+ private DrainDispatcher dispatcher;
@Before
public void createAndStartRM() {
@@ -120,8 +123,7 @@ public class TestOpportunisticContainerAllocatorAMService {
YarnConfiguration.OPPORTUNISTIC_CONTAINER_ALLOCATION_ENABLED, true);
conf.setInt(
YarnConfiguration.NM_CONTAINER_QUEUING_SORTING_NODES_INTERVAL_MS, 100);
- rm = new MockRM(conf);
- rm.start();
+ startRM(conf);
}
public void createAndStartRMWithAutoUpdateContainer() {
@@ -135,7 +137,17 @@ public class TestOpportunisticContainerAllocatorAMService {
YarnConfiguration.OPPORTUNISTIC_CONTAINER_ALLOCATION_ENABLED, true);
conf.setInt(
YarnConfiguration.NM_CONTAINER_QUEUING_SORTING_NODES_INTERVAL_MS, 100);
- rm = new MockRM(conf);
+ startRM(conf);
+ }
+
+ private void startRM(final YarnConfiguration conf) {
+ dispatcher = new DrainDispatcher();
+ rm = new MockRM(conf) {
+ @Override
+ protected Dispatcher createDispatcher() {
+ return dispatcher;
+ }
+ };
rm.start();
}
@@ -180,17 +192,6 @@ public class TestOpportunisticContainerAllocatorAMService {
nm3.nodeHeartbeat(true);
nm4.nodeHeartbeat(true);
- ((RMNodeImpl) rmNode1)
- .setOpportunisticContainersStatus(getOppurtunisticStatus(-1, 100));
- ((RMNodeImpl) rmNode2)
- .setOpportunisticContainersStatus(getOppurtunisticStatus(-1, 100));
- ((RMNodeImpl) rmNode3)
- .setOpportunisticContainersStatus(getOppurtunisticStatus(-1, 100));
- ((RMNodeImpl) rmNode4)
- .setOpportunisticContainersStatus(getOppurtunisticStatus(-1, 100));
-
- OpportunisticContainerContext ctxt = ((CapacityScheduler) scheduler)
- .getApplicationAttempt(attemptId).getOpportunisticContainerContext();
// Send add and update node events to AM Service.
amservice.handle(new NodeAddedSchedulerEvent(rmNode1));
amservice.handle(new NodeAddedSchedulerEvent(rmNode2));
@@ -246,6 +247,9 @@ public class TestOpportunisticContainerAllocatorAMService {
allocateResponse = am1.allocate(new ArrayList<>(), new ArrayList<>());
Assert.assertEquals(0, allocateResponse.getUpdatedContainers().size());
+ // Wait for scheduler to process all events
+ dispatcher.waitForEventThreadToWait();
+ Thread.sleep(1000);
// Verify Metrics After OPP allocation (Nothing should change again)
verifyMetrics(metrics, 15360, 15, 1024, 1, 1);
@@ -319,6 +323,8 @@ public class TestOpportunisticContainerAllocatorAMService {
Assert.assertEquals(uc.getId(), container.getId());
Assert.assertEquals(uc.getVersion(), container.getVersion() + 2);
+ // Wait for scheduler to finish processing events
+ dispatcher.waitForEventThreadToWait();
// Verify Metrics After OPP allocation :
// Everything should have reverted to what it was
verifyMetrics(metrics, 15360, 15, 1024, 1, 1);
@@ -663,6 +669,7 @@ public class TestOpportunisticContainerAllocatorAMService {
Assert.assertEquals(container.getId(), uc.getContainer().getId());
Assert.assertEquals(Resource.newInstance(2 * GB, 1),
uc.getContainer().getResource());
+ Thread.sleep(1000);
// Check that the container resources are increased in
// NM through NM heartbeat response
@@ -679,6 +686,7 @@ public class TestOpportunisticContainerAllocatorAMService {
ContainerUpdateType.DECREASE_RESOURCE,
Resources.createResource(1 * GB, 1), null)));
Assert.assertEquals(1, allocateResponse.getUpdatedContainers().size());
+ Thread.sleep(1000);
// Check that the container resources are decreased
// in NM through NM heartbeat response
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f49843a9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerResizing.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerResizing.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerResizing.java
index 291a74e..541539d 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerResizing.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestContainerResizing.java
@@ -37,6 +37,8 @@ import org.apache.hadoop.yarn.api.records.ResourceRequest;
import org.apache.hadoop.yarn.api.records.UpdateContainerRequest;
import org.apache.hadoop.yarn.api.records.UpdatedContainer;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.event.Dispatcher;
+import org.apache.hadoop.yarn.event.DrainDispatcher;
import org.apache.hadoop.yarn.server.resourcemanager.MockAM;
import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
@@ -163,11 +165,17 @@ public class TestContainerResizing {
* Application has a container running, try to decrease the container and
* check queue's usage and container resource will be updated.
*/
+ final DrainDispatcher dispatcher = new DrainDispatcher();
MockRM rm1 = new MockRM() {
@Override
public RMNodeLabelsManager createNodeLabelManager() {
return mgr;
}
+
+ @Override
+ protected Dispatcher createDispatcher() {
+ return dispatcher;
+ }
};
rm1.start();
MockNM nm1 = rm1.registerNode("h1:1234", 20 * GB);
@@ -194,6 +202,10 @@ public class TestContainerResizing {
Resources.createResource(1 * GB), null)));
verifyContainerDecreased(response, containerId1, 1 * GB);
+
+ // Wait for scheduler to finish processing kill events..
+ dispatcher.waitForEventThreadToWait();
+
checkUsedResource(rm1, "default", 1 * GB, null);
Assert.assertEquals(1 * GB,
app.getAppAttemptResourceUsage().getUsed().getMemorySize());
@@ -507,11 +519,17 @@ public class TestContainerResizing {
* the increase request reserved, it decreases the reserved container,
* container should be decreased and reservation will be cancelled
*/
+ final DrainDispatcher dispatcher = new DrainDispatcher();
MockRM rm1 = new MockRM() {
@Override
public RMNodeLabelsManager createNodeLabelManager() {
return mgr;
}
+
+ @Override
+ protected Dispatcher createDispatcher() {
+ return dispatcher;
+ }
};
rm1.start();
MockNM nm1 = rm1.registerNode("h1:1234", 8 * GB);
@@ -586,7 +604,8 @@ public class TestContainerResizing {
Resources.createResource(1 * GB), null)));
// Trigger a node heartbeat..
cs.handle(new NodeUpdateSchedulerEvent(rmNode1));
-
+
+ dispatcher.waitForEventThreadToWait();
/* Check statuses after reservation satisfied */
// Increase request should be unreserved
Assert.assertTrue(app.getReservedContainers().isEmpty());
@@ -617,11 +636,17 @@ public class TestContainerResizing {
* So increase container request will be reserved. When app releases
* container2, reserved part should be released as well.
*/
+ final DrainDispatcher dispatcher = new DrainDispatcher();
MockRM rm1 = new MockRM() {
@Override
public RMNodeLabelsManager createNodeLabelManager() {
return mgr;
}
+
+ @Override
+ protected Dispatcher createDispatcher() {
+ return dispatcher;
+ }
};
rm1.start();
MockNM nm1 = rm1.registerNode("h1:1234", 8 * GB);
@@ -687,6 +712,10 @@ public class TestContainerResizing {
cs.handle(new NodeUpdateSchedulerEvent(rmNode1));
am1.allocate(null, null);
+
+ // Wait for scheduler to process all events.
+ dispatcher.waitForEventThreadToWait();
+
/* Check statuses after reservation satisfied */
// Increase request should be unreserved
Assert.assertTrue(app.getReservedContainers().isEmpty());
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f49843a9/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestIncreaseAllocationExpirer.java
----------------------------------------------------------------------
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestIncreaseAllocationExpirer.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestIncreaseAllocationExpirer.java
index a76ed64..d2e28be 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestIncreaseAllocationExpirer.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestIncreaseAllocationExpirer.java
@@ -28,6 +28,8 @@ import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.UpdateContainerError;
import org.apache.hadoop.yarn.api.records.UpdateContainerRequest;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.event.Dispatcher;
+import org.apache.hadoop.yarn.event.DrainDispatcher;
import org.apache.hadoop.yarn.server.resourcemanager.MockAM;
import org.apache.hadoop.yarn.server.resourcemanager.MockNM;
import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
@@ -155,7 +157,13 @@ public class TestIncreaseAllocationExpirer {
*/
// Set the allocation expiration to 5 seconds
conf.setLong(YarnConfiguration.RM_CONTAINER_ALLOC_EXPIRY_INTERVAL_MS, 5000);
- MockRM rm1 = new MockRM(conf);
+ final DrainDispatcher disp = new DrainDispatcher();
+ MockRM rm1 = new MockRM(conf) {
+ @Override
+ protected Dispatcher createDispatcher() {
+ return disp;
+ }
+ };
rm1.start();
MockNM nm1 = rm1.registerNode("127.0.0.1:1234", 20 * GB);
RMApp app1 = rm1.submitApp(1 * GB, "app", "user", null, "default");
@@ -204,6 +212,7 @@ public class TestIncreaseAllocationExpirer {
Assert.assertEquals(
1 * GB, rm1.getResourceScheduler().getRMContainer(containerId2)
.getAllocatedResource().getMemorySize());
+ disp.waitForEventThreadToWait();
// Verify total resource usage is 2G
checkUsedResource(rm1, "default", 2 * GB, null);
Assert.assertEquals(2 * GB,
@@ -420,7 +429,7 @@ public class TestIncreaseAllocationExpirer {
nm1.containerIncreaseStatus(getContainer(
rm1, containerId4, Resources.createResource(6 * GB)));
// Wait for containerId3 token to expire,
- Thread.sleep(10000);
+ Thread.sleep(12000);
am1.allocate(null, null);
@@ -436,13 +445,21 @@ public class TestIncreaseAllocationExpirer {
// Verify NM receives 2 decrease message
List<Container> containersToDecrease =
nm1.nodeHeartbeat(true).getContainersToUpdate();
- Assert.assertEquals(2, containersToDecrease.size());
+ // NOTE: Can be more that 2 depending on which event arrives first.
+ // What is important is the final size of the containers.
+ Assert.assertTrue(containersToDecrease.size() >= 2);
+
// Sort the list to make sure containerId3 is the first
Collections.sort(containersToDecrease);
+ int i = 0;
+ if (containersToDecrease.size() > 2) {
+ Assert.assertEquals(
+ 2 * GB, containersToDecrease.get(i++).getResource().getMemorySize());
+ }
Assert.assertEquals(
- 3 * GB, containersToDecrease.get(0).getResource().getMemorySize());
+ 3 * GB, containersToDecrease.get(i++).getResource().getMemorySize());
Assert.assertEquals(
- 4 * GB, containersToDecrease.get(1).getResource().getMemorySize());
+ 4 * GB, containersToDecrease.get(i++).getResource().getMemorySize());
rm1.stop();
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org
[03/50] [abbrv] hadoop git commit: HDFS-12301. NN File Browser UI:
Navigate to a path when enter is pressed
Posted by as...@apache.org.
HDFS-12301. NN File Browser UI: Navigate to a path when enter is pressed
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/f34646d6
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/f34646d6
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/f34646d6
Branch: refs/heads/YARN-5972
Commit: f34646d652310442cb5339aabbbb269f10dfa838
Parents: d265459
Author: Ravi Prakash <ra...@altiscale.com>
Authored: Tue Aug 15 15:44:59 2017 -0700
Committer: Ravi Prakash <ra...@altiscale.com>
Committed: Tue Aug 15 15:44:59 2017 -0700
----------------------------------------------------------------------
.../hadoop-hdfs/src/main/webapps/hdfs/explorer.js | 6 ++++++
1 file changed, 6 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/f34646d6/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.js
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.js b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.js
index 3e276a9..dae3519 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.js
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/webapps/hdfs/explorer.js
@@ -370,6 +370,12 @@
var b = function() { browse_directory($('#directory').val()); };
$('#btn-nav-directory').click(b);
+ //Also navigate to the directory when a user presses enter.
+ $('#directory').on('keyup', function (e) {
+ if (e.which == 13) {
+ browse_directory($('#directory').val());
+ }
+ });
var dir = window.location.hash.slice(1);
if(dir == "") {
window.location.hash = "/";
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org