You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sn...@apache.org on 2019/08/12 12:03:01 UTC
[hadoop] branch branch-3.2 updated: YARN-9135. NM State store
ResourceMappings serialization are tested with Strings instead of real
Device objects. Contributed by Peter Bacsko
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.2 by this push:
new b20fd9e YARN-9135. NM State store ResourceMappings serialization are tested with Strings instead of real Device objects. Contributed by Peter Bacsko
b20fd9e is described below
commit b20fd9e21295add7e80f07b471bba5c76e433aed
Author: Szilard Nemeth <sn...@apache.org>
AuthorDate: Mon Aug 12 14:02:17 2019 +0200
YARN-9135. NM State store ResourceMappings serialization are tested with Strings instead of real Device objects. Contributed by Peter Bacsko
---
.../node_modules/.bin/apidoc | 1 +
.../node_modules/.bin/markdown-it | 1 +
.../node_modules/.bin/r.js | 1 +
.../node_modules/.bin/r_js | 1 +
.../node_modules/.bin/semver | 1 +
.../node_modules/.bin/shjs | 1 +
.../yarn.lock | 422 +++++++++++++++++++++
.../resources/numa/NumaResourceAllocation.java | 59 ++-
.../resources/numa/NumaResourceAllocator.java | 34 +-
.../recovery/NMLeveldbStateStoreService.java | 5 +-
.../recovery/TestNMLeveldbStateStoreService.java | 52 +--
11 files changed, 519 insertions(+), 59 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/node_modules/.bin/apidoc b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/node_modules/.bin/apidoc
new file mode 120000
index 0000000..a588095
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/node_modules/.bin/apidoc
@@ -0,0 +1 @@
+../../target/generated-sources/vendor/apidoc/bin/apidoc
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/node_modules/.bin/markdown-it b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/node_modules/.bin/markdown-it
new file mode 120000
index 0000000..1ce5019
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/node_modules/.bin/markdown-it
@@ -0,0 +1 @@
+../../target/generated-sources/vendor/markdown-it/bin/markdown-it.js
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/node_modules/.bin/r.js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/node_modules/.bin/r.js
new file mode 120000
index 0000000..d4c664a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/node_modules/.bin/r.js
@@ -0,0 +1 @@
+../../target/generated-sources/vendor/requirejs/bin/r.js
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/node_modules/.bin/r_js b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/node_modules/.bin/r_js
new file mode 120000
index 0000000..d4c664a
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/node_modules/.bin/r_js
@@ -0,0 +1 @@
+../../target/generated-sources/vendor/requirejs/bin/r.js
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/node_modules/.bin/semver b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/node_modules/.bin/semver
new file mode 120000
index 0000000..2bb3fcb
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/node_modules/.bin/semver
@@ -0,0 +1 @@
+../../target/generated-sources/vendor/semver/bin/semver
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/node_modules/.bin/shjs b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/node_modules/.bin/shjs
new file mode 120000
index 0000000..2928a9d
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/node_modules/.bin/shjs
@@ -0,0 +1 @@
+../../target/generated-sources/vendor/shelljs/bin/shjs
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/yarn.lock b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/yarn.lock
new file mode 100644
index 0000000..ddc122b
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-applications/hadoop-yarn-applications-catalog/hadoop-yarn-applications-catalog-webapp/yarn.lock
@@ -0,0 +1,422 @@
+# THIS IS AN AUTOGENERATED FILE. DO NOT EDIT THIS FILE DIRECTLY.
+# yarn lockfile v1
+
+
+angular-loader@~1.6.4:
+ version "1.6.10"
+ resolved "https://registry.yarnpkg.com/angular-loader/-/angular-loader-1.6.10.tgz#5ab1995c65dd38640e6d9d738dbd85eddd5ec310"
+
+angular-mocks@~1.6.4:
+ version "1.6.10"
+ resolved "https://registry.yarnpkg.com/angular-mocks/-/angular-mocks-1.6.10.tgz#6a139e43c461d0c9a5a1acebc91e63db16031176"
+
+angular-route@~1.6.4:
+ version "1.6.10"
+ resolved "https://registry.yarnpkg.com/angular-route/-/angular-route-1.6.10.tgz#4247a32eab19495624623e96c1626dfba17ebf21"
+
+angular@~1.6.4:
+ version "1.6.10"
+ resolved "https://registry.yarnpkg.com/angular/-/angular-1.6.10.tgz#eed3080a34d29d0f681ff119b18ce294e3f74826"
+
+apidoc-core@~0.8.2:
+ version "0.8.3"
+ resolved "https://registry.yarnpkg.com/apidoc-core/-/apidoc-core-0.8.3.tgz#d9d63545829df250d2cca049683a87e775364b96"
+ dependencies:
+ fs-extra "^3.0.1"
+ glob "^7.1.1"
+ iconv-lite "^0.4.17"
+ klaw-sync "^2.1.0"
+ lodash "~4.17.4"
+ semver "~5.3.0"
+
+apidoc@0.17.7:
+ version "0.17.7"
+ resolved "https://registry.yarnpkg.com/apidoc/-/apidoc-0.17.7.tgz#a49090cbd8b3aa457bd054f00e0037e975fd3ee7"
+ dependencies:
+ apidoc-core "~0.8.2"
+ commander "^2.19.0"
+ fs-extra "^7.0.0"
+ lodash "^4.17.10"
+ markdown-it "^8.3.1"
+ winston "^3.0.0"
+
+argparse@^1.0.7:
+ version "1.0.10"
+ resolved "https://registry.yarnpkg.com/argparse/-/argparse-1.0.10.tgz#bcd6791ea5ae09725e17e5ad988134cd40b3d911"
+ dependencies:
+ sprintf-js "~1.0.2"
+
+async@^2.6.1:
+ version "2.6.3"
+ resolved "https://registry.yarnpkg.com/async/-/async-2.6.3.tgz#d72625e2344a3656e3a3ad4fa749fa83299d82ff"
+ dependencies:
+ lodash "^4.17.14"
+
+balanced-match@^1.0.0:
+ version "1.0.0"
+ resolved "https://registry.yarnpkg.com/balanced-match/-/balanced-match-1.0.0.tgz#89b4d199ab2bee49de164ea02b89ce462d71b767"
+
+bootstrap@~3.3.7:
+ version "3.3.7"
+ resolved "https://registry.yarnpkg.com/bootstrap/-/bootstrap-3.3.7.tgz#5a389394549f23330875a3b150656574f8a9eb71"
+
+brace-expansion@^1.1.7:
+ version "1.1.11"
+ resolved "https://registry.yarnpkg.com/brace-expansion/-/brace-expansion-1.1.11.tgz#3c7fcbf529d87226f3d2f52b966ff5271eb441dd"
+ dependencies:
+ balanced-match "^1.0.0"
+ concat-map "0.0.1"
+
+color-convert@^1.9.1:
+ version "1.9.3"
+ resolved "https://registry.yarnpkg.com/color-convert/-/color-convert-1.9.3.tgz#bb71850690e1f136567de629d2d5471deda4c1e8"
+ dependencies:
+ color-name "1.1.3"
+
+color-name@1.1.3:
+ version "1.1.3"
+ resolved "https://registry.yarnpkg.com/color-name/-/color-name-1.1.3.tgz#a7d0558bd89c42f795dd42328f740831ca53bc25"
+
+color-name@^1.0.0:
+ version "1.1.4"
+ resolved "https://registry.yarnpkg.com/color-name/-/color-name-1.1.4.tgz#c2a09a87acbde69543de6f63fa3995c826c536a2"
+
+color-string@^1.5.2:
+ version "1.5.3"
+ resolved "https://registry.yarnpkg.com/color-string/-/color-string-1.5.3.tgz#c9bbc5f01b58b5492f3d6857459cb6590ce204cc"
+ dependencies:
+ color-name "^1.0.0"
+ simple-swizzle "^0.2.2"
+
+color@3.0.x:
+ version "3.0.0"
+ resolved "https://registry.yarnpkg.com/color/-/color-3.0.0.tgz#d920b4328d534a3ac8295d68f7bd4ba6c427be9a"
+ dependencies:
+ color-convert "^1.9.1"
+ color-string "^1.5.2"
+
+colornames@^1.1.1:
+ version "1.1.1"
+ resolved "https://registry.yarnpkg.com/colornames/-/colornames-1.1.1.tgz#f8889030685c7c4ff9e2a559f5077eb76a816f96"
+
+colors@^1.2.1:
+ version "1.3.3"
+ resolved "https://registry.yarnpkg.com/colors/-/colors-1.3.3.tgz#39e005d546afe01e01f9c4ca8fa50f686a01205d"
+
+colorspace@1.1.x:
+ version "1.1.2"
+ resolved "https://registry.yarnpkg.com/colorspace/-/colorspace-1.1.2.tgz#e0128950d082b86a2168580796a0aa5d6c68d8c5"
+ dependencies:
+ color "3.0.x"
+ text-hex "1.0.x"
+
+commander@^2.19.0:
+ version "2.20.0"
+ resolved "https://registry.yarnpkg.com/commander/-/commander-2.20.0.tgz#d58bb2b5c1ee8f87b0d340027e9e94e222c5a422"
+
+concat-map@0.0.1:
+ version "0.0.1"
+ resolved "https://registry.yarnpkg.com/concat-map/-/concat-map-0.0.1.tgz#d8a96bd77fd68df7793a73036a3ba0d5405d477b"
+
+core-util-is@~1.0.0:
+ version "1.0.2"
+ resolved "https://registry.yarnpkg.com/core-util-is/-/core-util-is-1.0.2.tgz#b5fd54220aa2bc5ab57aab7140c940754503c1a7"
+
+diagnostics@^1.1.1:
+ version "1.1.1"
+ resolved "https://registry.yarnpkg.com/diagnostics/-/diagnostics-1.1.1.tgz#cab6ac33df70c9d9a727490ae43ac995a769b22a"
+ dependencies:
+ colorspace "1.1.x"
+ enabled "1.0.x"
+ kuler "1.0.x"
+
+enabled@1.0.x:
+ version "1.0.2"
+ resolved "https://registry.yarnpkg.com/enabled/-/enabled-1.0.2.tgz#965f6513d2c2d1c5f4652b64a2e3396467fc2f93"
+ dependencies:
+ env-variable "0.0.x"
+
+entities@~1.1.1:
+ version "1.1.2"
+ resolved "https://registry.yarnpkg.com/entities/-/entities-1.1.2.tgz#bdfa735299664dfafd34529ed4f8522a275fea56"
+
+env-variable@0.0.x:
+ version "0.0.5"
+ resolved "https://registry.yarnpkg.com/env-variable/-/env-variable-0.0.5.tgz#913dd830bef11e96a039c038d4130604eba37f88"
+
+fast-safe-stringify@^2.0.4:
+ version "2.0.6"
+ resolved "https://registry.yarnpkg.com/fast-safe-stringify/-/fast-safe-stringify-2.0.6.tgz#04b26106cc56681f51a044cfc0d76cf0008ac2c2"
+
+fecha@^2.3.3:
+ version "2.3.3"
+ resolved "https://registry.yarnpkg.com/fecha/-/fecha-2.3.3.tgz#948e74157df1a32fd1b12c3a3c3cdcb6ec9d96cd"
+
+fs-extra@^3.0.1:
+ version "3.0.1"
+ resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-3.0.1.tgz#3794f378c58b342ea7dbbb23095109c4b3b62291"
+ dependencies:
+ graceful-fs "^4.1.2"
+ jsonfile "^3.0.0"
+ universalify "^0.1.0"
+
+fs-extra@^7.0.0:
+ version "7.0.1"
+ resolved "https://registry.yarnpkg.com/fs-extra/-/fs-extra-7.0.1.tgz#4f189c44aa123b895f722804f55ea23eadc348e9"
+ dependencies:
+ graceful-fs "^4.1.2"
+ jsonfile "^4.0.0"
+ universalify "^0.1.0"
+
+fs.realpath@^1.0.0:
+ version "1.0.0"
+ resolved "https://registry.yarnpkg.com/fs.realpath/-/fs.realpath-1.0.0.tgz#1504ad2523158caa40db4a2787cb01411994ea4f"
+
+glob@^7.1.1:
+ version "7.1.4"
+ resolved "https://registry.yarnpkg.com/glob/-/glob-7.1.4.tgz#aa608a2f6c577ad357e1ae5a5c26d9a8d1969255"
+ dependencies:
+ fs.realpath "^1.0.0"
+ inflight "^1.0.4"
+ inherits "2"
+ minimatch "^3.0.4"
+ once "^1.3.0"
+ path-is-absolute "^1.0.0"
+
+graceful-fs@^4.1.11, graceful-fs@^4.1.2, graceful-fs@^4.1.6:
+ version "4.2.0"
+ resolved "https://registry.yarnpkg.com/graceful-fs/-/graceful-fs-4.2.0.tgz#8d8fdc73977cb04104721cb53666c1ca64cd328b"
+
+iconv-lite@^0.4.17:
+ version "0.4.24"
+ resolved "https://registry.yarnpkg.com/iconv-lite/-/iconv-lite-0.4.24.tgz#2022b4b25fbddc21d2f524974a474aafe733908b"
+ dependencies:
+ safer-buffer ">= 2.1.2 < 3"
+
+inflight@^1.0.4:
+ version "1.0.6"
+ resolved "https://registry.yarnpkg.com/inflight/-/inflight-1.0.6.tgz#49bd6331d7d02d0c09bc910a1075ba8165b56df9"
+ dependencies:
+ once "^1.3.0"
+ wrappy "1"
+
+inherits@2, inherits@^2.0.3, inherits@~2.0.3:
+ version "2.0.4"
+ resolved "https://registry.yarnpkg.com/inherits/-/inherits-2.0.4.tgz#0fa2c64f932917c3433a0ded55363aae37416b7c"
+
+is-arrayish@^0.3.1:
+ version "0.3.2"
+ resolved "https://registry.yarnpkg.com/is-arrayish/-/is-arrayish-0.3.2.tgz#4574a2ae56f7ab206896fb431eaeed066fdf8f03"
+
+is-stream@^1.1.0:
+ version "1.1.0"
+ resolved "https://registry.yarnpkg.com/is-stream/-/is-stream-1.1.0.tgz#12d4a3dd4e68e0b79ceb8dbc84173ae80d91ca44"
+
+isarray@~1.0.0:
+ version "1.0.0"
+ resolved "https://registry.yarnpkg.com/isarray/-/isarray-1.0.0.tgz#bb935d48582cba168c06834957a54a3e07124f11"
+
+jquery@3.3.1:
+ version "3.3.1"
+ resolved "https://registry.yarnpkg.com/jquery/-/jquery-3.3.1.tgz#958ce29e81c9790f31be7792df5d4d95fc57fbca"
+
+jsonfile@^3.0.0:
+ version "3.0.1"
+ resolved "https://registry.yarnpkg.com/jsonfile/-/jsonfile-3.0.1.tgz#a5ecc6f65f53f662c4415c7675a0331d0992ec66"
+ optionalDependencies:
+ graceful-fs "^4.1.6"
+
+jsonfile@^4.0.0:
+ version "4.0.0"
+ resolved "https://registry.yarnpkg.com/jsonfile/-/jsonfile-4.0.0.tgz#8771aae0799b64076b76640fca058f9c10e33ecb"
+ optionalDependencies:
+ graceful-fs "^4.1.6"
+
+klaw-sync@^2.1.0:
+ version "2.1.0"
+ resolved "https://registry.yarnpkg.com/klaw-sync/-/klaw-sync-2.1.0.tgz#3d3bcd8600e7bfdef53231c739ff053aed560e44"
+ optionalDependencies:
+ graceful-fs "^4.1.11"
+
+kuler@1.0.x:
+ version "1.0.1"
+ resolved "https://registry.yarnpkg.com/kuler/-/kuler-1.0.1.tgz#ef7c784f36c9fb6e16dd3150d152677b2b0228a6"
+ dependencies:
+ colornames "^1.1.1"
+
+linkify-it@^2.0.0:
+ version "2.2.0"
+ resolved "https://registry.yarnpkg.com/linkify-it/-/linkify-it-2.2.0.tgz#e3b54697e78bf915c70a38acd78fd09e0058b1cf"
+ dependencies:
+ uc.micro "^1.0.1"
+
+lodash@^4.17.10, lodash@^4.17.14, lodash@~4.17.4:
+ version "4.17.14"
+ resolved "https://registry.yarnpkg.com/lodash/-/lodash-4.17.14.tgz#9ce487ae66c96254fe20b599f21b6816028078ba"
+
+logform@^2.1.1:
+ version "2.1.2"
+ resolved "https://registry.yarnpkg.com/logform/-/logform-2.1.2.tgz#957155ebeb67a13164069825ce67ddb5bb2dd360"
+ dependencies:
+ colors "^1.2.1"
+ fast-safe-stringify "^2.0.4"
+ fecha "^2.3.3"
+ ms "^2.1.1"
+ triple-beam "^1.3.0"
+
+markdown-it@^8.3.1:
+ version "8.4.2"
+ resolved "https://registry.yarnpkg.com/markdown-it/-/markdown-it-8.4.2.tgz#386f98998dc15a37722aa7722084f4020bdd9b54"
+ dependencies:
+ argparse "^1.0.7"
+ entities "~1.1.1"
+ linkify-it "^2.0.0"
+ mdurl "^1.0.1"
+ uc.micro "^1.0.5"
+
+mdurl@^1.0.1:
+ version "1.0.1"
+ resolved "https://registry.yarnpkg.com/mdurl/-/mdurl-1.0.1.tgz#fe85b2ec75a59037f2adfec100fd6c601761152e"
+
+minimatch@^3.0.4:
+ version "3.0.4"
+ resolved "https://registry.yarnpkg.com/minimatch/-/minimatch-3.0.4.tgz#5166e286457f03306064be5497e8dbb0c3d32083"
+ dependencies:
+ brace-expansion "^1.1.7"
+
+ms@^2.1.1:
+ version "2.1.2"
+ resolved "https://registry.yarnpkg.com/ms/-/ms-2.1.2.tgz#d09d1f357b443f493382a8eb3ccd183872ae6009"
+
+once@^1.3.0:
+ version "1.4.0"
+ resolved "https://registry.yarnpkg.com/once/-/once-1.4.0.tgz#583b1aa775961d4b113ac17d9c50baef9dd76bd1"
+ dependencies:
+ wrappy "1"
+
+one-time@0.0.4:
+ version "0.0.4"
+ resolved "https://registry.yarnpkg.com/one-time/-/one-time-0.0.4.tgz#f8cdf77884826fe4dff93e3a9cc37b1e4480742e"
+
+path-is-absolute@^1.0.0:
+ version "1.0.1"
+ resolved "https://registry.yarnpkg.com/path-is-absolute/-/path-is-absolute-1.0.1.tgz#174b9268735534ffbc7ace6bf53a5a9e1b5c5f5f"
+
+process-nextick-args@~2.0.0:
+ version "2.0.1"
+ resolved "https://registry.yarnpkg.com/process-nextick-args/-/process-nextick-args-2.0.1.tgz#7820d9b16120cc55ca9ae7792680ae7dba6d7fe2"
+
+readable-stream@^2.3.6:
+ version "2.3.6"
+ resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-2.3.6.tgz#b11c27d88b8ff1fbe070643cf94b0c79ae1b0aaf"
+ dependencies:
+ core-util-is "~1.0.0"
+ inherits "~2.0.3"
+ isarray "~1.0.0"
+ process-nextick-args "~2.0.0"
+ safe-buffer "~5.1.1"
+ string_decoder "~1.1.1"
+ util-deprecate "~1.0.1"
+
+readable-stream@^3.1.1:
+ version "3.4.0"
+ resolved "https://registry.yarnpkg.com/readable-stream/-/readable-stream-3.4.0.tgz#a51c26754658e0a3c21dbf59163bd45ba6f447fc"
+ dependencies:
+ inherits "^2.0.3"
+ string_decoder "^1.1.1"
+ util-deprecate "^1.0.1"
+
+requirejs@^2.1.0:
+ version "2.3.6"
+ resolved "https://registry.yarnpkg.com/requirejs/-/requirejs-2.3.6.tgz#e5093d9601c2829251258c0b9445d4d19fa9e7c9"
+
+roboto-fontface@0.10.0:
+ version "0.10.0"
+ resolved "https://registry.yarnpkg.com/roboto-fontface/-/roboto-fontface-0.10.0.tgz#7eee40cfa18b1f7e4e605eaf1a2740afb6fd71b0"
+
+safe-buffer@~5.1.0, safe-buffer@~5.1.1:
+ version "5.1.2"
+ resolved "https://registry.yarnpkg.com/safe-buffer/-/safe-buffer-5.1.2.tgz#991ec69d296e0313747d59bdfd2b745c35f8828d"
+
+"safer-buffer@>= 2.1.2 < 3":
+ version "2.1.2"
+ resolved "https://registry.yarnpkg.com/safer-buffer/-/safer-buffer-2.1.2.tgz#44fa161b0187b9549dd84bb91802f9bd8385cd6a"
+
+semver@~5.3.0:
+ version "5.3.0"
+ resolved "https://registry.yarnpkg.com/semver/-/semver-5.3.0.tgz#9b2ce5d3de02d17c6012ad326aa6b4d0cf54f94f"
+
+shelljs@^0.2.6:
+ version "0.2.6"
+ resolved "https://registry.yarnpkg.com/shelljs/-/shelljs-0.2.6.tgz#90492d72ffcc8159976baba62fb0f6884f0c3378"
+
+simple-swizzle@^0.2.2:
+ version "0.2.2"
+ resolved "https://registry.yarnpkg.com/simple-swizzle/-/simple-swizzle-0.2.2.tgz#a4da6b635ffcccca33f70d17cb92592de95e557a"
+ dependencies:
+ is-arrayish "^0.3.1"
+
+sprintf-js@~1.0.2:
+ version "1.0.3"
+ resolved "https://registry.yarnpkg.com/sprintf-js/-/sprintf-js-1.0.3.tgz#04e6926f662895354f3dd015203633b857297e2c"
+
+stack-trace@0.0.x:
+ version "0.0.10"
+ resolved "https://registry.yarnpkg.com/stack-trace/-/stack-trace-0.0.10.tgz#547c70b347e8d32b4e108ea1a2a159e5fdde19c0"
+
+string_decoder@^1.1.1:
+ version "1.2.0"
+ resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.2.0.tgz#fe86e738b19544afe70469243b2a1ee9240eae8d"
+ dependencies:
+ safe-buffer "~5.1.0"
+
+string_decoder@~1.1.1:
+ version "1.1.1"
+ resolved "https://registry.yarnpkg.com/string_decoder/-/string_decoder-1.1.1.tgz#9cf1611ba62685d7030ae9e4ba34149c3af03fc8"
+ dependencies:
+ safe-buffer "~5.1.0"
+
+text-hex@1.0.x:
+ version "1.0.0"
+ resolved "https://registry.yarnpkg.com/text-hex/-/text-hex-1.0.0.tgz#69dc9c1b17446ee79a92bf5b884bb4b9127506f5"
+
+triple-beam@^1.2.0, triple-beam@^1.3.0:
+ version "1.3.0"
+ resolved "https://registry.yarnpkg.com/triple-beam/-/triple-beam-1.3.0.tgz#a595214c7298db8339eeeee083e4d10bd8cb8dd9"
+
+uc.micro@^1.0.1, uc.micro@^1.0.5:
+ version "1.0.6"
+ resolved "https://registry.yarnpkg.com/uc.micro/-/uc.micro-1.0.6.tgz#9c411a802a409a91fc6cf74081baba34b24499ac"
+
+universalify@^0.1.0:
+ version "0.1.2"
+ resolved "https://registry.yarnpkg.com/universalify/-/universalify-0.1.2.tgz#b646f69be3942dabcecc9d6639c80dc105efaa66"
+
+util-deprecate@^1.0.1, util-deprecate@~1.0.1:
+ version "1.0.2"
+ resolved "https://registry.yarnpkg.com/util-deprecate/-/util-deprecate-1.0.2.tgz#450d4dc9fa70de732762fbd2d4a28981419a0ccf"
+
+winston-transport@^4.3.0:
+ version "4.3.0"
+ resolved "https://registry.yarnpkg.com/winston-transport/-/winston-transport-4.3.0.tgz#df68c0c202482c448d9b47313c07304c2d7c2c66"
+ dependencies:
+ readable-stream "^2.3.6"
+ triple-beam "^1.2.0"
+
+winston@^3.0.0:
+ version "3.2.1"
+ resolved "https://registry.yarnpkg.com/winston/-/winston-3.2.1.tgz#63061377976c73584028be2490a1846055f77f07"
+ dependencies:
+ async "^2.6.1"
+ diagnostics "^1.1.1"
+ is-stream "^1.1.0"
+ logform "^2.1.1"
+ one-time "0.0.4"
+ readable-stream "^3.1.1"
+ stack-trace "0.0.x"
+ triple-beam "^1.3.0"
+ winston-transport "^4.3.0"
+
+wrappy@1:
+ version "1.0.2"
+ resolved "https://registry.yarnpkg.com/wrappy/-/wrappy-1.0.2.tgz#b5243d8f3ec1aa35f1364605bc0d1036e30ab69f"
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/numa/NumaResourceAllocation.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/numa/NumaResourceAllocation.java
index f8d4739..e91ac3e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/numa/NumaResourceAllocation.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/numa/NumaResourceAllocation.java
@@ -17,9 +17,11 @@
*/
package org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.numa;
+import com.google.common.collect.ImmutableMap;
+
import java.io.Serializable;
-import java.util.HashMap;
import java.util.Map;
+import java.util.Objects;
import java.util.Set;
/**
@@ -28,27 +30,18 @@ import java.util.Set;
*/
public class NumaResourceAllocation implements Serializable {
private static final long serialVersionUID = 6339719798446595123L;
- private Map<String, Long> nodeVsMemory;
- private Map<String, Integer> nodeVsCpus;
+ private final ImmutableMap<String, Long> nodeVsMemory;
+ private final ImmutableMap<String, Integer> nodeVsCpus;
- public NumaResourceAllocation() {
- nodeVsMemory = new HashMap<>();
- nodeVsCpus = new HashMap<>();
+ public NumaResourceAllocation(Map<String, Long> memoryAllocations,
+ Map<String, Integer> cpuAllocations) {
+ nodeVsMemory = ImmutableMap.copyOf(memoryAllocations);
+ nodeVsCpus = ImmutableMap.copyOf(cpuAllocations);
}
public NumaResourceAllocation(String memNodeId, long memory, String cpuNodeId,
int cpus) {
- this();
- nodeVsMemory.put(memNodeId, memory);
- nodeVsCpus.put(cpuNodeId, cpus);
- }
-
- public void addMemoryNode(String memNodeId, long memory) {
- nodeVsMemory.put(memNodeId, memory);
- }
-
- public void addCpuNode(String cpuNodeId, int cpus) {
- nodeVsCpus.put(cpuNodeId, cpus);
+ this(ImmutableMap.of(memNodeId, memory), ImmutableMap.of(cpuNodeId, cpus));
}
public Set<String> getMemNodes() {
@@ -59,11 +52,37 @@ public class NumaResourceAllocation implements Serializable {
return nodeVsCpus.keySet();
}
- public Map<String, Long> getNodeVsMemory() {
+ public ImmutableMap<String, Long> getNodeVsMemory() {
return nodeVsMemory;
}
- public Map<String, Integer> getNodeVsCpus() {
+ public ImmutableMap<String, Integer> getNodeVsCpus() {
return nodeVsCpus;
}
-}
+
+ @Override
+ public String toString() {
+ return "NumaResourceAllocation{" +
+ "nodeVsMemory=" + nodeVsMemory +
+ ", nodeVsCpus=" + nodeVsCpus +
+ '}';
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ }
+ if (o == null || getClass() != o.getClass()) {
+ return false;
+ }
+ NumaResourceAllocation that = (NumaResourceAllocation) o;
+ return Objects.equals(nodeVsMemory, that.nodeVsMemory) &&
+ Objects.equals(nodeVsCpus, that.nodeVsCpus);
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hash(nodeVsMemory, nodeVsCpus);
+ }
+}
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/numa/NumaResourceAllocator.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/numa/NumaResourceAllocator.java
index e152bda..f95e55e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/numa/NumaResourceAllocator.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/linux/resources/numa/NumaResourceAllocator.java
@@ -31,6 +31,7 @@ import java.util.regex.Pattern;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import com.google.common.collect.Maps;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.util.Shell.ShellCommandExecutor;
import org.apache.hadoop.util.StringUtils;
@@ -247,17 +248,19 @@ public class NumaResourceAllocator {
// If there is no single node matched for the container resource
// Check the NUMA nodes for Memory resources
- NumaResourceAllocation assignedNumaNodeInfo = new NumaResourceAllocation();
- long memreq = resource.getMemorySize();
+ long memoryRequirement = resource.getMemorySize();
+ Map<String, Long> memoryAllocations = Maps.newHashMap();
for (NumaNodeResource numaNode : numaNodesList) {
- long memrem = numaNode.assignAvailableMemory(memreq, containerId);
- assignedNumaNodeInfo.addMemoryNode(numaNode.getNodeId(), memreq - memrem);
- memreq = memrem;
- if (memreq == 0) {
+ long memoryRemaining = numaNode.
+ assignAvailableMemory(memoryRequirement, containerId);
+ memoryAllocations.put(numaNode.getNodeId(),
+ memoryRequirement - memoryRemaining);
+ memoryRequirement = memoryRemaining;
+ if (memoryRequirement == 0) {
break;
}
}
- if (memreq != 0) {
+ if (memoryRequirement != 0) {
LOG.info("There is no available memory:" + resource.getMemorySize()
+ " in numa nodes for " + containerId);
releaseNumaResource(containerId);
@@ -265,26 +268,31 @@ public class NumaResourceAllocator {
}
// Check the NUMA nodes for CPU resources
- int cpusreq = resource.getVirtualCores();
+ int cpusRequirement = resource.getVirtualCores();
+ Map<String, Integer> cpuAllocations = Maps.newHashMap();
for (int index = 0; index < numaNodesList.size(); index++) {
NumaNodeResource numaNode = numaNodesList
.get((currentAssignNode + index) % numaNodesList.size());
- int cpusrem = numaNode.assignAvailableCpus(cpusreq, containerId);
- assignedNumaNodeInfo.addCpuNode(numaNode.getNodeId(), cpusreq - cpusrem);
- cpusreq = cpusrem;
- if (cpusreq == 0) {
+ int cpusRemaining = numaNode.
+ assignAvailableCpus(cpusRequirement, containerId);
+ cpuAllocations.put(numaNode.getNodeId(), cpusRequirement - cpusRemaining);
+ cpusRequirement = cpusRemaining;
+ if (cpusRequirement == 0) {
currentAssignNode = (currentAssignNode + index + 1)
% numaNodesList.size();
break;
}
}
- if (cpusreq != 0) {
+ if (cpusRequirement != 0) {
LOG.info("There are no available cpus:" + resource.getVirtualCores()
+ " in numa nodes for " + containerId);
releaseNumaResource(containerId);
return null;
}
+
+ NumaResourceAllocation assignedNumaNodeInfo =
+ new NumaResourceAllocation(memoryAllocations, cpuAllocations);
LOG.info("Assigning multiple NUMA nodes ("
+ StringUtils.join(",", assignedNumaNodeInfo.getMemNodes())
+ ") for memory, ("
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java
index 1d7771a..8de94a5 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/recovery/NMLeveldbStateStoreService.java
@@ -1459,8 +1459,7 @@ public class NMLeveldbStateStoreService extends NMStateStoreService {
String keyResChng = CONTAINERS_KEY_PREFIX + container.getContainerId().toString()
+ CONTAINER_ASSIGNED_RESOURCES_KEY_SUFFIX + resourceType;
try {
- WriteBatch batch = db.createWriteBatch();
- try {
+ try (WriteBatch batch = db.createWriteBatch()) {
ResourceMappings.AssignedResources res =
new ResourceMappings.AssignedResources();
res.updateAssignedResources(assignedResources);
@@ -1468,8 +1467,6 @@ public class NMLeveldbStateStoreService extends NMStateStoreService {
// New value will overwrite old values for the same key
batch.put(bytes(keyResChng), res.toBytes());
db.write(batch);
- } finally {
- batch.close();
}
} catch (DBException e) {
markStoreUnHealthy(e);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/TestNMLeveldbStateStoreService.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/TestNMLeveldbStateStoreService.java
index 87208f7..c4c194c 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/TestNMLeveldbStateStoreService.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/recovery/TestNMLeveldbStateStoreService.java
@@ -75,6 +75,9 @@ import org.apache.hadoop.yarn.server.api.records.MasterKey;
import org.apache.hadoop.yarn.server.nodemanager.amrmproxy.AMRMProxyTokenSecretManager;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.ResourceMappings;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.fpga.FpgaResourceAllocator.FpgaDevice;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.linux.resources.numa.NumaResourceAllocation;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.resourceplugin.gpu.GpuDevice;
import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService.LocalResourceTrackerState;
import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService.RecoveredAMRMProxyState;
import org.apache.hadoop.yarn.server.nodemanager.recovery.NMStateStoreService.RecoveredApplicationsState;
@@ -1448,7 +1451,7 @@ public class TestNMLeveldbStateStoreService {
@Test
public void testStateStoreForResourceMapping() throws IOException {
- // test empty when no state
+ // test that stateStore is initially empty
List<RecoveredContainerState> recoveredContainers =
loadContainersState(stateStore.getContainerStateIterator());
assertTrue(recoveredContainers.isEmpty());
@@ -1464,38 +1467,43 @@ public class TestNMLeveldbStateStoreService {
ResourceMappings resourceMappings = new ResourceMappings();
when(container.getResourceMappings()).thenReturn(resourceMappings);
- // Store ResourceMapping
stateStore.storeAssignedResources(container, "gpu",
- Arrays.asList("1", "2", "3"));
- // This will overwrite above
- List<Serializable> gpuRes1 = Arrays.asList("1", "2", "4");
+ Arrays.asList(new GpuDevice(1, 1), new GpuDevice(2, 2),
+ new GpuDevice(3, 3)));
+
+ // This will overwrite the above
+ List<Serializable> gpuRes1 = Arrays.asList(
+ new GpuDevice(1, 1), new GpuDevice(2, 2), new GpuDevice(4, 4));
stateStore.storeAssignedResources(container, "gpu", gpuRes1);
- List<Serializable> fpgaRes = Arrays.asList("3", "4", "5", "6");
+
+ List<Serializable> fpgaRes = Arrays.asList(
+ new FpgaDevice("testType", 3, 3, "testIPID"),
+ new FpgaDevice("testType", 4, 4, "testIPID"),
+ new FpgaDevice("testType", 5, 5, "testIPID"),
+ new FpgaDevice("testType", 6, 6, "testIPID"));
stateStore.storeAssignedResources(container, "fpga", fpgaRes);
- List<Serializable> numaRes = Arrays.asList("numa1");
+
+ List<Serializable> numaRes = Arrays.asList(
+ new NumaResourceAllocation("testmemNodeId", 2048, "testCpuNodeId", 10));
stateStore.storeAssignedResources(container, "numa", numaRes);
- // add a invalid key
restartStateStore();
recoveredContainers =
loadContainersState(stateStore.getContainerStateIterator());
assertEquals(1, recoveredContainers.size());
RecoveredContainerState rcs = recoveredContainers.get(0);
- List<Serializable> res = rcs.getResourceMappings()
+ List<Serializable> resources = rcs.getResourceMappings()
.getAssignedResources("gpu");
- Assert.assertTrue(res.equals(gpuRes1));
- Assert.assertTrue(
- resourceMappings.getAssignedResources("gpu").equals(gpuRes1));
-
- res = rcs.getResourceMappings().getAssignedResources("fpga");
- Assert.assertTrue(res.equals(fpgaRes));
- Assert.assertTrue(
- resourceMappings.getAssignedResources("fpga").equals(fpgaRes));
-
- res = rcs.getResourceMappings().getAssignedResources("numa");
- Assert.assertTrue(res.equals(numaRes));
- Assert.assertTrue(
- resourceMappings.getAssignedResources("numa").equals(numaRes));
+ Assert.assertEquals(gpuRes1, resources);
+ Assert.assertEquals(gpuRes1, resourceMappings.getAssignedResources("gpu"));
+
+ resources = rcs.getResourceMappings().getAssignedResources("fpga");
+ Assert.assertEquals(fpgaRes, resources);
+ Assert.assertEquals(fpgaRes, resourceMappings.getAssignedResources("fpga"));
+
+ resources = rcs.getResourceMappings().getAssignedResources("numa");
+ Assert.assertEquals(numaRes, resources);
+ Assert.assertEquals(numaRes, resourceMappings.getAssignedResources("numa"));
}
@Test
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org