You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-issues@hadoop.apache.org by GitBox <gi...@apache.org> on 2022/11/14 06:55:40 UTC
[GitHub] [hadoop] slfan1989 opened a new pull request, #5131: YARN-11350. [Federation] Router Support DelegationToken With ZK.
slfan1989 opened a new pull request, #5131:
URL: https://github.com/apache/hadoop/pull/5131
JIRA: YARN-11350. [Federation] Router Support DelegationToken With ZK.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-issues-help@hadoop.apache.org
[GitHub] [hadoop] slfan1989 commented on a diff in pull request #5131: YARN-11350. [Federation] Router Support DelegationToken With ZK.
Posted by GitBox <gi...@apache.org>.
slfan1989 commented on code in PR #5131:
URL: https://github.com/apache/hadoop/pull/5131#discussion_r1039707916
##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/TestZookeeperFederationStateStore.java:
##########
@@ -171,38 +203,293 @@ public void testMetricsInited() throws Exception {
MetricsRecords.assertMetric(record, "UpdateReservationHomeSubClusterNumOps", expectOps);
}
- @Test(expected = NotImplementedException.class)
+ @Test
public void testStoreNewMasterKey() throws Exception {
- super.testStoreNewMasterKey();
+
+ // Manually create a DelegationKey,
+ // and call the interface storeNewMasterKey to write the data to zk.
+ DelegationKey key = new DelegationKey(1234, Time.now() + 60 * 60, "keyBytes".getBytes());
+ RouterMasterKey paramRouterMasterKey = RouterMasterKey.newInstance(key.getKeyId(),
+ ByteBuffer.wrap(key.getEncodedKey()), key.getExpiryDate());
+ FederationStateStore stateStore = this.getStateStore();
+
+ assertTrue(stateStore instanceof ZookeeperFederationStateStore);
+
+ // Compare the data returned by the storeNewMasterKey
+ // interface with the data queried by zk, and ensure that the data is consistent.
+ RouterMasterKeyRequest routerMasterKeyRequest =
+ RouterMasterKeyRequest.newInstance(paramRouterMasterKey);
+ RouterMasterKeyResponse response = stateStore.storeNewMasterKey(routerMasterKeyRequest);
+ assertNotNull(response);
+ RouterMasterKey respRouterMasterKey = response.getRouterMasterKey();
+ assertNotNull(respRouterMasterKey);
+
+ // Get Data From zk.
+ String nodeName = ROUTER_RM_DELEGATION_KEY_PREFIX + key.getKeyId();
+ String nodePath = ZNODE_MASTER_KEY_PREFIX + nodeName;
+ RouterMasterKey zkRouterMasterKey = getRouterMasterKeyFromZK(nodePath);
+
+ assertNotNull(zkRouterMasterKey);
+ assertEquals(paramRouterMasterKey, respRouterMasterKey);
+ assertEquals(paramRouterMasterKey, zkRouterMasterKey);
+ assertEquals(zkRouterMasterKey, respRouterMasterKey);
}
- @Test(expected = NotImplementedException.class)
+ @Test
public void testGetMasterKeyByDelegationKey() throws YarnException, IOException {
- super.testGetMasterKeyByDelegationKey();
+
+ // Manually create a DelegationKey,
+ // and call the interface storeNewMasterKey to write the data to zk.
+ DelegationKey key = new DelegationKey(5678, Time.now() + 60 * 60, "keyBytes".getBytes());
+ RouterMasterKey paramRouterMasterKey = RouterMasterKey.newInstance(key.getKeyId(),
+ ByteBuffer.wrap(key.getEncodedKey()), key.getExpiryDate());
+ FederationStateStore stateStore = this.getStateStore();
+
+ assertTrue(stateStore instanceof ZookeeperFederationStateStore);
Review Comment:
Thank you very much for helping to review the code, I will fix it.
##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/TestZookeeperFederationStateStore.java:
##########
@@ -171,38 +203,293 @@ public void testMetricsInited() throws Exception {
MetricsRecords.assertMetric(record, "UpdateReservationHomeSubClusterNumOps", expectOps);
}
- @Test(expected = NotImplementedException.class)
+ @Test
public void testStoreNewMasterKey() throws Exception {
- super.testStoreNewMasterKey();
+
+ // Manually create a DelegationKey,
+ // and call the interface storeNewMasterKey to write the data to zk.
+ DelegationKey key = new DelegationKey(1234, Time.now() + 60 * 60, "keyBytes".getBytes());
+ RouterMasterKey paramRouterMasterKey = RouterMasterKey.newInstance(key.getKeyId(),
+ ByteBuffer.wrap(key.getEncodedKey()), key.getExpiryDate());
+ FederationStateStore stateStore = this.getStateStore();
+
+ assertTrue(stateStore instanceof ZookeeperFederationStateStore);
+
+ // Compare the data returned by the storeNewMasterKey
+ // interface with the data queried by zk, and ensure that the data is consistent.
+ RouterMasterKeyRequest routerMasterKeyRequest =
+ RouterMasterKeyRequest.newInstance(paramRouterMasterKey);
+ RouterMasterKeyResponse response = stateStore.storeNewMasterKey(routerMasterKeyRequest);
+ assertNotNull(response);
+ RouterMasterKey respRouterMasterKey = response.getRouterMasterKey();
+ assertNotNull(respRouterMasterKey);
+
+ // Get Data From zk.
+ String nodeName = ROUTER_RM_DELEGATION_KEY_PREFIX + key.getKeyId();
+ String nodePath = ZNODE_MASTER_KEY_PREFIX + nodeName;
+ RouterMasterKey zkRouterMasterKey = getRouterMasterKeyFromZK(nodePath);
+
+ assertNotNull(zkRouterMasterKey);
+ assertEquals(paramRouterMasterKey, respRouterMasterKey);
+ assertEquals(paramRouterMasterKey, zkRouterMasterKey);
+ assertEquals(zkRouterMasterKey, respRouterMasterKey);
}
- @Test(expected = NotImplementedException.class)
+ @Test
public void testGetMasterKeyByDelegationKey() throws YarnException, IOException {
- super.testGetMasterKeyByDelegationKey();
+
+ // Manually create a DelegationKey,
+ // and call the interface storeNewMasterKey to write the data to zk.
+ DelegationKey key = new DelegationKey(5678, Time.now() + 60 * 60, "keyBytes".getBytes());
+ RouterMasterKey paramRouterMasterKey = RouterMasterKey.newInstance(key.getKeyId(),
+ ByteBuffer.wrap(key.getEncodedKey()), key.getExpiryDate());
+ FederationStateStore stateStore = this.getStateStore();
+
+ assertTrue(stateStore instanceof ZookeeperFederationStateStore);
+
+ // Call the getMasterKeyByDelegationKey interface of stateStore to get the MasterKey data.
+ RouterMasterKeyRequest routerMasterKeyRequest =
+ RouterMasterKeyRequest.newInstance(paramRouterMasterKey);
+ RouterMasterKeyResponse response = stateStore.storeNewMasterKey(routerMasterKeyRequest);
+ assertNotNull(response);
+
+ // Get Data From zk.
+ String nodeName = ROUTER_RM_DELEGATION_KEY_PREFIX + key.getKeyId();
+ String nodePath = ZNODE_MASTER_KEY_PREFIX + nodeName;
+ RouterMasterKey zkRouterMasterKey = getRouterMasterKeyFromZK(nodePath);
+
+ // Call the getMasterKeyByDelegationKey interface to get the returned result.
+ // The zk data should be consistent with the returned data.
+ RouterMasterKeyResponse response1 =
+ stateStore.getMasterKeyByDelegationKey(routerMasterKeyRequest);
+ assertNotNull(response1);
+ RouterMasterKey respRouterMasterKey = response1.getRouterMasterKey();
+ assertEquals(paramRouterMasterKey, respRouterMasterKey);
+ assertEquals(paramRouterMasterKey, zkRouterMasterKey);
+ assertEquals(zkRouterMasterKey, respRouterMasterKey);
}
- @Test(expected = NotImplementedException.class)
+ @Test
public void testRemoveStoredMasterKey() throws YarnException, IOException {
- super.testRemoveStoredMasterKey();
+
+ // Manually create a DelegationKey,
+ // and call the interface storeNewMasterKey to write the data to zk.
+ DelegationKey key = new DelegationKey(2345, Time.now() + 60 * 60, "keyBytes".getBytes());
+ RouterMasterKey paramRouterMasterKey = RouterMasterKey.newInstance(key.getKeyId(),
+ ByteBuffer.wrap(key.getEncodedKey()), key.getExpiryDate());
+ FederationStateStore stateStore = this.getStateStore();
+
+ assertTrue(stateStore instanceof ZookeeperFederationStateStore);
+
+ // We need to ensure that the returned result is not empty.
+ RouterMasterKeyRequest routerMasterKeyRequest =
+ RouterMasterKeyRequest.newInstance(paramRouterMasterKey);
+ RouterMasterKeyResponse response = stateStore.storeNewMasterKey(routerMasterKeyRequest);
+ assertNotNull(response);
+
+ // We will check if delegationToken exists in zk.
+ String nodeName = ROUTER_RM_DELEGATION_KEY_PREFIX + key.getKeyId();
+ String nodePath = ZNODE_MASTER_KEY_PREFIX + nodeName;
+ assertTrue(isExists(nodePath));
+
+ // Call removeStoredMasterKey to remove the MasterKey data in zk.
+ RouterMasterKeyResponse response1 = stateStore.removeStoredMasterKey(routerMasterKeyRequest);
+ assertNotNull(response1);
+ RouterMasterKey respRouterMasterKey = response1.getRouterMasterKey();
+ assertNotNull(respRouterMasterKey);
+ assertEquals(paramRouterMasterKey, respRouterMasterKey);
+
+ // We have removed the RouterMasterKey data from zk,
+ // the path should be empty at this point.
+ assertFalse(isExists(nodePath));
}
- @Test(expected = NotImplementedException.class)
- public void testStoreNewToken() throws IOException, YarnException {
- super.testStoreNewToken();
+ @Test
+ public void testStoreNewToken() throws YarnException, IOException {
+
+ // We manually generate the DelegationToken,
+ // and then call the StoreNewToken method to store the Token in zk.
+ RMDelegationTokenIdentifier identifier = new RMDelegationTokenIdentifier(
+ new Text("owner2"), new Text("renewer2"), new Text("realuser2"));
+ FederationStateStore stateStore = this.getStateStore();
+ int seqNum = stateStore.incrementDelegationTokenSeqNum();
+ identifier.setSequenceNumber(seqNum);
+ Long renewDate = Time.now();
+
+ // Store new rm-token
+ RouterStoreToken paramStoreToken = RouterStoreToken.newInstance(identifier, renewDate);
+ RouterRMTokenRequest request = RouterRMTokenRequest.newInstance(paramStoreToken);
+ RouterRMTokenResponse routerRMTokenResponse = stateStore.storeNewToken(request);
+ assertNotNull(routerRMTokenResponse);
+ RouterStoreToken respStoreToken = routerRMTokenResponse.getRouterStoreToken();
+ assertNotNull(respStoreToken);
+
+ // Get delegationToken Path
+ String nodeName = ROUTER_RM_DELEGATION_TOKEN_PREFIX + identifier.getSequenceNumber();
+ String nodePath = getNodePath(ZNODE_DT_PREFIX, nodeName);
+
+ // Check if the path exists, we expect the result to exist.
+ assertTrue(isExists(nodePath));
+
+ // Check whether the token (paramStoreToken)
+ // We generated is consistent with the data stored in zk.
+ // We expect data to be consistent.
+ RouterStoreToken zkRouterStoreToken = getStoreTokenFromZK(nodePath);
+ assertNotNull(zkRouterStoreToken);
+ assertEquals(paramStoreToken, zkRouterStoreToken);
+ assertEquals(respStoreToken, zkRouterStoreToken);
}
- @Test(expected = NotImplementedException.class)
- public void testUpdateStoredToken() throws IOException, YarnException {
- super.testUpdateStoredToken();
+ @Test
+ public void testUpdateStoredToken() throws YarnException, IOException {
+
+ // We manually generate the DelegationToken,
+ // and then call the StoreNewToken method to store the Token in zk.
+ RMDelegationTokenIdentifier identifier = new RMDelegationTokenIdentifier(
+ new Text("owner2"), new Text("renewer2"), new Text("realuser2"));
+ FederationStateStore stateStore = this.getStateStore();
+ int seqNum = stateStore.incrementDelegationTokenSeqNum();
+ identifier.setSequenceNumber(seqNum);
+ Long renewDate = Time.now();
+
+ // Store new rm-token()
+ RouterStoreToken storeToken = RouterStoreToken.newInstance(identifier, renewDate);
+ RouterRMTokenRequest request = RouterRMTokenRequest.newInstance(storeToken);
+ RouterRMTokenResponse routerRMTokenResponse = stateStore.storeNewToken(request);
+ Assert.assertNotNull(routerRMTokenResponse);
+
+ // We are ready to update some data renewDate2 & sequenceNumber2
+ Long renewDate2 = Time.now();
+ int sequenceNumber2 = stateStore.incrementDelegationTokenSeqNum();
+ identifier.setSequenceNumber(sequenceNumber2);
+
+ // Update rm-token
+ RouterStoreToken paramStoreToken = RouterStoreToken.newInstance(identifier, renewDate2);
+ RouterRMTokenRequest updateTokenRequest = RouterRMTokenRequest.newInstance(paramStoreToken);
+ RouterRMTokenResponse updateTokenResponse = stateStore.updateStoredToken(updateTokenRequest);
+ Assert.assertNotNull(updateTokenResponse);
+ RouterStoreToken updateTokenResp = updateTokenResponse.getRouterStoreToken();
+
+ // Get delegationToken Path
+ String nodeName = ROUTER_RM_DELEGATION_TOKEN_PREFIX + identifier.getSequenceNumber();
+ String nodePath = getNodePath(ZNODE_DT_PREFIX, nodeName);
+
+ // Check if the path exists, we expect the result to exist.
+ assertTrue(isExists(nodePath));
+
+ // Check whether the token (paramStoreToken)
+ // We generated is consistent with the data stored in zk.
+ // We expect data to be consistent.
+ RouterStoreToken zkRouterStoreToken = getStoreTokenFromZK(nodePath);
+ assertNotNull(zkRouterStoreToken);
+ assertEquals(paramStoreToken, zkRouterStoreToken);
+ assertEquals(updateTokenResp, zkRouterStoreToken);
}
- @Test(expected = NotImplementedException.class)
- public void testRemoveStoredToken() throws IOException, YarnException {
- super.testRemoveStoredToken();
+ @Test
+ public void testRemoveStoredToken() throws YarnException, IOException {
+
+ // We manually generate the DelegationToken,
+ // and then call the StoreNewToken method to store the Token in zk.
+ RMDelegationTokenIdentifier identifier = new RMDelegationTokenIdentifier(
+ new Text("owner2"), new Text("renewer2"), new Text("realuser2"));
+ FederationStateStore stateStore = this.getStateStore();
+ int seqNum = stateStore.incrementDelegationTokenSeqNum();
+ identifier.setSequenceNumber(seqNum);
+ Long renewDate = Time.now();
+
+ // Store new rm-token
+ RouterStoreToken storeToken = RouterStoreToken.newInstance(identifier, renewDate);
+ RouterRMTokenRequest request = RouterRMTokenRequest.newInstance(storeToken);
+ RouterRMTokenResponse routerRMTokenResponse = stateStore.storeNewToken(request);
+ Assert.assertNotNull(routerRMTokenResponse);
Review Comment:
I will fix it.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-issues-help@hadoop.apache.org
[GitHub] [hadoop] hadoop-yetus commented on pull request #5131: YARN-11350. [Federation] Router Support DelegationToken With ZK.
Posted by GitBox <gi...@apache.org>.
hadoop-yetus commented on PR #5131:
URL: https://github.com/apache/hadoop/pull/5131#issuecomment-1335059141
:broken_heart: **-1 overall**
| Vote | Subsystem | Runtime | Logfile | Comment |
|:----:|----------:|--------:|:--------:|:-------:|
| +0 :ok: | reexec | 0m 39s | | Docker mode activated. |
|||| _ Prechecks _ |
| +1 :green_heart: | dupname | 0m 1s | | No case conflicting files found. |
| +0 :ok: | codespell | 0m 0s | | codespell was not available. |
| +0 :ok: | detsecrets | 0m 0s | | detect-secrets was not available. |
| +1 :green_heart: | @author | 0m 0s | | The patch does not contain any @author tags. |
| +1 :green_heart: | test4tests | 0m 0s | | The patch appears to include 1 new or modified test files. |
|||| _ trunk Compile Tests _ |
| +0 :ok: | mvndep | 15m 23s | | Maven dependency ordering for branch |
| +1 :green_heart: | mvninstall | 26m 10s | | trunk passed |
| +1 :green_heart: | compile | 4m 11s | | trunk passed with JDK Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 |
| +1 :green_heart: | compile | 3m 36s | | trunk passed with JDK Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | checkstyle | 1m 33s | | trunk passed |
| +1 :green_heart: | mvnsite | 3m 6s | | trunk passed |
| +1 :green_heart: | javadoc | 2m 58s | | trunk passed with JDK Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 |
| +1 :green_heart: | javadoc | 2m 44s | | trunk passed with JDK Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | spotbugs | 5m 8s | | trunk passed |
| +1 :green_heart: | shadedclient | 25m 8s | | branch has no errors when building and testing our client artifacts. |
| -0 :warning: | patch | 25m 37s | | Used diff version of patch file. Binary files and potentially other changes not applied. Please rebase and squash commits if necessary. |
|||| _ Patch Compile Tests _ |
| +0 :ok: | mvndep | 0m 29s | | Maven dependency ordering for patch |
| +1 :green_heart: | mvninstall | 2m 14s | | the patch passed |
| +1 :green_heart: | compile | 3m 55s | | the patch passed with JDK Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 |
| +1 :green_heart: | javac | 3m 55s | | the patch passed |
| +1 :green_heart: | compile | 3m 34s | | the patch passed with JDK Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | javac | 3m 34s | | the patch passed |
| +1 :green_heart: | blanks | 0m 0s | | The patch has no blanks issues. |
| -0 :warning: | checkstyle | 1m 14s | [/results-checkstyle-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/15/artifact/out/results-checkstyle-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server.txt) | hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server: The patch generated 1 new + 10 unchanged - 0 fixed = 11 total (was 10) |
| +1 :green_heart: | mvnsite | 2m 33s | | the patch passed |
| -1 :x: | javadoc | 0m 41s | [/patch-javadoc-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-common-jdkUbuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/15/artifact/out/patch-javadoc-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-common-jdkUbuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04.txt) | hadoop-yarn-server-common in the patch failed with JDK Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04. |
| -1 :x: | javadoc | 0m 38s | [/patch-javadoc-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-common-jdkPrivateBuild-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/15/artifact/out/patch-javadoc-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-common-jdkPrivateBuild-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07.txt) | hadoop-yarn-server-common in the patch failed with JDK Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07. |
| +1 :green_heart: | spotbugs | 4m 54s | | the patch passed |
| +1 :green_heart: | shadedclient | 20m 52s | | patch has no errors when building and testing our client artifacts. |
|||| _ Other Tests _ |
| +1 :green_heart: | unit | 3m 22s | | hadoop-yarn-server-common in the patch passed. |
| +1 :green_heart: | unit | 99m 6s | | hadoop-yarn-server-resourcemanager in the patch passed. |
| +1 :green_heart: | unit | 0m 46s | | hadoop-yarn-server-router in the patch passed. |
| +1 :green_heart: | asflicense | 0m 52s | | The patch does not generate ASF License warnings. |
| | | 241m 20s | | |
| Subsystem | Report/Notes |
|----------:|:-------------|
| Docker | ClientAPI=1.41 ServerAPI=1.41 base: https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/15/artifact/out/Dockerfile |
| GITHUB PR | https://github.com/apache/hadoop/pull/5131 |
| Optional Tests | dupname asflicense compile javac javadoc mvninstall mvnsite unit shadedclient spotbugs checkstyle codespell detsecrets |
| uname | Linux 59afe0640f7a 4.15.0-191-generic #202-Ubuntu SMP Thu Aug 4 01:49:29 UTC 2022 x86_64 x86_64 x86_64 GNU/Linux |
| Build tool | maven |
| Personality | dev-support/bin/hadoop.sh |
| git revision | trunk / dadfceb49edc37af0503770a442c794b8e841304 |
| Default Java | Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| Multi-JDK versions | /usr/lib/jvm/java-11-openjdk-amd64:Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 /usr/lib/jvm/java-8-openjdk-amd64:Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| Test Results | https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/15/testReport/ |
| Max. process+thread count | 944 (vs. ulimit of 5500) |
| modules | C: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router U: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server |
| Console output | https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/15/console |
| versions | git=2.25.1 maven=3.6.3 spotbugs=4.2.2 |
| Powered by | Apache Yetus 0.14.0 https://yetus.apache.org |
This message was automatically generated.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-issues-help@hadoop.apache.org
[GitHub] [hadoop] slfan1989 commented on pull request #5131: YARN-11350. [Federation] Router Support DelegationToken With ZK.
Posted by GitBox <gi...@apache.org>.
slfan1989 commented on PR #5131:
URL: https://github.com/apache/hadoop/pull/5131#issuecomment-1323873187
@goiri Can you help to review this PR again? Thank you very much!
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-issues-help@hadoop.apache.org
[GitHub] [hadoop] hadoop-yetus commented on pull request #5131: YARN-11350. [Federation] Router Support DelegationToken With ZK.
Posted by GitBox <gi...@apache.org>.
hadoop-yetus commented on PR #5131:
URL: https://github.com/apache/hadoop/pull/5131#issuecomment-1341309186
:broken_heart: **-1 overall**
| Vote | Subsystem | Runtime | Logfile | Comment |
|:----:|----------:|--------:|:--------:|:-------:|
| +0 :ok: | reexec | 0m 46s | | Docker mode activated. |
|||| _ Prechecks _ |
| +1 :green_heart: | dupname | 0m 1s | | No case conflicting files found. |
| +0 :ok: | codespell | 0m 0s | | codespell was not available. |
| +0 :ok: | detsecrets | 0m 0s | | detect-secrets was not available. |
| +1 :green_heart: | @author | 0m 0s | | The patch does not contain any @author tags. |
| +1 :green_heart: | test4tests | 0m 0s | | The patch appears to include 4 new or modified test files. |
|||| _ trunk Compile Tests _ |
| +0 :ok: | mvndep | 15m 17s | | Maven dependency ordering for branch |
| +1 :green_heart: | mvninstall | 25m 50s | | trunk passed |
| +1 :green_heart: | compile | 3m 55s | | trunk passed with JDK Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 |
| +1 :green_heart: | compile | 3m 14s | | trunk passed with JDK Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | checkstyle | 1m 13s | | trunk passed |
| +1 :green_heart: | mvnsite | 2m 20s | | trunk passed |
| +1 :green_heart: | javadoc | 2m 6s | | trunk passed with JDK Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 |
| +1 :green_heart: | javadoc | 1m 46s | | trunk passed with JDK Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | spotbugs | 4m 42s | | trunk passed |
| +1 :green_heart: | shadedclient | 21m 50s | | branch has no errors when building and testing our client artifacts. |
| -0 :warning: | patch | 22m 9s | | Used diff version of patch file. Binary files and potentially other changes not applied. Please rebase and squash commits if necessary. |
|||| _ Patch Compile Tests _ |
| +0 :ok: | mvndep | 0m 28s | | Maven dependency ordering for patch |
| +1 :green_heart: | mvninstall | 1m 52s | | the patch passed |
| +1 :green_heart: | compile | 3m 46s | | the patch passed with JDK Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 |
| +1 :green_heart: | javac | 3m 46s | | the patch passed |
| +1 :green_heart: | compile | 3m 9s | | the patch passed with JDK Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | javac | 3m 9s | | the patch passed |
| +1 :green_heart: | blanks | 0m 0s | | The patch has no blanks issues. |
| -0 :warning: | checkstyle | 1m 1s | [/results-checkstyle-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/19/artifact/out/results-checkstyle-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server.txt) | hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server: The patch generated 5 new + 10 unchanged - 0 fixed = 15 total (was 10) |
| +1 :green_heart: | mvnsite | 1m 59s | | the patch passed |
| -1 :x: | javadoc | 0m 31s | [/patch-javadoc-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-common-jdkUbuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/19/artifact/out/patch-javadoc-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-common-jdkUbuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04.txt) | hadoop-yarn-server-common in the patch failed with JDK Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04. |
| -1 :x: | javadoc | 0m 28s | [/patch-javadoc-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-common-jdkPrivateBuild-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/19/artifact/out/patch-javadoc-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-common-jdkPrivateBuild-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07.txt) | hadoop-yarn-server-common in the patch failed with JDK Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07. |
| +1 :green_heart: | spotbugs | 4m 23s | | the patch passed |
| +1 :green_heart: | shadedclient | 19m 46s | | patch has no errors when building and testing our client artifacts. |
|||| _ Other Tests _ |
| +1 :green_heart: | unit | 3m 6s | | hadoop-yarn-server-common in the patch passed. |
| +1 :green_heart: | unit | 98m 54s | | hadoop-yarn-server-resourcemanager in the patch passed. |
| +1 :green_heart: | unit | 0m 32s | | hadoop-yarn-server-router in the patch passed. |
| +1 :green_heart: | asflicense | 0m 39s | | The patch does not generate ASF License warnings. |
| | | 228m 5s | | |
| Subsystem | Report/Notes |
|----------:|:-------------|
| Docker | ClientAPI=1.41 ServerAPI=1.41 base: https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/19/artifact/out/Dockerfile |
| GITHUB PR | https://github.com/apache/hadoop/pull/5131 |
| Optional Tests | dupname asflicense compile javac javadoc mvninstall mvnsite unit shadedclient spotbugs checkstyle codespell detsecrets |
| uname | Linux 047fd7c4895c 4.15.0-200-generic #211-Ubuntu SMP Thu Nov 24 18:16:04 UTC 2022 x86_64 x86_64 x86_64 GNU/Linux |
| Build tool | maven |
| Personality | dev-support/bin/hadoop.sh |
| git revision | trunk / dd52ab76d28ec87ed3f052f9fb5342491319630b |
| Default Java | Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| Multi-JDK versions | /usr/lib/jvm/java-11-openjdk-amd64:Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 /usr/lib/jvm/java-8-openjdk-amd64:Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| Test Results | https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/19/testReport/ |
| Max. process+thread count | 982 (vs. ulimit of 5500) |
| modules | C: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router U: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server |
| Console output | https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/19/console |
| versions | git=2.25.1 maven=3.6.3 spotbugs=4.2.2 |
| Powered by | Apache Yetus 0.14.0 https://yetus.apache.org |
This message was automatically generated.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-issues-help@hadoop.apache.org
[GitHub] [hadoop] hadoop-yetus commented on pull request #5131: YARN-11350. [Federation] Router Support DelegationToken With ZK.
Posted by GitBox <gi...@apache.org>.
hadoop-yetus commented on PR #5131:
URL: https://github.com/apache/hadoop/pull/5131#issuecomment-1343029825
:broken_heart: **-1 overall**
| Vote | Subsystem | Runtime | Logfile | Comment |
|:----:|----------:|--------:|:--------:|:-------:|
| +0 :ok: | reexec | 0m 35s | | Docker mode activated. |
|||| _ Prechecks _ |
| +1 :green_heart: | dupname | 0m 0s | | No case conflicting files found. |
| +0 :ok: | codespell | 0m 0s | | codespell was not available. |
| +0 :ok: | detsecrets | 0m 0s | | detect-secrets was not available. |
| +1 :green_heart: | @author | 0m 0s | | The patch does not contain any @author tags. |
| +1 :green_heart: | test4tests | 0m 0s | | The patch appears to include 4 new or modified test files. |
|||| _ trunk Compile Tests _ |
| +0 :ok: | mvndep | 15m 15s | | Maven dependency ordering for branch |
| +1 :green_heart: | mvninstall | 25m 39s | | trunk passed |
| +1 :green_heart: | compile | 3m 55s | | trunk passed with JDK Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 |
| +1 :green_heart: | compile | 3m 16s | | trunk passed with JDK Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | checkstyle | 1m 15s | | trunk passed |
| +1 :green_heart: | mvnsite | 2m 21s | | trunk passed |
| +1 :green_heart: | javadoc | 2m 2s | | trunk passed with JDK Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 |
| +1 :green_heart: | javadoc | 1m 47s | | trunk passed with JDK Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | spotbugs | 4m 15s | | trunk passed |
| +1 :green_heart: | shadedclient | 22m 41s | | branch has no errors when building and testing our client artifacts. |
| -0 :warning: | patch | 22m 59s | | Used diff version of patch file. Binary files and potentially other changes not applied. Please rebase and squash commits if necessary. |
|||| _ Patch Compile Tests _ |
| +0 :ok: | mvndep | 0m 27s | | Maven dependency ordering for patch |
| +1 :green_heart: | mvninstall | 1m 48s | | the patch passed |
| +1 :green_heart: | compile | 3m 59s | | the patch passed with JDK Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 |
| +1 :green_heart: | javac | 3m 59s | | the patch passed |
| +1 :green_heart: | compile | 3m 17s | | the patch passed with JDK Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | javac | 3m 17s | | the patch passed |
| +1 :green_heart: | blanks | 0m 0s | | The patch has no blanks issues. |
| -0 :warning: | checkstyle | 1m 3s | [/results-checkstyle-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/20/artifact/out/results-checkstyle-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server.txt) | hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server: The patch generated 1 new + 10 unchanged - 0 fixed = 11 total (was 10) |
| +1 :green_heart: | mvnsite | 1m 59s | | the patch passed |
| -1 :x: | javadoc | 0m 27s | [/patch-javadoc-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-common-jdkUbuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/20/artifact/out/patch-javadoc-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-common-jdkUbuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04.txt) | hadoop-yarn-server-common in the patch failed with JDK Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04. |
| -1 :x: | javadoc | 0m 26s | [/patch-javadoc-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-common-jdkPrivateBuild-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/20/artifact/out/patch-javadoc-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-common-jdkPrivateBuild-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07.txt) | hadoop-yarn-server-common in the patch failed with JDK Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07. |
| +1 :green_heart: | spotbugs | 4m 15s | | the patch passed |
| +1 :green_heart: | shadedclient | 21m 45s | | patch has no errors when building and testing our client artifacts. |
|||| _ Other Tests _ |
| +1 :green_heart: | unit | 3m 43s | | hadoop-yarn-server-common in the patch passed. |
| +1 :green_heart: | unit | 98m 36s | | hadoop-yarn-server-resourcemanager in the patch passed. |
| +1 :green_heart: | unit | 0m 32s | | hadoop-yarn-server-router in the patch passed. |
| +1 :green_heart: | asflicense | 0m 37s | | The patch does not generate ASF License warnings. |
| | | 230m 2s | | |
| Subsystem | Report/Notes |
|----------:|:-------------|
| Docker | ClientAPI=1.41 ServerAPI=1.41 base: https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/20/artifact/out/Dockerfile |
| GITHUB PR | https://github.com/apache/hadoop/pull/5131 |
| Optional Tests | dupname asflicense compile javac javadoc mvninstall mvnsite unit shadedclient spotbugs checkstyle codespell detsecrets |
| uname | Linux dc143b05b61f 4.15.0-200-generic #211-Ubuntu SMP Thu Nov 24 18:16:04 UTC 2022 x86_64 x86_64 x86_64 GNU/Linux |
| Build tool | maven |
| Personality | dev-support/bin/hadoop.sh |
| git revision | trunk / 8e1cbf0c590502c2852baa511a159cd34462240c |
| Default Java | Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| Multi-JDK versions | /usr/lib/jvm/java-11-openjdk-amd64:Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 /usr/lib/jvm/java-8-openjdk-amd64:Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| Test Results | https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/20/testReport/ |
| Max. process+thread count | 917 (vs. ulimit of 5500) |
| modules | C: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router U: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server |
| Console output | https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/20/console |
| versions | git=2.25.1 maven=3.6.3 spotbugs=4.2.2 |
| Powered by | Apache Yetus 0.14.0 https://yetus.apache.org |
This message was automatically generated.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-issues-help@hadoop.apache.org
[GitHub] [hadoop] hadoop-yetus commented on pull request #5131: YARN-11350. [Federation] Router Support DelegationToken With ZK.
Posted by GitBox <gi...@apache.org>.
hadoop-yetus commented on PR #5131:
URL: https://github.com/apache/hadoop/pull/5131#issuecomment-1339648549
:broken_heart: **-1 overall**
| Vote | Subsystem | Runtime | Logfile | Comment |
|:----:|----------:|--------:|:--------:|:-------:|
| +0 :ok: | reexec | 0m 38s | | Docker mode activated. |
|||| _ Prechecks _ |
| +1 :green_heart: | dupname | 0m 0s | | No case conflicting files found. |
| +0 :ok: | codespell | 0m 1s | | codespell was not available. |
| +0 :ok: | detsecrets | 0m 1s | | detect-secrets was not available. |
| +1 :green_heart: | @author | 0m 0s | | The patch does not contain any @author tags. |
| +1 :green_heart: | test4tests | 0m 0s | | The patch appears to include 4 new or modified test files. |
|||| _ trunk Compile Tests _ |
| +0 :ok: | mvndep | 15m 15s | | Maven dependency ordering for branch |
| +1 :green_heart: | mvninstall | 26m 38s | | trunk passed |
| +1 :green_heart: | compile | 4m 13s | | trunk passed with JDK Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 |
| +1 :green_heart: | compile | 3m 36s | | trunk passed with JDK Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | checkstyle | 1m 33s | | trunk passed |
| +1 :green_heart: | mvnsite | 3m 12s | | trunk passed |
| +1 :green_heart: | javadoc | 2m 51s | | trunk passed with JDK Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 |
| +1 :green_heart: | javadoc | 2m 43s | | trunk passed with JDK Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | spotbugs | 5m 8s | | trunk passed |
| +1 :green_heart: | shadedclient | 20m 57s | | branch has no errors when building and testing our client artifacts. |
| -0 :warning: | patch | 21m 25s | | Used diff version of patch file. Binary files and potentially other changes not applied. Please rebase and squash commits if necessary. |
|||| _ Patch Compile Tests _ |
| +0 :ok: | mvndep | 0m 32s | | Maven dependency ordering for patch |
| +1 :green_heart: | mvninstall | 2m 14s | | the patch passed |
| +1 :green_heart: | compile | 3m 54s | | the patch passed with JDK Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 |
| +1 :green_heart: | javac | 3m 54s | | the patch passed |
| +1 :green_heart: | compile | 3m 23s | | the patch passed with JDK Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | javac | 3m 23s | | the patch passed |
| +1 :green_heart: | blanks | 0m 1s | | The patch has no blanks issues. |
| -0 :warning: | checkstyle | 1m 13s | [/results-checkstyle-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/18/artifact/out/results-checkstyle-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server.txt) | hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server: The patch generated 5 new + 10 unchanged - 0 fixed = 15 total (was 10) |
| +1 :green_heart: | mvnsite | 2m 29s | | the patch passed |
| -1 :x: | javadoc | 0m 40s | [/patch-javadoc-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-common-jdkUbuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/18/artifact/out/patch-javadoc-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-common-jdkUbuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04.txt) | hadoop-yarn-server-common in the patch failed with JDK Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04. |
| -1 :x: | javadoc | 0m 38s | [/patch-javadoc-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-common-jdkPrivateBuild-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/18/artifact/out/patch-javadoc-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-common-jdkPrivateBuild-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07.txt) | hadoop-yarn-server-common in the patch failed with JDK Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07. |
| +1 :green_heart: | spotbugs | 5m 29s | | the patch passed |
| +1 :green_heart: | shadedclient | 22m 19s | | patch has no errors when building and testing our client artifacts. |
|||| _ Other Tests _ |
| +1 :green_heart: | unit | 3m 17s | | hadoop-yarn-server-common in the patch passed. |
| +1 :green_heart: | unit | 99m 27s | | hadoop-yarn-server-resourcemanager in the patch passed. |
| +1 :green_heart: | unit | 0m 40s | | hadoop-yarn-server-router in the patch passed. |
| +1 :green_heart: | asflicense | 0m 45s | | The patch does not generate ASF License warnings. |
| | | 239m 20s | | |
| Subsystem | Report/Notes |
|----------:|:-------------|
| Docker | ClientAPI=1.41 ServerAPI=1.41 base: https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/18/artifact/out/Dockerfile |
| GITHUB PR | https://github.com/apache/hadoop/pull/5131 |
| Optional Tests | dupname asflicense compile javac javadoc mvninstall mvnsite unit shadedclient spotbugs checkstyle codespell detsecrets |
| uname | Linux 9903370c1867 4.15.0-191-generic #202-Ubuntu SMP Thu Aug 4 01:49:29 UTC 2022 x86_64 x86_64 x86_64 GNU/Linux |
| Build tool | maven |
| Personality | dev-support/bin/hadoop.sh |
| git revision | trunk / 0897eedbee0587feafa84fc51e7976e8db24f6e9 |
| Default Java | Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| Multi-JDK versions | /usr/lib/jvm/java-11-openjdk-amd64:Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 /usr/lib/jvm/java-8-openjdk-amd64:Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| Test Results | https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/18/testReport/ |
| Max. process+thread count | 1001 (vs. ulimit of 5500) |
| modules | C: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router U: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server |
| Console output | https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/18/console |
| versions | git=2.25.1 maven=3.6.3 spotbugs=4.2.2 |
| Powered by | Apache Yetus 0.14.0 https://yetus.apache.org |
This message was automatically generated.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-issues-help@hadoop.apache.org
[GitHub] [hadoop] hadoop-yetus commented on pull request #5131: YARN-11350. [Federation] Router Support DelegationToken With ZK.
Posted by GitBox <gi...@apache.org>.
hadoop-yetus commented on PR #5131:
URL: https://github.com/apache/hadoop/pull/5131#issuecomment-1345449868
:broken_heart: **-1 overall**
| Vote | Subsystem | Runtime | Logfile | Comment |
|:----:|----------:|--------:|:--------:|:-------:|
| +0 :ok: | reexec | 12m 25s | | Docker mode activated. |
|||| _ Prechecks _ |
| +1 :green_heart: | dupname | 0m 0s | | No case conflicting files found. |
| +0 :ok: | codespell | 0m 1s | | codespell was not available. |
| +0 :ok: | detsecrets | 0m 1s | | detect-secrets was not available. |
| +1 :green_heart: | @author | 0m 0s | | The patch does not contain any @author tags. |
| +1 :green_heart: | test4tests | 0m 0s | | The patch appears to include 4 new or modified test files. |
|||| _ trunk Compile Tests _ |
| +0 :ok: | mvndep | 15m 23s | | Maven dependency ordering for branch |
| +1 :green_heart: | mvninstall | 25m 55s | | trunk passed |
| +1 :green_heart: | compile | 3m 53s | | trunk passed with JDK Ubuntu-11.0.17+8-post-Ubuntu-1ubuntu220.04 |
| +1 :green_heart: | compile | 3m 18s | | trunk passed with JDK Private Build-1.8.0_352-8u352-ga-1~20.04-b08 |
| +1 :green_heart: | checkstyle | 1m 14s | | trunk passed |
| +1 :green_heart: | mvnsite | 2m 23s | | trunk passed |
| -1 :x: | javadoc | 0m 46s | [/branch-javadoc-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-resourcemanager-jdkUbuntu-11.0.17+8-post-Ubuntu-1ubuntu220.04.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/21/artifact/out/branch-javadoc-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-resourcemanager-jdkUbuntu-11.0.17+8-post-Ubuntu-1ubuntu220.04.txt) | hadoop-yarn-server-resourcemanager in trunk failed with JDK Ubuntu-11.0.17+8-post-Ubuntu-1ubuntu220.04. |
| +1 :green_heart: | javadoc | 1m 49s | | trunk passed with JDK Private Build-1.8.0_352-8u352-ga-1~20.04-b08 |
| +1 :green_heart: | spotbugs | 4m 15s | | trunk passed |
| +1 :green_heart: | shadedclient | 19m 54s | | branch has no errors when building and testing our client artifacts. |
| -0 :warning: | patch | 20m 14s | | Used diff version of patch file. Binary files and potentially other changes not applied. Please rebase and squash commits if necessary. |
|||| _ Patch Compile Tests _ |
| +0 :ok: | mvndep | 0m 29s | | Maven dependency ordering for patch |
| +1 :green_heart: | mvninstall | 1m 49s | | the patch passed |
| +1 :green_heart: | compile | 3m 44s | | the patch passed with JDK Ubuntu-11.0.17+8-post-Ubuntu-1ubuntu220.04 |
| +1 :green_heart: | javac | 3m 44s | | the patch passed |
| +1 :green_heart: | compile | 3m 14s | | the patch passed with JDK Private Build-1.8.0_352-8u352-ga-1~20.04-b08 |
| +1 :green_heart: | javac | 3m 14s | | the patch passed |
| +1 :green_heart: | blanks | 0m 0s | | The patch has no blanks issues. |
| -0 :warning: | checkstyle | 1m 2s | [/results-checkstyle-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/21/artifact/out/results-checkstyle-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server.txt) | hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server: The patch generated 1 new + 10 unchanged - 0 fixed = 11 total (was 10) |
| +1 :green_heart: | mvnsite | 1m 59s | | the patch passed |
| -1 :x: | javadoc | 0m 30s | [/patch-javadoc-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-common-jdkUbuntu-11.0.17+8-post-Ubuntu-1ubuntu220.04.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/21/artifact/out/patch-javadoc-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-common-jdkUbuntu-11.0.17+8-post-Ubuntu-1ubuntu220.04.txt) | hadoop-yarn-server-common in the patch failed with JDK Ubuntu-11.0.17+8-post-Ubuntu-1ubuntu220.04. |
| -1 :x: | javadoc | 0m 40s | [/patch-javadoc-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-resourcemanager-jdkUbuntu-11.0.17+8-post-Ubuntu-1ubuntu220.04.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/21/artifact/out/patch-javadoc-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-resourcemanager-jdkUbuntu-11.0.17+8-post-Ubuntu-1ubuntu220.04.txt) | hadoop-yarn-server-resourcemanager in the patch failed with JDK Ubuntu-11.0.17+8-post-Ubuntu-1ubuntu220.04. |
| -1 :x: | javadoc | 0m 29s | [/patch-javadoc-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-common-jdkPrivateBuild-1.8.0_352-8u352-ga-1~20.04-b08.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/21/artifact/out/patch-javadoc-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-common-jdkPrivateBuild-1.8.0_352-8u352-ga-1~20.04-b08.txt) | hadoop-yarn-server-common in the patch failed with JDK Private Build-1.8.0_352-8u352-ga-1~20.04-b08. |
| +1 :green_heart: | spotbugs | 4m 18s | | the patch passed |
| +1 :green_heart: | shadedclient | 20m 2s | | patch has no errors when building and testing our client artifacts. |
|||| _ Other Tests _ |
| +1 :green_heart: | unit | 3m 6s | | hadoop-yarn-server-common in the patch passed. |
| +1 :green_heart: | unit | 113m 23s | | hadoop-yarn-server-resourcemanager in the patch passed. |
| +1 :green_heart: | unit | 0m 33s | | hadoop-yarn-server-router in the patch passed. |
| +1 :green_heart: | asflicense | 0m 37s | | The patch does not generate ASF License warnings. |
| | | 252m 24s | | |
| Subsystem | Report/Notes |
|----------:|:-------------|
| Docker | ClientAPI=1.41 ServerAPI=1.41 base: https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/21/artifact/out/Dockerfile |
| GITHUB PR | https://github.com/apache/hadoop/pull/5131 |
| Optional Tests | dupname asflicense compile javac javadoc mvninstall mvnsite unit shadedclient spotbugs checkstyle codespell detsecrets |
| uname | Linux bcae0a2d993f 4.15.0-200-generic #211-Ubuntu SMP Thu Nov 24 18:16:04 UTC 2022 x86_64 x86_64 x86_64 GNU/Linux |
| Build tool | maven |
| Personality | dev-support/bin/hadoop.sh |
| git revision | trunk / 4feabd1971fbec4a69cc23c91ab59c4ad9997eaa |
| Default Java | Private Build-1.8.0_352-8u352-ga-1~20.04-b08 |
| Multi-JDK versions | /usr/lib/jvm/java-11-openjdk-amd64:Ubuntu-11.0.17+8-post-Ubuntu-1ubuntu220.04 /usr/lib/jvm/java-8-openjdk-amd64:Private Build-1.8.0_352-8u352-ga-1~20.04-b08 |
| Test Results | https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/21/testReport/ |
| Max. process+thread count | 1000 (vs. ulimit of 5500) |
| modules | C: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router U: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server |
| Console output | https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/21/console |
| versions | git=2.25.1 maven=3.6.3 spotbugs=4.2.2 |
| Powered by | Apache Yetus 0.14.0 https://yetus.apache.org |
This message was automatically generated.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-issues-help@hadoop.apache.org
[GitHub] [hadoop] slfan1989 commented on pull request #5131: YARN-11350. [Federation] Router Support DelegationToken With ZK.
Posted by GitBox <gi...@apache.org>.
slfan1989 commented on PR #5131:
URL: https://github.com/apache/hadoop/pull/5131#issuecomment-1315342828
@goiri Can you help review this pr? Thank you very much!
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-issues-help@hadoop.apache.org
[GitHub] [hadoop] slfan1989 commented on a diff in pull request #5131: YARN-11350. [Federation] Router Support DelegationToken With ZK.
Posted by GitBox <gi...@apache.org>.
slfan1989 commented on code in PR #5131:
URL: https://github.com/apache/hadoop/pull/5131#discussion_r1030951065
##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/ZookeeperFederationStateStore.java:
##########
@@ -886,45 +975,554 @@ public UpdateReservationHomeSubClusterResponse updateReservationHomeSubCluster(
return UpdateReservationHomeSubClusterResponse.newInstance();
}
+ /**
+ * ZookeeperFederationStateStore Supports Store NewMasterKey.
+ *
+ * @param request The request contains RouterMasterKey, which is an abstraction for DelegationKey
+ * @return routerMasterKeyResponse, the response contains the RouterMasterKey.
+ * @throws YarnException if the call to the state store is unsuccessful.
+ * @throws IOException An IO Error occurred.
+ */
@Override
- public RouterMasterKeyResponse storeNewMasterKey(RouterMasterKeyRequest request)
+ public synchronized RouterMasterKeyResponse storeNewMasterKey(RouterMasterKeyRequest request)
throws YarnException, IOException {
- throw new NotImplementedException("Code is not implemented");
+
+ long start = clock.getTime();
+ // For the verification of the request, after passing the verification,
+ // the request and the internal objects will not be empty and can be used directly.
+ FederationRouterRMTokenInputValidator.validate(request);
+
+ // Parse the delegationKey from the request and get the ZK storage path.
+ DelegationKey delegationKey = convertMasterKeyToDelegationKey(request);
+ String nodeCreatePath = getMasterKeyZNodePathByDelegationKey(delegationKey);
+ LOG.debug("Storing RMDelegationKey_{}, ZkNodePath = {}.", delegationKey.getKeyId(),
+ nodeCreatePath);
+
+ // Write master key data to zk.
+ try(ByteArrayOutputStream os = new ByteArrayOutputStream();
+ DataOutputStream fsOut = new DataOutputStream(os)) {
+ delegationKey.write(fsOut);
+ put(nodeCreatePath, os.toByteArray(), false);
+ }
+
+ // Get the stored masterKey from zk.
+ RouterMasterKey masterKeyFromZK = getRouterMasterKeyFromZK(nodeCreatePath);
+ long end = clock.getTime();
+ opDurations.addStoreNewMasterKeyDuration(start, end);
+ return RouterMasterKeyResponse.newInstance(masterKeyFromZK);
}
+ /**
+ * ZookeeperFederationStateStore Supports Remove MasterKey.
+ *
+ * @param request The request contains RouterMasterKey, which is an abstraction for DelegationKey
+ * @return routerMasterKeyResponse, the response contains the RouterMasterKey.
+ * @throws YarnException if the call to the state store is unsuccessful.
+ * @throws IOException An IO Error occurred.
+ */
@Override
- public RouterMasterKeyResponse removeStoredMasterKey(RouterMasterKeyRequest request)
+ public synchronized RouterMasterKeyResponse removeStoredMasterKey(RouterMasterKeyRequest request)
throws YarnException, IOException {
- throw new NotImplementedException("Code is not implemented");
+
+ long start = clock.getTime();
+ // For the verification of the request, after passing the verification,
+ // the request and the internal objects will not be empty and can be used directly.
+ FederationRouterRMTokenInputValidator.validate(request);
+
+ try {
+ // Parse the delegationKey from the request and get the ZK storage path.
+ RouterMasterKey masterKey = request.getRouterMasterKey();
+ DelegationKey delegationKey = convertMasterKeyToDelegationKey(request);
+ String nodeRemovePath = getMasterKeyZNodePathByDelegationKey(delegationKey);
+ LOG.debug("Removing RMDelegationKey_{}, ZkNodePath = {}.", delegationKey.getKeyId(),
+ nodeRemovePath);
+
+ // Check if the path exists, Throws an exception if the path does not exist.
+ if (!exists(nodeRemovePath)) {
+ throw new YarnException("ZkNodePath = " + nodeRemovePath + " not exists!");
+ }
+
+ // try to remove masterKey.
+ zkManager.delete(nodeRemovePath);
+ long end = clock.getTime();
+ opDurations.removeStoredMasterKeyDuration(start, end);
+ return RouterMasterKeyResponse.newInstance(masterKey);
+ } catch (Exception e) {
+ throw new YarnException(e);
+ }
}
+ /**
+ * ZookeeperFederationStateStore Supports Remove MasterKey.
+ *
+ * @param request The request contains RouterMasterKey, which is an abstraction for DelegationKey
+ * @return routerMasterKeyResponse, the response contains the RouterMasterKey.
+ * @throws YarnException if the call to the state store is unsuccessful.
+ * @throws IOException An IO Error occurred.
+ */
@Override
public RouterMasterKeyResponse getMasterKeyByDelegationKey(RouterMasterKeyRequest request)
throws YarnException, IOException {
- throw new NotImplementedException("Code is not implemented");
+
+ long start = clock.getTime();
+ // For the verification of the request, after passing the verification,
+ // the request and the internal objects will not be empty and can be used directly.
+ FederationRouterRMTokenInputValidator.validate(request);
+
+ try {
+
+ // Parse the delegationKey from the request and get the ZK storage path.
+ DelegationKey delegationKey = convertMasterKeyToDelegationKey(request);
+ String nodePath = getMasterKeyZNodePathByDelegationKey(delegationKey);
+
+ // Check if the path exists, Throws an exception if the path does not exist.
+ if (!exists(nodePath)) {
+ throw new YarnException("ZkNodePath = " + nodePath + " not exists!");
+ }
+
+ // Get the stored masterKey from zk.
+ RouterMasterKey routerMasterKey = getRouterMasterKeyFromZK(nodePath);
+ long end = clock.getTime();
+ opDurations.getMasterKeyByDelegationKeyDuration(start, end);
+ return RouterMasterKeyResponse.newInstance(routerMasterKey);
+ } catch (Exception e) {
+ throw new YarnException(e);
+ }
+ }
+
+ /**
+ * Get MasterKeyZNodePath based on DelegationKey.
+ *
+ * @param delegationKey delegationKey.
+ * @return masterKey ZNodePath.
+ */
+ private String getMasterKeyZNodePathByDelegationKey(DelegationKey delegationKey) {
+ return getMasterKeyZNodePathByKeyId(delegationKey.getKeyId());
+ }
+
+ /**
+ * Get MasterKeyZNodePath based on KeyId.
+ *
+ * @param keyId master key id.
+ * @return masterKey ZNodePath.
+ */
+ private String getMasterKeyZNodePathByKeyId(int keyId) {
+ String nodeName = ROUTER_RM_DELEGATION_KEY_PREFIX + keyId;
+ return getNodePath(routerRMDTMasterKeysRootPath, nodeName);
+ }
+
+ /**
+ * Get RouterMasterKey from ZK.
+ *
+ * @param nodePath The path where masterKey is stored in zk.
+ *
+ * @return RouterMasterKey.
+ * @throws IOException An IO Error occurred.
+ */
+ private RouterMasterKey getRouterMasterKeyFromZK(String nodePath)
+ throws IOException {
+ try {
+ byte[] data = get(nodePath);
+ if ((data == null) || (data.length == 0)) {
+ return null;
+ }
+
+ ByteArrayInputStream bin = new ByteArrayInputStream(data);
+ DataInputStream din = new DataInputStream(bin);
+ DelegationKey key = new DelegationKey();
+ key.readFields(din);
+
+ return RouterMasterKey.newInstance(key.getKeyId(),
+ ByteBuffer.wrap(key.getEncodedKey()), key.getExpiryDate());
+ } catch (Exception ex) {
+ LOG.error("No node in path {}.", nodePath);
+ throw new IOException(ex);
+ }
}
+ /**
+ * ZookeeperFederationStateStore Supports Store RMDelegationTokenIdentifier.
+ *
+ * The stored token method is a synchronized method
+ * used to ensure that storeNewToken is a thread-safe method.
+ *
+ * @param request The request contains RouterRMToken (RMDelegationTokenIdentifier and renewDate)
+ * @return routerRMTokenResponse, the response contains the RouterStoreToken.
+ * @throws YarnException if the call to the state store is unsuccessful.
+ * @throws IOException An IO Error occurred.
+ */
@Override
- public RouterRMTokenResponse storeNewToken(RouterRMTokenRequest request)
+ public synchronized RouterRMTokenResponse storeNewToken(RouterRMTokenRequest request)
Review Comment:
Thanks for your suggestion, I will refactor this part of the code using `lock`.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-issues-help@hadoop.apache.org
[GitHub] [hadoop] goiri commented on a diff in pull request #5131: YARN-11350. [Federation] Router Support DelegationToken With ZK.
Posted by GitBox <gi...@apache.org>.
goiri commented on code in PR #5131:
URL: https://github.com/apache/hadoop/pull/5131#discussion_r1029923406
##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/FederationStateStoreBaseTest.java:
##########
@@ -905,7 +905,7 @@ public void testGetMasterKeyByDelegationKey() throws YarnException, IOException
}
@Test
- public void testRemoveStoredMasterKey() throws YarnException, IOException {
+ public void testRemoveStoredMasterKey() throws IOException, YarnException {
Review Comment:
Avoid?
##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/ZookeeperFederationStateStore.java:
##########
@@ -886,45 +975,554 @@ public UpdateReservationHomeSubClusterResponse updateReservationHomeSubCluster(
return UpdateReservationHomeSubClusterResponse.newInstance();
}
+ /**
+ * ZookeeperFederationStateStore Supports Store NewMasterKey.
+ *
+ * @param request The request contains RouterMasterKey, which is an abstraction for DelegationKey
+ * @return routerMasterKeyResponse, the response contains the RouterMasterKey.
+ * @throws YarnException if the call to the state store is unsuccessful.
+ * @throws IOException An IO Error occurred.
+ */
@Override
- public RouterMasterKeyResponse storeNewMasterKey(RouterMasterKeyRequest request)
+ public synchronized RouterMasterKeyResponse storeNewMasterKey(RouterMasterKeyRequest request)
throws YarnException, IOException {
- throw new NotImplementedException("Code is not implemented");
+
+ long start = clock.getTime();
+ // For the verification of the request, after passing the verification,
+ // the request and the internal objects will not be empty and can be used directly.
+ FederationRouterRMTokenInputValidator.validate(request);
+
+ // Parse the delegationKey from the request and get the ZK storage path.
+ DelegationKey delegationKey = convertMasterKeyToDelegationKey(request);
+ String nodeCreatePath = getMasterKeyZNodePathByDelegationKey(delegationKey);
+ LOG.debug("Storing RMDelegationKey_{}, ZkNodePath = {}.", delegationKey.getKeyId(),
+ nodeCreatePath);
+
+ // Write master key data to zk.
+ try(ByteArrayOutputStream os = new ByteArrayOutputStream();
+ DataOutputStream fsOut = new DataOutputStream(os)) {
+ delegationKey.write(fsOut);
+ put(nodeCreatePath, os.toByteArray(), false);
+ }
+
+ // Get the stored masterKey from zk.
+ RouterMasterKey masterKeyFromZK = getRouterMasterKeyFromZK(nodeCreatePath);
+ long end = clock.getTime();
+ opDurations.addStoreNewMasterKeyDuration(start, end);
+ return RouterMasterKeyResponse.newInstance(masterKeyFromZK);
}
+ /**
+ * ZookeeperFederationStateStore Supports Remove MasterKey.
+ *
+ * @param request The request contains RouterMasterKey, which is an abstraction for DelegationKey
+ * @return routerMasterKeyResponse, the response contains the RouterMasterKey.
+ * @throws YarnException if the call to the state store is unsuccessful.
+ * @throws IOException An IO Error occurred.
+ */
@Override
- public RouterMasterKeyResponse removeStoredMasterKey(RouterMasterKeyRequest request)
+ public synchronized RouterMasterKeyResponse removeStoredMasterKey(RouterMasterKeyRequest request)
throws YarnException, IOException {
- throw new NotImplementedException("Code is not implemented");
+
+ long start = clock.getTime();
+ // For the verification of the request, after passing the verification,
+ // the request and the internal objects will not be empty and can be used directly.
+ FederationRouterRMTokenInputValidator.validate(request);
+
+ try {
+ // Parse the delegationKey from the request and get the ZK storage path.
+ RouterMasterKey masterKey = request.getRouterMasterKey();
+ DelegationKey delegationKey = convertMasterKeyToDelegationKey(request);
+ String nodeRemovePath = getMasterKeyZNodePathByDelegationKey(delegationKey);
+ LOG.debug("Removing RMDelegationKey_{}, ZkNodePath = {}.", delegationKey.getKeyId(),
+ nodeRemovePath);
+
+ // Check if the path exists, Throws an exception if the path does not exist.
+ if (!exists(nodeRemovePath)) {
+ throw new YarnException("ZkNodePath = " + nodeRemovePath + " not exists!");
+ }
+
+ // try to remove masterKey.
+ zkManager.delete(nodeRemovePath);
+ long end = clock.getTime();
+ opDurations.removeStoredMasterKeyDuration(start, end);
+ return RouterMasterKeyResponse.newInstance(masterKey);
+ } catch (Exception e) {
+ throw new YarnException(e);
+ }
}
+ /**
+ * ZookeeperFederationStateStore Supports Remove MasterKey.
+ *
+ * @param request The request contains RouterMasterKey, which is an abstraction for DelegationKey
+ * @return routerMasterKeyResponse, the response contains the RouterMasterKey.
+ * @throws YarnException if the call to the state store is unsuccessful.
+ * @throws IOException An IO Error occurred.
+ */
@Override
public RouterMasterKeyResponse getMasterKeyByDelegationKey(RouterMasterKeyRequest request)
throws YarnException, IOException {
- throw new NotImplementedException("Code is not implemented");
+
+ long start = clock.getTime();
+ // For the verification of the request, after passing the verification,
+ // the request and the internal objects will not be empty and can be used directly.
+ FederationRouterRMTokenInputValidator.validate(request);
+
+ try {
+
+ // Parse the delegationKey from the request and get the ZK storage path.
+ DelegationKey delegationKey = convertMasterKeyToDelegationKey(request);
+ String nodePath = getMasterKeyZNodePathByDelegationKey(delegationKey);
+
+ // Check if the path exists, Throws an exception if the path does not exist.
+ if (!exists(nodePath)) {
+ throw new YarnException("ZkNodePath = " + nodePath + " not exists!");
+ }
+
+ // Get the stored masterKey from zk.
+ RouterMasterKey routerMasterKey = getRouterMasterKeyFromZK(nodePath);
+ long end = clock.getTime();
+ opDurations.getMasterKeyByDelegationKeyDuration(start, end);
+ return RouterMasterKeyResponse.newInstance(routerMasterKey);
+ } catch (Exception e) {
+ throw new YarnException(e);
+ }
+ }
+
+ /**
+ * Get MasterKeyZNodePath based on DelegationKey.
+ *
+ * @param delegationKey delegationKey.
+ * @return masterKey ZNodePath.
+ */
+ private String getMasterKeyZNodePathByDelegationKey(DelegationKey delegationKey) {
+ return getMasterKeyZNodePathByKeyId(delegationKey.getKeyId());
+ }
+
+ /**
+ * Get MasterKeyZNodePath based on KeyId.
+ *
+ * @param keyId master key id.
+ * @return masterKey ZNodePath.
+ */
+ private String getMasterKeyZNodePathByKeyId(int keyId) {
+ String nodeName = ROUTER_RM_DELEGATION_KEY_PREFIX + keyId;
+ return getNodePath(routerRMDTMasterKeysRootPath, nodeName);
+ }
+
+ /**
+ * Get RouterMasterKey from ZK.
+ *
+ * @param nodePath The path where masterKey is stored in zk.
+ *
+ * @return RouterMasterKey.
+ * @throws IOException An IO Error occurred.
+ */
+ private RouterMasterKey getRouterMasterKeyFromZK(String nodePath)
+ throws IOException {
+ try {
+ byte[] data = get(nodePath);
+ if ((data == null) || (data.length == 0)) {
+ return null;
+ }
+
+ ByteArrayInputStream bin = new ByteArrayInputStream(data);
+ DataInputStream din = new DataInputStream(bin);
+ DelegationKey key = new DelegationKey();
+ key.readFields(din);
+
+ return RouterMasterKey.newInstance(key.getKeyId(),
+ ByteBuffer.wrap(key.getEncodedKey()), key.getExpiryDate());
+ } catch (Exception ex) {
+ LOG.error("No node in path {}.", nodePath);
+ throw new IOException(ex);
+ }
}
+ /**
+ * ZookeeperFederationStateStore Supports Store RMDelegationTokenIdentifier.
+ *
+ * The stored token method is a synchronized method
+ * used to ensure that storeNewToken is a thread-safe method.
+ *
+ * @param request The request contains RouterRMToken (RMDelegationTokenIdentifier and renewDate)
+ * @return routerRMTokenResponse, the response contains the RouterStoreToken.
+ * @throws YarnException if the call to the state store is unsuccessful.
+ * @throws IOException An IO Error occurred.
+ */
@Override
- public RouterRMTokenResponse storeNewToken(RouterRMTokenRequest request)
+ public synchronized RouterRMTokenResponse storeNewToken(RouterRMTokenRequest request)
Review Comment:
What are we protecting with the synchronized?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-issues-help@hadoop.apache.org
[GitHub] [hadoop] hadoop-yetus commented on pull request #5131: YARN-11350. [Federation] Router Support DelegationToken With ZK.
Posted by GitBox <gi...@apache.org>.
hadoop-yetus commented on PR #5131:
URL: https://github.com/apache/hadoop/pull/5131#issuecomment-1314769675
:confetti_ball: **+1 overall**
| Vote | Subsystem | Runtime | Logfile | Comment |
|:----:|----------:|--------:|:--------:|:-------:|
| +0 :ok: | reexec | 0m 36s | | Docker mode activated. |
|||| _ Prechecks _ |
| +1 :green_heart: | dupname | 0m 0s | | No case conflicting files found. |
| +0 :ok: | codespell | 0m 1s | | codespell was not available. |
| +0 :ok: | detsecrets | 0m 1s | | detect-secrets was not available. |
| +1 :green_heart: | @author | 0m 0s | | The patch does not contain any @author tags. |
| +1 :green_heart: | test4tests | 0m 0s | | The patch appears to include 3 new or modified test files. |
|||| _ trunk Compile Tests _ |
| +0 :ok: | mvndep | 16m 12s | | Maven dependency ordering for branch |
| +1 :green_heart: | mvninstall | 25m 59s | | trunk passed |
| +1 :green_heart: | compile | 4m 1s | | trunk passed with JDK Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 |
| +1 :green_heart: | compile | 3m 28s | | trunk passed with JDK Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | checkstyle | 1m 21s | | trunk passed |
| +1 :green_heart: | mvnsite | 2m 45s | | trunk passed |
| +1 :green_heart: | javadoc | 2m 23s | | trunk passed with JDK Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 |
| +1 :green_heart: | javadoc | 2m 10s | | trunk passed with JDK Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | spotbugs | 4m 41s | | trunk passed |
| +1 :green_heart: | shadedclient | 20m 27s | | branch has no errors when building and testing our client artifacts. |
|||| _ Patch Compile Tests _ |
| +0 :ok: | mvndep | 0m 28s | | Maven dependency ordering for patch |
| +1 :green_heart: | mvninstall | 2m 5s | | the patch passed |
| +1 :green_heart: | compile | 3m 49s | | the patch passed with JDK Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 |
| +1 :green_heart: | javac | 3m 49s | | the patch passed |
| +1 :green_heart: | compile | 3m 17s | | the patch passed with JDK Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | javac | 3m 17s | | the patch passed |
| +1 :green_heart: | blanks | 0m 0s | | The patch has no blanks issues. |
| +1 :green_heart: | checkstyle | 1m 13s | | the patch passed |
| +1 :green_heart: | mvnsite | 2m 20s | | the patch passed |
| +1 :green_heart: | javadoc | 0m 39s | | hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-common-jdkUbuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 with JDK Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 generated 0 new + 47 unchanged - 1 fixed = 47 total (was 48) |
| +1 :green_heart: | javadoc | 0m 45s | | hadoop-yarn-server-resourcemanager in the patch passed with JDK Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04. |
| +1 :green_heart: | javadoc | 0m 33s | | hadoop-yarn-server-router in the patch passed with JDK Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04. |
| +1 :green_heart: | javadoc | 0m 32s | | hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-common-jdkPrivateBuild-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 with JDK Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 generated 0 new + 47 unchanged - 1 fixed = 47 total (was 48) |
| +1 :green_heart: | javadoc | 0m 43s | | hadoop-yarn-server-resourcemanager in the patch passed with JDK Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07. |
| +1 :green_heart: | javadoc | 0m 26s | | hadoop-yarn-server-router in the patch passed with JDK Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07. |
| +1 :green_heart: | spotbugs | 4m 33s | | the patch passed |
| +1 :green_heart: | shadedclient | 20m 49s | | patch has no errors when building and testing our client artifacts. |
|||| _ Other Tests _ |
| +1 :green_heart: | unit | 3m 17s | | hadoop-yarn-server-common in the patch passed. |
| +1 :green_heart: | unit | 99m 5s | | hadoop-yarn-server-resourcemanager in the patch passed. |
| +1 :green_heart: | unit | 5m 52s | | hadoop-yarn-server-router in the patch passed. |
| +1 :green_heart: | asflicense | 0m 48s | | The patch does not generate ASF License warnings. |
| | | 238m 6s | | |
| Subsystem | Report/Notes |
|----------:|:-------------|
| Docker | ClientAPI=1.41 ServerAPI=1.41 base: https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/3/artifact/out/Dockerfile |
| GITHUB PR | https://github.com/apache/hadoop/pull/5131 |
| Optional Tests | dupname asflicense compile javac javadoc mvninstall mvnsite unit shadedclient spotbugs checkstyle codespell detsecrets |
| uname | Linux 27659b959354 4.15.0-191-generic #202-Ubuntu SMP Thu Aug 4 01:49:29 UTC 2022 x86_64 x86_64 x86_64 GNU/Linux |
| Build tool | maven |
| Personality | dev-support/bin/hadoop.sh |
| git revision | trunk / 3b2b62f7f099f551a1f721547ae651a07bcc448f |
| Default Java | Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| Multi-JDK versions | /usr/lib/jvm/java-11-openjdk-amd64:Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 /usr/lib/jvm/java-8-openjdk-amd64:Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| Test Results | https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/3/testReport/ |
| Max. process+thread count | 959 (vs. ulimit of 5500) |
| modules | C: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router U: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server |
| Console output | https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/3/console |
| versions | git=2.25.1 maven=3.6.3 spotbugs=4.2.2 |
| Powered by | Apache Yetus 0.14.0 https://yetus.apache.org |
This message was automatically generated.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-issues-help@hadoop.apache.org
[GitHub] [hadoop] hadoop-yetus commented on pull request #5131: YARN-11350. [Federation] Router Support DelegationToken With ZK.
Posted by GitBox <gi...@apache.org>.
hadoop-yetus commented on PR #5131:
URL: https://github.com/apache/hadoop/pull/5131#issuecomment-1314040098
:broken_heart: **-1 overall**
| Vote | Subsystem | Runtime | Logfile | Comment |
|:----:|----------:|--------:|:--------:|:-------:|
| +0 :ok: | reexec | 0m 39s | | Docker mode activated. |
|||| _ Prechecks _ |
| +1 :green_heart: | dupname | 0m 0s | | No case conflicting files found. |
| +0 :ok: | codespell | 0m 0s | | codespell was not available. |
| +0 :ok: | detsecrets | 0m 0s | | detect-secrets was not available. |
| +1 :green_heart: | @author | 0m 0s | | The patch does not contain any @author tags. |
| +1 :green_heart: | test4tests | 0m 0s | | The patch appears to include 3 new or modified test files. |
|||| _ trunk Compile Tests _ |
| +0 :ok: | mvndep | 15m 15s | | Maven dependency ordering for branch |
| +1 :green_heart: | mvninstall | 26m 18s | | trunk passed |
| +1 :green_heart: | compile | 4m 5s | | trunk passed with JDK Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 |
| +1 :green_heart: | compile | 3m 34s | | trunk passed with JDK Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | checkstyle | 1m 24s | | trunk passed |
| +1 :green_heart: | mvnsite | 2m 55s | | trunk passed |
| +1 :green_heart: | javadoc | 2m 25s | | trunk passed with JDK Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 |
| +1 :green_heart: | javadoc | 2m 12s | | trunk passed with JDK Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | spotbugs | 4m 56s | | trunk passed |
| +1 :green_heart: | shadedclient | 21m 5s | | branch has no errors when building and testing our client artifacts. |
|||| _ Patch Compile Tests _ |
| +0 :ok: | mvndep | 0m 35s | | Maven dependency ordering for patch |
| +1 :green_heart: | mvninstall | 2m 10s | | the patch passed |
| +1 :green_heart: | compile | 3m 55s | | the patch passed with JDK Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 |
| +1 :green_heart: | javac | 3m 55s | | the patch passed |
| +1 :green_heart: | compile | 3m 20s | | the patch passed with JDK Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | javac | 3m 20s | | the patch passed |
| +1 :green_heart: | blanks | 0m 0s | | The patch has no blanks issues. |
| +1 :green_heart: | checkstyle | 1m 12s | | the patch passed |
| +1 :green_heart: | mvnsite | 2m 17s | | the patch passed |
| -1 :x: | javadoc | 0m 40s | [/patch-javadoc-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-common-jdkUbuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/2/artifact/out/patch-javadoc-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-common-jdkUbuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04.txt) | hadoop-yarn-server-common in the patch failed with JDK Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04. |
| -1 :x: | javadoc | 0m 32s | [/patch-javadoc-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-common-jdkPrivateBuild-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/2/artifact/out/patch-javadoc-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-common-jdkPrivateBuild-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07.txt) | hadoop-yarn-server-common in the patch failed with JDK Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07. |
| +1 :green_heart: | spotbugs | 4m 39s | | the patch passed |
| +1 :green_heart: | shadedclient | 23m 40s | | patch has no errors when building and testing our client artifacts. |
|||| _ Other Tests _ |
| +1 :green_heart: | unit | 3m 10s | | hadoop-yarn-server-common in the patch passed. |
| +1 :green_heart: | unit | 100m 17s | | hadoop-yarn-server-resourcemanager in the patch passed. |
| +1 :green_heart: | unit | 5m 55s | | hadoop-yarn-server-router in the patch passed. |
| +1 :green_heart: | asflicense | 0m 49s | | The patch does not generate ASF License warnings. |
| | | 243m 0s | | |
| Subsystem | Report/Notes |
|----------:|:-------------|
| Docker | ClientAPI=1.41 ServerAPI=1.41 base: https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/2/artifact/out/Dockerfile |
| GITHUB PR | https://github.com/apache/hadoop/pull/5131 |
| Optional Tests | dupname asflicense compile javac javadoc mvninstall mvnsite unit shadedclient spotbugs checkstyle codespell detsecrets |
| uname | Linux 8bf26826ac56 4.15.0-191-generic #202-Ubuntu SMP Thu Aug 4 01:49:29 UTC 2022 x86_64 x86_64 x86_64 GNU/Linux |
| Build tool | maven |
| Personality | dev-support/bin/hadoop.sh |
| git revision | trunk / 14ce609e4cf0dd6cb6d8d29886b5c0fb9284d9ed |
| Default Java | Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| Multi-JDK versions | /usr/lib/jvm/java-11-openjdk-amd64:Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 /usr/lib/jvm/java-8-openjdk-amd64:Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| Test Results | https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/2/testReport/ |
| Max. process+thread count | 963 (vs. ulimit of 5500) |
| modules | C: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router U: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server |
| Console output | https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/2/console |
| versions | git=2.25.1 maven=3.6.3 spotbugs=4.2.2 |
| Powered by | Apache Yetus 0.14.0 https://yetus.apache.org |
This message was automatically generated.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-issues-help@hadoop.apache.org
[GitHub] [hadoop] hadoop-yetus commented on pull request #5131: YARN-11350. [Federation] Router Support DelegationToken With ZK.
Posted by GitBox <gi...@apache.org>.
hadoop-yetus commented on PR #5131:
URL: https://github.com/apache/hadoop/pull/5131#issuecomment-1324581269
:broken_heart: **-1 overall**
| Vote | Subsystem | Runtime | Logfile | Comment |
|:----:|----------:|--------:|:--------:|:-------:|
| +0 :ok: | reexec | 0m 48s | | Docker mode activated. |
|||| _ Prechecks _ |
| +1 :green_heart: | dupname | 0m 0s | | No case conflicting files found. |
| +0 :ok: | codespell | 0m 1s | | codespell was not available. |
| +0 :ok: | detsecrets | 0m 1s | | detect-secrets was not available. |
| +1 :green_heart: | @author | 0m 0s | | The patch does not contain any @author tags. |
| +1 :green_heart: | test4tests | 0m 0s | | The patch appears to include 2 new or modified test files. |
|||| _ trunk Compile Tests _ |
| +0 :ok: | mvndep | 16m 14s | | Maven dependency ordering for branch |
| +1 :green_heart: | mvninstall | 25m 55s | | trunk passed |
| +1 :green_heart: | compile | 4m 11s | | trunk passed with JDK Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 |
| +1 :green_heart: | compile | 3m 27s | | trunk passed with JDK Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | checkstyle | 1m 25s | | trunk passed |
| +1 :green_heart: | mvnsite | 2m 56s | | trunk passed |
| +1 :green_heart: | javadoc | 2m 34s | | trunk passed with JDK Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 |
| +1 :green_heart: | javadoc | 2m 22s | | trunk passed with JDK Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | spotbugs | 5m 5s | | trunk passed |
| +1 :green_heart: | shadedclient | 20m 57s | | branch has no errors when building and testing our client artifacts. |
| -0 :warning: | patch | 21m 24s | | Used diff version of patch file. Binary files and potentially other changes not applied. Please rebase and squash commits if necessary. |
|||| _ Patch Compile Tests _ |
| +0 :ok: | mvndep | 0m 31s | | Maven dependency ordering for patch |
| +1 :green_heart: | mvninstall | 2m 12s | | the patch passed |
| +1 :green_heart: | compile | 3m 50s | | the patch passed with JDK Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 |
| +1 :green_heart: | javac | 3m 50s | | the patch passed |
| +1 :green_heart: | compile | 3m 13s | | the patch passed with JDK Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | javac | 3m 14s | | the patch passed |
| -1 :x: | blanks | 0m 0s | [/blanks-eol.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/9/artifact/out/blanks-eol.txt) | The patch has 1 line(s) that end in blanks. Use git apply --whitespace=fix <<patch_file>>. Refer https://git-scm.com/docs/git-apply |
| +1 :green_heart: | checkstyle | 1m 6s | | the patch passed |
| +1 :green_heart: | mvnsite | 2m 13s | | the patch passed |
| +1 :green_heart: | javadoc | 1m 53s | | the patch passed with JDK Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 |
| +1 :green_heart: | javadoc | 1m 45s | | the patch passed with JDK Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | spotbugs | 4m 37s | | the patch passed |
| +1 :green_heart: | shadedclient | 20m 17s | | patch has no errors when building and testing our client artifacts. |
|||| _ Other Tests _ |
| +1 :green_heart: | unit | 3m 13s | | hadoop-yarn-server-common in the patch passed. |
| +1 :green_heart: | unit | 99m 41s | | hadoop-yarn-server-resourcemanager in the patch passed. |
| +1 :green_heart: | unit | 0m 37s | | hadoop-yarn-server-router in the patch passed. |
| +1 :green_heart: | asflicense | 0m 46s | | The patch does not generate ASF License warnings. |
| | | 234m 38s | | |
| Subsystem | Report/Notes |
|----------:|:-------------|
| Docker | ClientAPI=1.41 ServerAPI=1.41 base: https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/9/artifact/out/Dockerfile |
| GITHUB PR | https://github.com/apache/hadoop/pull/5131 |
| Optional Tests | dupname asflicense compile javac javadoc mvninstall mvnsite unit shadedclient spotbugs checkstyle codespell detsecrets |
| uname | Linux 1a8ee6cd0f85 4.15.0-191-generic #202-Ubuntu SMP Thu Aug 4 01:49:29 UTC 2022 x86_64 x86_64 x86_64 GNU/Linux |
| Build tool | maven |
| Personality | dev-support/bin/hadoop.sh |
| git revision | trunk / 3687ef59c27d76d0515df453d7708ea4ea9a180a |
| Default Java | Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| Multi-JDK versions | /usr/lib/jvm/java-11-openjdk-amd64:Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 /usr/lib/jvm/java-8-openjdk-amd64:Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| Test Results | https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/9/testReport/ |
| Max. process+thread count | 941 (vs. ulimit of 5500) |
| modules | C: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router U: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server |
| Console output | https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/9/console |
| versions | git=2.25.1 maven=3.6.3 spotbugs=4.2.2 |
| Powered by | Apache Yetus 0.14.0 https://yetus.apache.org |
This message was automatically generated.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-issues-help@hadoop.apache.org
[GitHub] [hadoop] slfan1989 commented on a diff in pull request #5131: YARN-11350. [Federation] Router Support DelegationToken With ZK.
Posted by GitBox <gi...@apache.org>.
slfan1989 commented on code in PR #5131:
URL: https://github.com/apache/hadoop/pull/5131#discussion_r1041589119
##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/TestSQLFederationStateStore.java:
##########
@@ -592,4 +588,14 @@ public void testRemoveStoredToken() throws IOException, YarnException {
public void testGetTokenByRouterStoreToken() throws IOException, YarnException {
super.testGetTokenByRouterStoreToken();
}
+
+ @Override
+ protected void checkRouterMasterKey(DelegationKey delegationKey,
+ RouterMasterKey routerMasterKey) throws YarnException, IOException {
Review Comment:
Thank you for your suggestion! I'll add a comment explaining what the method does.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-issues-help@hadoop.apache.org
[GitHub] [hadoop] slfan1989 commented on pull request #5131: YARN-11350. [Federation] Router Support DelegationToken With ZK.
Posted by GitBox <gi...@apache.org>.
slfan1989 commented on PR #5131:
URL: https://github.com/apache/hadoop/pull/5131#issuecomment-1320064507
@goiri Can you help review this PR again? Thank you very much!
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-issues-help@hadoop.apache.org
[GitHub] [hadoop] hadoop-yetus commented on pull request #5131: YARN-11350. [Federation] Router Support DelegationToken With ZK.
Posted by GitBox <gi...@apache.org>.
hadoop-yetus commented on PR #5131:
URL: https://github.com/apache/hadoop/pull/5131#issuecomment-1320241006
:confetti_ball: **+1 overall**
| Vote | Subsystem | Runtime | Logfile | Comment |
|:----:|----------:|--------:|:--------:|:-------:|
| +0 :ok: | reexec | 0m 37s | | Docker mode activated. |
|||| _ Prechecks _ |
| +1 :green_heart: | dupname | 0m 1s | | No case conflicting files found. |
| +0 :ok: | codespell | 0m 0s | | codespell was not available. |
| +0 :ok: | detsecrets | 0m 0s | | detect-secrets was not available. |
| +1 :green_heart: | @author | 0m 0s | | The patch does not contain any @author tags. |
| +1 :green_heart: | test4tests | 0m 0s | | The patch appears to include 3 new or modified test files. |
|||| _ trunk Compile Tests _ |
| +0 :ok: | mvndep | 15m 22s | | Maven dependency ordering for branch |
| +1 :green_heart: | mvninstall | 25m 39s | | trunk passed |
| +1 :green_heart: | compile | 4m 3s | | trunk passed with JDK Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 |
| +1 :green_heart: | compile | 3m 26s | | trunk passed with JDK Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | checkstyle | 1m 24s | | trunk passed |
| +1 :green_heart: | mvnsite | 2m 54s | | trunk passed |
| +1 :green_heart: | javadoc | 2m 26s | | trunk passed with JDK Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 |
| +1 :green_heart: | javadoc | 2m 18s | | trunk passed with JDK Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | spotbugs | 4m 51s | | trunk passed |
| +1 :green_heart: | shadedclient | 20m 41s | | branch has no errors when building and testing our client artifacts. |
|||| _ Patch Compile Tests _ |
| +0 :ok: | mvndep | 0m 29s | | Maven dependency ordering for patch |
| +1 :green_heart: | mvninstall | 2m 1s | | the patch passed |
| +1 :green_heart: | compile | 3m 53s | | the patch passed with JDK Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 |
| +1 :green_heart: | javac | 3m 53s | | the patch passed |
| +1 :green_heart: | compile | 3m 15s | | the patch passed with JDK Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | javac | 3m 15s | | the patch passed |
| +1 :green_heart: | blanks | 0m 0s | | The patch has no blanks issues. |
| +1 :green_heart: | checkstyle | 1m 12s | | the patch passed |
| +1 :green_heart: | mvnsite | 2m 14s | | the patch passed |
| +1 :green_heart: | javadoc | 0m 35s | | hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-common-jdkUbuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 with JDK Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 generated 0 new + 47 unchanged - 1 fixed = 47 total (was 48) |
| +1 :green_heart: | javadoc | 0m 48s | | hadoop-yarn-server-resourcemanager in the patch passed with JDK Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04. |
| +1 :green_heart: | javadoc | 0m 27s | | hadoop-yarn-server-router in the patch passed with JDK Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04. |
| +1 :green_heart: | javadoc | 0m 31s | | hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-common-jdkPrivateBuild-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 with JDK Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 generated 0 new + 47 unchanged - 1 fixed = 47 total (was 48) |
| +1 :green_heart: | javadoc | 0m 44s | | hadoop-yarn-server-resourcemanager in the patch passed with JDK Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07. |
| +1 :green_heart: | javadoc | 0m 26s | | hadoop-yarn-server-router in the patch passed with JDK Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07. |
| +1 :green_heart: | spotbugs | 4m 34s | | the patch passed |
| +1 :green_heart: | shadedclient | 20m 20s | | patch has no errors when building and testing our client artifacts. |
|||| _ Other Tests _ |
| +1 :green_heart: | unit | 3m 10s | | hadoop-yarn-server-common in the patch passed. |
| +1 :green_heart: | unit | 114m 29s | | hadoop-yarn-server-resourcemanager in the patch passed. |
| +1 :green_heart: | unit | 5m 58s | | hadoop-yarn-server-router in the patch passed. |
| +1 :green_heart: | asflicense | 0m 44s | | The patch does not generate ASF License warnings. |
| | | 252m 30s | | |
| Subsystem | Report/Notes |
|----------:|:-------------|
| Docker | ClientAPI=1.41 ServerAPI=1.41 base: https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/7/artifact/out/Dockerfile |
| GITHUB PR | https://github.com/apache/hadoop/pull/5131 |
| Optional Tests | dupname asflicense compile javac javadoc mvninstall mvnsite unit shadedclient spotbugs checkstyle codespell detsecrets |
| uname | Linux a14520275a9e 4.15.0-191-generic #202-Ubuntu SMP Thu Aug 4 01:49:29 UTC 2022 x86_64 x86_64 x86_64 GNU/Linux |
| Build tool | maven |
| Personality | dev-support/bin/hadoop.sh |
| git revision | trunk / dd4643c822712166580c7e410ffc30edf8c2ce12 |
| Default Java | Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| Multi-JDK versions | /usr/lib/jvm/java-11-openjdk-amd64:Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 /usr/lib/jvm/java-8-openjdk-amd64:Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| Test Results | https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/7/testReport/ |
| Max. process+thread count | 966 (vs. ulimit of 5500) |
| modules | C: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router U: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server |
| Console output | https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/7/console |
| versions | git=2.25.1 maven=3.6.3 spotbugs=4.2.2 |
| Powered by | Apache Yetus 0.14.0 https://yetus.apache.org |
This message was automatically generated.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-issues-help@hadoop.apache.org
[GitHub] [hadoop] slfan1989 commented on pull request #5131: YARN-11350. [Federation] Router Support DelegationToken With ZK.
Posted by GitBox <gi...@apache.org>.
slfan1989 commented on PR #5131:
URL: https://github.com/apache/hadoop/pull/5131#issuecomment-1352408149
@goiri Thank you very much for helping to review the code!
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-issues-help@hadoop.apache.org
[GitHub] [hadoop] slfan1989 commented on a diff in pull request #5131: YARN-11350. [Federation] Router Support DelegationToken With ZK.
Posted by GitBox <gi...@apache.org>.
slfan1989 commented on code in PR #5131:
URL: https://github.com/apache/hadoop/pull/5131#discussion_r1023520913
##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/ZookeeperFederationStateStore.java:
##########
@@ -121,6 +134,14 @@
* |--- RESERVATION
* | |----- RESERVATION1
* | |----- RESERVATION2
+ * |--- ROUTER_RM_DT_SECRET_MANAGER_ROOT
+ * | |----- ROUTER_RM_DELEGATION_TOKENS_ROOT
+ * | | |----- RM_DELEGATION_TOKEN_1
+ * | | |----- RM_DELEGATION_TOKEN_2
+ * | | |----- RM_DELEGATION_TOKEN_3
+ * | |----- ROUTER_RM_DT_MASTER_KEYS_ROOT
+ * | | |----- DELEGATION_KEY_1
+ * |--- |----- ROUTER_RM_DT_SEQUENTIAL_NUMBER
Review Comment:
Thank you very much for reviewing the code, I will fix it.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-issues-help@hadoop.apache.org
[GitHub] [hadoop] hadoop-yetus commented on pull request #5131: YARN-11350. [Federation] Router Support DelegationToken With ZK.
Posted by GitBox <gi...@apache.org>.
hadoop-yetus commented on PR #5131:
URL: https://github.com/apache/hadoop/pull/5131#issuecomment-1337907868
:broken_heart: **-1 overall**
| Vote | Subsystem | Runtime | Logfile | Comment |
|:----:|----------:|--------:|:--------:|:-------:|
| +0 :ok: | reexec | 0m 47s | | Docker mode activated. |
|||| _ Prechecks _ |
| +1 :green_heart: | dupname | 0m 0s | | No case conflicting files found. |
| +0 :ok: | codespell | 0m 1s | | codespell was not available. |
| +0 :ok: | detsecrets | 0m 1s | | detect-secrets was not available. |
| +1 :green_heart: | @author | 0m 0s | | The patch does not contain any @author tags. |
| +1 :green_heart: | test4tests | 0m 0s | | The patch appears to include 1 new or modified test files. |
|||| _ trunk Compile Tests _ |
| +0 :ok: | mvndep | 16m 34s | | Maven dependency ordering for branch |
| +1 :green_heart: | mvninstall | 26m 8s | | trunk passed |
| +1 :green_heart: | compile | 3m 59s | | trunk passed with JDK Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 |
| +1 :green_heart: | compile | 3m 33s | | trunk passed with JDK Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | checkstyle | 1m 22s | | trunk passed |
| +1 :green_heart: | mvnsite | 2m 48s | | trunk passed |
| +1 :green_heart: | javadoc | 2m 40s | | trunk passed with JDK Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 |
| +1 :green_heart: | javadoc | 2m 17s | | trunk passed with JDK Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | spotbugs | 4m 52s | | trunk passed |
| +1 :green_heart: | shadedclient | 20m 49s | | branch has no errors when building and testing our client artifacts. |
| -0 :warning: | patch | 21m 16s | | Used diff version of patch file. Binary files and potentially other changes not applied. Please rebase and squash commits if necessary. |
|||| _ Patch Compile Tests _ |
| +0 :ok: | mvndep | 0m 30s | | Maven dependency ordering for patch |
| +1 :green_heart: | mvninstall | 2m 6s | | the patch passed |
| +1 :green_heart: | compile | 3m 50s | | the patch passed with JDK Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 |
| +1 :green_heart: | javac | 3m 50s | | the patch passed |
| +1 :green_heart: | compile | 3m 18s | | the patch passed with JDK Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | javac | 3m 18s | | the patch passed |
| +1 :green_heart: | blanks | 0m 1s | | The patch has no blanks issues. |
| +1 :green_heart: | checkstyle | 1m 8s | | the patch passed |
| +1 :green_heart: | mvnsite | 2m 18s | | the patch passed |
| -1 :x: | javadoc | 0m 39s | [/patch-javadoc-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-common-jdkUbuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/17/artifact/out/patch-javadoc-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-common-jdkUbuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04.txt) | hadoop-yarn-server-common in the patch failed with JDK Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04. |
| -1 :x: | javadoc | 0m 32s | [/patch-javadoc-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-common-jdkPrivateBuild-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/17/artifact/out/patch-javadoc-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-common-jdkPrivateBuild-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07.txt) | hadoop-yarn-server-common in the patch failed with JDK Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07. |
| +1 :green_heart: | spotbugs | 4m 35s | | the patch passed |
| +1 :green_heart: | shadedclient | 20m 46s | | patch has no errors when building and testing our client artifacts. |
|||| _ Other Tests _ |
| +1 :green_heart: | unit | 3m 13s | | hadoop-yarn-server-common in the patch passed. |
| +1 :green_heart: | unit | 99m 9s | | hadoop-yarn-server-resourcemanager in the patch passed. |
| +1 :green_heart: | unit | 0m 39s | | hadoop-yarn-server-router in the patch passed. |
| +1 :green_heart: | asflicense | 0m 43s | | The patch does not generate ASF License warnings. |
| | | 234m 26s | | |
| Subsystem | Report/Notes |
|----------:|:-------------|
| Docker | ClientAPI=1.41 ServerAPI=1.41 base: https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/17/artifact/out/Dockerfile |
| GITHUB PR | https://github.com/apache/hadoop/pull/5131 |
| Optional Tests | dupname asflicense compile javac javadoc mvninstall mvnsite unit shadedclient spotbugs checkstyle codespell detsecrets |
| uname | Linux 3f0e325cdbb0 4.15.0-191-generic #202-Ubuntu SMP Thu Aug 4 01:49:29 UTC 2022 x86_64 x86_64 x86_64 GNU/Linux |
| Build tool | maven |
| Personality | dev-support/bin/hadoop.sh |
| git revision | trunk / 481885676f755048767a0465f0a8c254877284b0 |
| Default Java | Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| Multi-JDK versions | /usr/lib/jvm/java-11-openjdk-amd64:Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 /usr/lib/jvm/java-8-openjdk-amd64:Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| Test Results | https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/17/testReport/ |
| Max. process+thread count | 977 (vs. ulimit of 5500) |
| modules | C: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router U: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server |
| Console output | https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/17/console |
| versions | git=2.25.1 maven=3.6.3 spotbugs=4.2.2 |
| Powered by | Apache Yetus 0.14.0 https://yetus.apache.org |
This message was automatically generated.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-issues-help@hadoop.apache.org
[GitHub] [hadoop] slfan1989 commented on a diff in pull request #5131: YARN-11350. [Federation] Router Support DelegationToken With ZK.
Posted by GitBox <gi...@apache.org>.
slfan1989 commented on code in PR #5131:
URL: https://github.com/apache/hadoop/pull/5131#discussion_r1041588318
##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/TestSQLFederationStateStore.java:
##########
@@ -18,21 +18,17 @@
package org.apache.hadoop.yarn.server.federation.store.impl;
import org.apache.commons.lang3.NotImplementedException;
+import org.apache.hadoop.security.token.delegation.DelegationKey;
import org.apache.hadoop.test.LambdaTestUtils;
import org.apache.hadoop.util.Time;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ReservationId;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier;
import org.apache.hadoop.yarn.server.federation.store.FederationStateStore;
import org.apache.hadoop.yarn.server.federation.store.metrics.FederationStateStoreClientMetrics;
-import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
-import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
-import org.apache.hadoop.yarn.server.federation.store.records.SubClusterRegisterRequest;
-import org.apache.hadoop.yarn.server.federation.store.records.ReservationHomeSubCluster;
-import org.apache.hadoop.yarn.server.federation.store.records.AddReservationHomeSubClusterRequest;
-import org.apache.hadoop.yarn.server.federation.store.records.UpdateReservationHomeSubClusterRequest;
-import org.apache.hadoop.yarn.server.federation.store.records.DeleteReservationHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.*;
Review Comment:
Thank you very much for helping to review the code, I will fix it.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-issues-help@hadoop.apache.org
[GitHub] [hadoop] hadoop-yetus commented on pull request #5131: YARN-11350. [Federation] Router Support DelegationToken With ZK.
Posted by GitBox <gi...@apache.org>.
hadoop-yetus commented on PR #5131:
URL: https://github.com/apache/hadoop/pull/5131#issuecomment-1330082726
:broken_heart: **-1 overall**
| Vote | Subsystem | Runtime | Logfile | Comment |
|:----:|----------:|--------:|:--------:|:-------:|
| +0 :ok: | reexec | 0m 44s | | Docker mode activated. |
|||| _ Prechecks _ |
| +1 :green_heart: | dupname | 0m 0s | | No case conflicting files found. |
| +0 :ok: | codespell | 0m 2s | | codespell was not available. |
| +0 :ok: | detsecrets | 0m 2s | | detect-secrets was not available. |
| +1 :green_heart: | @author | 0m 0s | | The patch does not contain any @author tags. |
| +1 :green_heart: | test4tests | 0m 0s | | The patch appears to include 1 new or modified test files. |
|||| _ trunk Compile Tests _ |
| +0 :ok: | mvndep | 16m 5s | | Maven dependency ordering for branch |
| +1 :green_heart: | mvninstall | 26m 21s | | trunk passed |
| +1 :green_heart: | compile | 4m 6s | | trunk passed with JDK Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 |
| +1 :green_heart: | compile | 3m 29s | | trunk passed with JDK Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | checkstyle | 1m 33s | | trunk passed |
| +1 :green_heart: | mvnsite | 2m 48s | | trunk passed |
| +1 :green_heart: | javadoc | 2m 42s | | trunk passed with JDK Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 |
| +1 :green_heart: | javadoc | 2m 19s | | trunk passed with JDK Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | spotbugs | 4m 46s | | trunk passed |
| +1 :green_heart: | shadedclient | 20m 20s | | branch has no errors when building and testing our client artifacts. |
| -0 :warning: | patch | 20m 44s | | Used diff version of patch file. Binary files and potentially other changes not applied. Please rebase and squash commits if necessary. |
|||| _ Patch Compile Tests _ |
| +0 :ok: | mvndep | 0m 30s | | Maven dependency ordering for patch |
| +1 :green_heart: | mvninstall | 2m 4s | | the patch passed |
| +1 :green_heart: | compile | 3m 49s | | the patch passed with JDK Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 |
| +1 :green_heart: | javac | 3m 49s | | the patch passed |
| +1 :green_heart: | compile | 3m 16s | | the patch passed with JDK Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | javac | 3m 16s | | the patch passed |
| +1 :green_heart: | blanks | 0m 0s | | The patch has no blanks issues. |
| +1 :green_heart: | checkstyle | 1m 11s | | the patch passed |
| +1 :green_heart: | mvnsite | 2m 16s | | the patch passed |
| -1 :x: | javadoc | 0m 40s | [/patch-javadoc-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-common-jdkUbuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/13/artifact/out/patch-javadoc-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-common-jdkUbuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04.txt) | hadoop-yarn-server-common in the patch failed with JDK Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04. |
| -1 :x: | javadoc | 0m 33s | [/patch-javadoc-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-common-jdkPrivateBuild-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/13/artifact/out/patch-javadoc-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-common-jdkPrivateBuild-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07.txt) | hadoop-yarn-server-common in the patch failed with JDK Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07. |
| +1 :green_heart: | spotbugs | 4m 32s | | the patch passed |
| +1 :green_heart: | shadedclient | 20m 25s | | patch has no errors when building and testing our client artifacts. |
|||| _ Other Tests _ |
| +1 :green_heart: | unit | 3m 12s | | hadoop-yarn-server-common in the patch passed. |
| +1 :green_heart: | unit | 99m 17s | | hadoop-yarn-server-resourcemanager in the patch passed. |
| +1 :green_heart: | unit | 0m 39s | | hadoop-yarn-server-router in the patch passed. |
| +1 :green_heart: | asflicense | 0m 49s | | The patch does not generate ASF License warnings. |
| | | 233m 49s | | |
| Subsystem | Report/Notes |
|----------:|:-------------|
| Docker | ClientAPI=1.41 ServerAPI=1.41 base: https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/13/artifact/out/Dockerfile |
| GITHUB PR | https://github.com/apache/hadoop/pull/5131 |
| Optional Tests | dupname asflicense compile javac javadoc mvninstall mvnsite unit shadedclient spotbugs checkstyle codespell detsecrets |
| uname | Linux 7a9ea753c924 4.15.0-191-generic #202-Ubuntu SMP Thu Aug 4 01:49:29 UTC 2022 x86_64 x86_64 x86_64 GNU/Linux |
| Build tool | maven |
| Personality | dev-support/bin/hadoop.sh |
| git revision | trunk / d20a9fc250d6c14644fd834bae89877ff5b20d8b |
| Default Java | Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| Multi-JDK versions | /usr/lib/jvm/java-11-openjdk-amd64:Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 /usr/lib/jvm/java-8-openjdk-amd64:Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| Test Results | https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/13/testReport/ |
| Max. process+thread count | 967 (vs. ulimit of 5500) |
| modules | C: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router U: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server |
| Console output | https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/13/console |
| versions | git=2.25.1 maven=3.6.3 spotbugs=4.2.2 |
| Powered by | Apache Yetus 0.14.0 https://yetus.apache.org |
This message was automatically generated.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-issues-help@hadoop.apache.org
[GitHub] [hadoop] slfan1989 commented on a diff in pull request #5131: YARN-11350. [Federation] Router Support DelegationToken With ZK.
Posted by GitBox <gi...@apache.org>.
slfan1989 commented on code in PR #5131:
URL: https://github.com/apache/hadoop/pull/5131#discussion_r1036628806
##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/ZookeeperFederationStateStore.java:
##########
@@ -886,45 +1000,608 @@ public UpdateReservationHomeSubClusterResponse updateReservationHomeSubCluster(
return UpdateReservationHomeSubClusterResponse.newInstance();
}
+ /**
+ * ZookeeperFederationStateStore Supports Store NewMasterKey.
+ *
+ * @param request The request contains RouterMasterKey, which is an abstraction for DelegationKey
+ * @return routerMasterKeyResponse, the response contains the RouterMasterKey.
+ * @throws YarnException if the call to the state store is unsuccessful.
+ * @throws IOException An IO Error occurred.
+ */
@Override
public RouterMasterKeyResponse storeNewMasterKey(RouterMasterKeyRequest request)
throws YarnException, IOException {
- throw new NotImplementedException("Code is not implemented");
+
+ long start = clock.getTime();
+ // For the verification of the request, after passing the verification,
+ // the request and the internal objects will not be empty and can be used directly.
+ FederationRouterRMTokenInputValidator.validate(request);
+
+ // Parse the delegationKey from the request and get the ZK storage path.
+ DelegationKey delegationKey = convertMasterKeyToDelegationKey(request);
+ String nodeCreatePath = getMasterKeyZNodePathByDelegationKey(delegationKey);
+ LOG.debug("Storing RMDelegationKey_{}, ZkNodePath = {}.", delegationKey.getKeyId(),
+ nodeCreatePath);
+
+ // Write master key data to zk.
+ try(ByteArrayOutputStream os = new ByteArrayOutputStream();
+ DataOutputStream fsOut = new DataOutputStream(os)) {
+ delegationKey.write(fsOut);
+ put(nodeCreatePath, os.toByteArray(), false);
+ }
+
+ // Get the stored masterKey from zk.
+ RouterMasterKey masterKeyFromZK = getRouterMasterKeyFromZK(nodeCreatePath);
+ long end = clock.getTime();
+ opDurations.addStoreNewMasterKeyDuration(start, end);
+ return RouterMasterKeyResponse.newInstance(masterKeyFromZK);
}
+ /**
+ * ZookeeperFederationStateStore Supports Remove MasterKey.
+ *
+ * @param request The request contains RouterMasterKey, which is an abstraction for DelegationKey
+ * @return routerMasterKeyResponse, the response contains the RouterMasterKey.
+ * @throws YarnException if the call to the state store is unsuccessful.
+ * @throws IOException An IO Error occurred.
+ */
@Override
public RouterMasterKeyResponse removeStoredMasterKey(RouterMasterKeyRequest request)
throws YarnException, IOException {
- throw new NotImplementedException("Code is not implemented");
+
+ long start = clock.getTime();
+ // For the verification of the request, after passing the verification,
+ // the request and the internal objects will not be empty and can be used directly.
+ FederationRouterRMTokenInputValidator.validate(request);
+
+ try {
+ // Parse the delegationKey from the request and get the ZK storage path.
+ RouterMasterKey masterKey = request.getRouterMasterKey();
+ DelegationKey delegationKey = convertMasterKeyToDelegationKey(request);
+ String nodeRemovePath = getMasterKeyZNodePathByDelegationKey(delegationKey);
+ LOG.debug("Removing RMDelegationKey_{}, ZkNodePath = {}.", delegationKey.getKeyId(),
+ nodeRemovePath);
+
+ // Check if the path exists, Throws an exception if the path does not exist.
+ if (!exists(nodeRemovePath)) {
+ throw new YarnException("ZkNodePath = " + nodeRemovePath + " not exists!");
+ }
+
+ // try to remove masterKey.
+ zkManager.delete(nodeRemovePath);
+ long end = clock.getTime();
+ opDurations.removeStoredMasterKeyDuration(start, end);
+ return RouterMasterKeyResponse.newInstance(masterKey);
+ } catch (Exception e) {
+ throw new YarnException(e);
+ }
}
+ /**
+ * ZookeeperFederationStateStore Supports Remove MasterKey.
+ *
+ * @param request The request contains RouterMasterKey, which is an abstraction for DelegationKey
+ * @return routerMasterKeyResponse, the response contains the RouterMasterKey.
+ * @throws YarnException if the call to the state store is unsuccessful.
+ * @throws IOException An IO Error occurred.
+ */
@Override
public RouterMasterKeyResponse getMasterKeyByDelegationKey(RouterMasterKeyRequest request)
throws YarnException, IOException {
- throw new NotImplementedException("Code is not implemented");
+
+ long start = clock.getTime();
+ // For the verification of the request, after passing the verification,
+ // the request and the internal objects will not be empty and can be used directly.
+ FederationRouterRMTokenInputValidator.validate(request);
+
+ try {
+
+ // Parse the delegationKey from the request and get the ZK storage path.
+ DelegationKey delegationKey = convertMasterKeyToDelegationKey(request);
+ String nodePath = getMasterKeyZNodePathByDelegationKey(delegationKey);
+
+ // Check if the path exists, Throws an exception if the path does not exist.
+ if (!exists(nodePath)) {
+ throw new YarnException("ZkNodePath = " + nodePath + " not exists!");
+ }
+
+ // Get the stored masterKey from zk.
+ RouterMasterKey routerMasterKey = getRouterMasterKeyFromZK(nodePath);
+ long end = clock.getTime();
+ opDurations.getMasterKeyByDelegationKeyDuration(start, end);
+ return RouterMasterKeyResponse.newInstance(routerMasterKey);
+ } catch (Exception e) {
+ throw new YarnException(e);
+ }
+ }
+
+ /**
+ * Get MasterKeyZNodePath based on DelegationKey.
+ *
+ * @param delegationKey delegationKey.
+ * @return masterKey ZNodePath.
+ */
+ private String getMasterKeyZNodePathByDelegationKey(DelegationKey delegationKey) {
+ return getMasterKeyZNodePathByKeyId(delegationKey.getKeyId());
}
+ /**
+ * Get MasterKeyZNodePath based on KeyId.
+ *
+ * @param keyId master key id.
+ * @return masterKey ZNodePath.
+ */
+ private String getMasterKeyZNodePathByKeyId(int keyId) {
+ String nodeName = ROUTER_RM_DELEGATION_KEY_PREFIX + keyId;
+ return getNodePath(routerRMDTMasterKeysRootPath, nodeName);
+ }
+
+ /**
+ * Get RouterMasterKey from ZK.
+ *
+ * @param nodePath The path where masterKey is stored in zk.
+ *
+ * @return RouterMasterKey.
+ * @throws IOException An IO Error occurred.
+ */
+ private RouterMasterKey getRouterMasterKeyFromZK(String nodePath)
+ throws IOException {
+ try {
+ byte[] data = get(nodePath);
+ if ((data == null) || (data.length == 0)) {
+ return null;
+ }
+
+ ByteArrayInputStream bin = new ByteArrayInputStream(data);
+ DataInputStream din = new DataInputStream(bin);
+ DelegationKey key = new DelegationKey();
+ key.readFields(din);
+
+ return RouterMasterKey.newInstance(key.getKeyId(),
+ ByteBuffer.wrap(key.getEncodedKey()), key.getExpiryDate());
+ } catch (Exception ex) {
+ LOG.error("No node in path {}.", nodePath);
+ throw new IOException(ex);
+ }
+ }
+
+ /**
+ * ZookeeperFederationStateStore Supports Store RMDelegationTokenIdentifier.
+ *
+ * The stored token method is a synchronized method
+ * used to ensure that storeNewToken is a thread-safe method.
+ *
+ * @param request The request contains RouterRMToken (RMDelegationTokenIdentifier and renewDate)
+ * @return routerRMTokenResponse, the response contains the RouterStoreToken.
+ * @throws YarnException if the call to the state store is unsuccessful.
+ * @throws IOException An IO Error occurred.
+ */
@Override
public RouterRMTokenResponse storeNewToken(RouterRMTokenRequest request)
throws YarnException, IOException {
- throw new NotImplementedException("Code is not implemented");
+
+ long start = clock.getTime();
+ // We verify the RouterRMTokenRequest to ensure that the request is not empty,
+ // and that the internal RouterStoreToken is not empty.
+ FederationRouterRMTokenInputValidator.validate(request);
+
+ try {
+
+ // add delegationToken
+ storeOrUpdateRouterRMDT(request, false);
+
+ // Get the stored delegationToken from ZK and return.
+ RouterStoreToken resultStoreToken = getStoreTokenFromZK(request, false);
+ long end = clock.getTime();
+ opDurations.getStoreNewTokenDuration(start, end);
+ return RouterRMTokenResponse.newInstance(resultStoreToken);
+ } catch (YarnException | IOException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new YarnException(e);
+ }
}
+ /**
+ * ZookeeperFederationStateStore Supports Update RMDelegationTokenIdentifier.
+ *
+ * The update stored token method is a synchronized method
+ * used to ensure that storeNewToken is a thread-safe method.
+ *
+ * @param request The request contains RouterRMToken (RMDelegationTokenIdentifier and renewDate)
+ * @return routerRMTokenResponse, the response contains the RouterStoreToken.
+ * @throws YarnException if the call to the state store is unsuccessful.
+ * @throws IOException An IO Error occurred.
+ */
@Override
public RouterRMTokenResponse updateStoredToken(RouterRMTokenRequest request)
throws YarnException, IOException {
- throw new NotImplementedException("Code is not implemented");
+
+ long start = clock.getTime();
+ // We verify the RouterRMTokenRequest to ensure that the request is not empty,
+ // and that the internal RouterStoreToken is not empty.
+ FederationRouterRMTokenInputValidator.validate(request);
+
+ try {
+
+ // get the Token storage path
+ String nodePath = getStoreTokenZNodePathByTokenRequest(request);
+
+ // updateStoredToken needs to determine whether the zkNode exists.
+ // If it exists, update the token data.
+ // If it does not exist, write the new token data directly.
+ boolean pathExists = true;
+ if (!exists(nodePath)) {
+ pathExists = false;
+ }
+
+ if (pathExists) {
+ // update delegationToken
+ storeOrUpdateRouterRMDT(request, true);
+ } else {
+ // add new delegationToken
+ storeNewToken(request);
+ }
+
+ // Get the stored delegationToken from ZK and return.
+ RouterStoreToken resultStoreToken = getStoreTokenFromZK(request, false);
+ long end = clock.getTime();
+ opDurations.updateStoredTokenDuration(start, end);
+ return RouterRMTokenResponse.newInstance(resultStoreToken);
+ } catch (YarnException | IOException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new YarnException(e);
+ }
}
+ /**
+ * ZookeeperFederationStateStore Supports Remove RMDelegationTokenIdentifier.
+ *
+ * The remove stored token method is a synchronized method
+ * used to ensure that storeNewToken is a thread-safe method.
+ *
+ * @param request The request contains RouterRMToken (RMDelegationTokenIdentifier and renewDate)
+ * @return routerRMTokenResponse, the response contains the RouterStoreToken.
+ * @throws YarnException if the call to the state store is unsuccessful.
+ * @throws IOException An IO Error occurred.
+ */
@Override
public RouterRMTokenResponse removeStoredToken(RouterRMTokenRequest request)
throws YarnException, IOException {
- throw new NotImplementedException("Code is not implemented");
+
+ long start = clock.getTime();
+ // We verify the RouterRMTokenRequest to ensure that the request is not empty,
+ // and that the internal RouterStoreToken is not empty.
+ FederationRouterRMTokenInputValidator.validate(request);
+
+ try {
+
+ // get the Token storage path
+ String nodePath = getStoreTokenZNodePathByTokenRequest(request);
+
+ // If the path to be deleted does not exist, throw an exception directly.
+ if (!exists(nodePath)) {
+ throw new YarnException("ZkNodePath = " + nodePath + " not exists!");
+ }
+
+ // Check again, first get the data from ZK,
+ // if the data is not empty, then delete it
+ RouterStoreToken storeToken = getStoreTokenFromZK(request, false);
+ if (storeToken != null) {
+ zkManager.delete(nodePath);
+ }
+
+ // return deleted token data.
+ long end = clock.getTime();
+ opDurations.removeStoredTokenDuration(start, end);
+ return RouterRMTokenResponse.newInstance(storeToken);
+ } catch (YarnException | IOException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new YarnException(e);
+ }
}
+ /**
+ * The Router Supports GetTokenByRouterStoreToken.
+ *
+ * @param request The request contains RouterRMToken (RMDelegationTokenIdentifier and renewDate)
+ * @return RouterRMTokenResponse.
+ * @throws YarnException if the call to the state store is unsuccessful
+ * @throws IOException An IO Error occurred
+ */
@Override
public RouterRMTokenResponse getTokenByRouterStoreToken(RouterRMTokenRequest request)
throws YarnException, IOException {
- throw new NotImplementedException("Code is not implemented");
+
+ long start = clock.getTime();
+ // We verify the RouterRMTokenRequest to ensure that the request is not empty,
+ // and that the internal RouterStoreToken is not empty.
+ FederationRouterRMTokenInputValidator.validate(request);
+
+ try {
+
+ // Before get the token,
+ // we need to determine whether the path where the token is stored exists.
+ // If it doesn't exist, we will throw an exception.
+ String nodePath = getStoreTokenZNodePathByTokenRequest(request);
+ if (!exists(nodePath)) {
+ throw new YarnException("ZkNodePath = " + nodePath + " not exists!");
+ }
+
+ // Get the stored delegationToken from ZK and return.
+ RouterStoreToken resultStoreToken = getStoreTokenFromZK(request, false);
+ // return deleted token data.
+ long end = clock.getTime();
+ opDurations.getTokenByRouterStoreTokenDuration(start, end);
+ return RouterRMTokenResponse.newInstance(resultStoreToken);
+ } catch (YarnException | IOException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new YarnException(e);
+ }
+ }
+
+ /**
+ * Convert MasterKey to DelegationKey.
+ *
+ * Before using this function,
+ * please use FederationRouterRMTokenInputValidator to verify the request.
+ * By default, the request is not empty, and the internal object is not empty.
+ *
+ * @param request RouterMasterKeyRequest
+ * @return DelegationKey.
+ */
+ private DelegationKey convertMasterKeyToDelegationKey(RouterMasterKeyRequest request) {
+ RouterMasterKey masterKey = request.getRouterMasterKey();
+ return convertMasterKeyToDelegationKey(masterKey);
+ }
+
+ /**
+ * Convert MasterKey to DelegationKey.
+ *
+ * @param masterKey masterKey.
+ * @return DelegationKey.
+ */
+ private DelegationKey convertMasterKeyToDelegationKey(RouterMasterKey masterKey) {
+ ByteBuffer keyByteBuf = masterKey.getKeyBytes();
+ byte[] keyBytes = new byte[keyByteBuf.remaining()];
+ keyByteBuf.get(keyBytes);
+ return new DelegationKey(masterKey.getKeyId(), masterKey.getExpiryDate(), keyBytes);
+ }
+
+ /**
+ * Check if a path exists in zk.
+ *
+ * @param path Path to be checked.
+ * @return Returns true if the path exists, false if the path does not exist.
+ * @throws Exception When an exception to access zk occurs.
+ */
+ @VisibleForTesting
+ boolean exists(final String path) throws Exception {
+ return zkManager.exists(path);
+ }
+
+ /**
+ * Add or update delegationToken.
+ *
+ * Before using this function,
+ * please use FederationRouterRMTokenInputValidator to verify the request.
+ * By default, the request is not empty, and the internal object is not empty.
+ *
+ * @param request storeToken
+ * @param isUpdate true, update the token; false, create a new token.
+ * @throws Exception exception occurs.
+ */
+ private void storeOrUpdateRouterRMDT(RouterRMTokenRequest request, boolean isUpdate)
+ throws Exception {
+
+ RouterStoreToken routerStoreToken = request.getRouterStoreToken();
+ String nodeCreatePath = getStoreTokenZNodePathByTokenRequest(request);
+ LOG.debug("nodeCreatePath = {}, isUpdate = {}", nodeCreatePath, isUpdate);
+ put(nodeCreatePath, routerStoreToken.toByteArray(), isUpdate);
+ }
+
+ /**
+ * Get ZNode Path of StoreToken.
+ *
+ * Before using this method, we should use FederationRouterRMTokenInputValidator
+ * to verify the request,ensure that the request is not empty,
+ * and ensure that the object in the request is not empty.
+ *
+ * @param request RouterMasterKeyRequest.
+ * @return RouterRMToken ZNode Path.
+ * @throws IOException io exception occurs.
+ */
+ private String getStoreTokenZNodePathByTokenRequest(RouterRMTokenRequest request)
+ throws IOException {
+ RouterStoreToken routerStoreToken = request.getRouterStoreToken();
+ YARNDelegationTokenIdentifier identifier = routerStoreToken.getTokenIdentifier();
+ return getStoreTokenZNodePathByIdentifier(identifier);
+ }
+
+ /**
+ * Get ZNode Path of StoreToken.
+ *
+ * @param identifier YARNDelegationTokenIdentifier
+ * @return RouterRMToken ZNode Path.
+ */
+ private String getStoreTokenZNodePathByIdentifier(YARNDelegationTokenIdentifier identifier) {
+ String nodePath = getNodePath(routerRMDelegationTokensRootPath,
+ ROUTER_RM_DELEGATION_TOKEN_PREFIX + identifier.getSequenceNumber());
+ return nodePath;
+ }
+
+ /**
+ * Get RouterStoreToken from ZK.
+ *
+ * @param request RouterMasterKeyRequest.
+ * @param quiet If true is silent mode, no error message is printed at this time,
+ * if false is non-silent mode, error message is printed at this time.
+ * @return RouterStoreToken.
+ * @throws IOException io exception occurs.
+ */
+ private RouterStoreToken getStoreTokenFromZK(RouterRMTokenRequest request,
+ boolean quiet) throws IOException {
+ RouterStoreToken routerStoreToken = request.getRouterStoreToken();
+ YARNDelegationTokenIdentifier identifier = routerStoreToken.getTokenIdentifier();
+ return getStoreTokenFromZK(identifier, quiet);
+ }
+
+ /**
+ * Get RouterStoreToken from ZK.
+ *
+ * @param identifier YARN DelegationToken Identifier
+ * @param quiet Whether it is in quiet mode,
+ * if it is in quiet mode, no exception information will be output.
+ * @return RouterStoreToken.
+ * @throws IOException io exception occurs.
+ */
+ private RouterStoreToken getStoreTokenFromZK(YARNDelegationTokenIdentifier identifier,
+ boolean quiet) throws IOException {
+ // get the Token storage path
+ String nodePath = getStoreTokenZNodePathByIdentifier(identifier);
+ return getStoreTokenFromZK(nodePath, quiet);
+ }
+
+ /**
+ * Get RouterStoreToken from ZK.
+ *
+ * @param nodePath Znode location where data is stored.
+ * @param quiet Whether it is in quiet mode,
+ * if it is in quiet mode, no exception information will be output.
+ * @return RouterStoreToken.
+ * @throws IOException io exception occurs.
+ */
+ private RouterStoreToken getStoreTokenFromZK(String nodePath, boolean quiet)
+ throws IOException {
+ try {
+ byte[] data = get(nodePath);
+ if ((data == null) || (data.length == 0)) {
+ return null;
+ }
+ ByteArrayInputStream bin = new ByteArrayInputStream(data);
+ DataInputStream din = new DataInputStream(bin);
+ RouterStoreToken storeToken = Records.newRecord(RouterStoreToken.class);
+ storeToken.readFields(din);
+ return storeToken;
+ } catch (Exception ex) {
+ if (!quiet) {
+ LOG.error("No node in path [" + nodePath + "]");
Review Comment:
Thank you very much for reviewing the code, I will fix it.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-issues-help@hadoop.apache.org
[GitHub] [hadoop] slfan1989 commented on pull request #5131: YARN-11350. [Federation] Router Support DelegationToken With ZK.
Posted by GitBox <gi...@apache.org>.
slfan1989 commented on PR #5131:
URL: https://github.com/apache/hadoop/pull/5131#issuecomment-1320648545
> Thanks @slfan1989 , Great work here!
> One concern out of this PR, we have HDFS RBF / YARN Federation / KMS both depend on zookeeper to store DelegationToken. From my view, both of their logic are similar, we should consider implement one common module about ZKDelegationTokenManager? Thus it will reduce overhead to implement at each sub-project side. WDYT? cc @goiri
> NOTE: This is not blocker comments. Thanks.
Thank you very much for your suggestion, I agree with your idea. After completing this pr, I will continue to carefully read the codes of the HDFS RBF / KMS modules about delegation, and then extract this part of the code to the common module. If other modules need it, we will can be used directly.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-issues-help@hadoop.apache.org
[GitHub] [hadoop] hadoop-yetus commented on pull request #5131: YARN-11350. [Federation] Router Support DelegationToken With ZK.
Posted by GitBox <gi...@apache.org>.
hadoop-yetus commented on PR #5131:
URL: https://github.com/apache/hadoop/pull/5131#issuecomment-1317398111
:broken_heart: **-1 overall**
| Vote | Subsystem | Runtime | Logfile | Comment |
|:----:|----------:|--------:|:--------:|:-------:|
| +0 :ok: | reexec | 0m 42s | | Docker mode activated. |
|||| _ Prechecks _ |
| +1 :green_heart: | dupname | 0m 1s | | No case conflicting files found. |
| +0 :ok: | codespell | 0m 0s | | codespell was not available. |
| +0 :ok: | detsecrets | 0m 0s | | detect-secrets was not available. |
| +1 :green_heart: | @author | 0m 0s | | The patch does not contain any @author tags. |
| +1 :green_heart: | test4tests | 0m 0s | | The patch appears to include 3 new or modified test files. |
|||| _ trunk Compile Tests _ |
| +0 :ok: | mvndep | 15m 18s | | Maven dependency ordering for branch |
| +1 :green_heart: | mvninstall | 25m 57s | | trunk passed |
| +1 :green_heart: | compile | 3m 58s | | trunk passed with JDK Ubuntu-11.0.17+8-post-Ubuntu-1ubuntu220.04 |
| +1 :green_heart: | compile | 3m 34s | | trunk passed with JDK Private Build-1.8.0_352-8u352-ga-1~20.04-b08 |
| +1 :green_heart: | checkstyle | 1m 32s | | trunk passed |
| +1 :green_heart: | mvnsite | 2m 44s | | trunk passed |
| -1 :x: | javadoc | 0m 52s | [/branch-javadoc-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-common-jdkUbuntu-11.0.17+8-post-Ubuntu-1ubuntu220.04.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/5/artifact/out/branch-javadoc-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-common-jdkUbuntu-11.0.17+8-post-Ubuntu-1ubuntu220.04.txt) | hadoop-yarn-server-common in trunk failed with JDK Ubuntu-11.0.17+8-post-Ubuntu-1ubuntu220.04. |
| -1 :x: | javadoc | 0m 54s | [/branch-javadoc-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-resourcemanager-jdkUbuntu-11.0.17+8-post-Ubuntu-1ubuntu220.04.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/5/artifact/out/branch-javadoc-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-resourcemanager-jdkUbuntu-11.0.17+8-post-Ubuntu-1ubuntu220.04.txt) | hadoop-yarn-server-resourcemanager in trunk failed with JDK Ubuntu-11.0.17+8-post-Ubuntu-1ubuntu220.04. |
| +1 :green_heart: | javadoc | 2m 15s | | trunk passed with JDK Private Build-1.8.0_352-8u352-ga-1~20.04-b08 |
| +1 :green_heart: | spotbugs | 5m 6s | | trunk passed |
| +1 :green_heart: | shadedclient | 23m 44s | | branch has no errors when building and testing our client artifacts. |
|||| _ Patch Compile Tests _ |
| +0 :ok: | mvndep | 0m 28s | | Maven dependency ordering for patch |
| +1 :green_heart: | mvninstall | 2m 6s | | the patch passed |
| +1 :green_heart: | compile | 4m 2s | | the patch passed with JDK Ubuntu-11.0.17+8-post-Ubuntu-1ubuntu220.04 |
| +1 :green_heart: | javac | 4m 2s | | the patch passed |
| +1 :green_heart: | compile | 3m 26s | | the patch passed with JDK Private Build-1.8.0_352-8u352-ga-1~20.04-b08 |
| +1 :green_heart: | javac | 3m 26s | | the patch passed |
| +1 :green_heart: | blanks | 0m 0s | | The patch has no blanks issues. |
| +1 :green_heart: | checkstyle | 1m 13s | | the patch passed |
| +1 :green_heart: | mvnsite | 2m 23s | | the patch passed |
| -1 :x: | javadoc | 0m 35s | [/patch-javadoc-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-common-jdkUbuntu-11.0.17+8-post-Ubuntu-1ubuntu220.04.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/5/artifact/out/patch-javadoc-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-common-jdkUbuntu-11.0.17+8-post-Ubuntu-1ubuntu220.04.txt) | hadoop-yarn-server-common in the patch failed with JDK Ubuntu-11.0.17+8-post-Ubuntu-1ubuntu220.04. |
| -1 :x: | javadoc | 0m 50s | [/patch-javadoc-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-resourcemanager-jdkUbuntu-11.0.17+8-post-Ubuntu-1ubuntu220.04.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/5/artifact/out/patch-javadoc-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-resourcemanager-jdkUbuntu-11.0.17+8-post-Ubuntu-1ubuntu220.04.txt) | hadoop-yarn-server-resourcemanager in the patch failed with JDK Ubuntu-11.0.17+8-post-Ubuntu-1ubuntu220.04. |
| +1 :green_heart: | javadoc | 0m 33s | | hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-common-jdkPrivateBuild-1.8.0_352-8u352-ga-1~20.04-b08 with JDK Private Build-1.8.0_352-8u352-ga-1~20.04-b08 generated 0 new + 47 unchanged - 1 fixed = 47 total (was 48) |
| +1 :green_heart: | javadoc | 0m 46s | | hadoop-yarn-server-resourcemanager in the patch passed with JDK Private Build-1.8.0_352-8u352-ga-1~20.04-b08. |
| +1 :green_heart: | javadoc | 0m 26s | | hadoop-yarn-server-router in the patch passed with JDK Private Build-1.8.0_352-8u352-ga-1~20.04-b08. |
| +1 :green_heart: | spotbugs | 4m 40s | | the patch passed |
| +1 :green_heart: | shadedclient | 20m 36s | | patch has no errors when building and testing our client artifacts. |
|||| _ Other Tests _ |
| +1 :green_heart: | unit | 3m 12s | | hadoop-yarn-server-common in the patch passed. |
| +1 :green_heart: | unit | 99m 4s | | hadoop-yarn-server-resourcemanager in the patch passed. |
| +1 :green_heart: | unit | 5m 52s | | hadoop-yarn-server-router in the patch passed. |
| +1 :green_heart: | asflicense | 0m 44s | | The patch does not generate ASF License warnings. |
| | | 241m 31s | | |
| Subsystem | Report/Notes |
|----------:|:-------------|
| Docker | ClientAPI=1.41 ServerAPI=1.41 base: https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/5/artifact/out/Dockerfile |
| GITHUB PR | https://github.com/apache/hadoop/pull/5131 |
| Optional Tests | dupname asflicense compile javac javadoc mvninstall mvnsite unit shadedclient spotbugs checkstyle codespell detsecrets |
| uname | Linux 8aba9d12aaf7 4.15.0-191-generic #202-Ubuntu SMP Thu Aug 4 01:49:29 UTC 2022 x86_64 x86_64 x86_64 GNU/Linux |
| Build tool | maven |
| Personality | dev-support/bin/hadoop.sh |
| git revision | trunk / 033f712251063470c3085981d56ea36a50109f97 |
| Default Java | Private Build-1.8.0_352-8u352-ga-1~20.04-b08 |
| Multi-JDK versions | /usr/lib/jvm/java-11-openjdk-amd64:Ubuntu-11.0.17+8-post-Ubuntu-1ubuntu220.04 /usr/lib/jvm/java-8-openjdk-amd64:Private Build-1.8.0_352-8u352-ga-1~20.04-b08 |
| Test Results | https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/5/testReport/ |
| Max. process+thread count | 969 (vs. ulimit of 5500) |
| modules | C: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router U: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server |
| Console output | https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/5/console |
| versions | git=2.25.1 maven=3.6.3 spotbugs=4.2.2 |
| Powered by | Apache Yetus 0.14.0 https://yetus.apache.org |
This message was automatically generated.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-issues-help@hadoop.apache.org
[GitHub] [hadoop] slfan1989 commented on a diff in pull request #5131: YARN-11350. [Federation] Router Support DelegationToken With ZK.
Posted by GitBox <gi...@apache.org>.
slfan1989 commented on code in PR #5131:
URL: https://github.com/apache/hadoop/pull/5131#discussion_r1030632183
##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/ZookeeperFederationStateStore.java:
##########
@@ -886,45 +975,554 @@ public UpdateReservationHomeSubClusterResponse updateReservationHomeSubCluster(
return UpdateReservationHomeSubClusterResponse.newInstance();
}
+ /**
+ * ZookeeperFederationStateStore Supports Store NewMasterKey.
+ *
+ * @param request The request contains RouterMasterKey, which is an abstraction for DelegationKey
+ * @return routerMasterKeyResponse, the response contains the RouterMasterKey.
+ * @throws YarnException if the call to the state store is unsuccessful.
+ * @throws IOException An IO Error occurred.
+ */
@Override
- public RouterMasterKeyResponse storeNewMasterKey(RouterMasterKeyRequest request)
+ public synchronized RouterMasterKeyResponse storeNewMasterKey(RouterMasterKeyRequest request)
throws YarnException, IOException {
- throw new NotImplementedException("Code is not implemented");
+
+ long start = clock.getTime();
+ // For the verification of the request, after passing the verification,
+ // the request and the internal objects will not be empty and can be used directly.
+ FederationRouterRMTokenInputValidator.validate(request);
+
+ // Parse the delegationKey from the request and get the ZK storage path.
+ DelegationKey delegationKey = convertMasterKeyToDelegationKey(request);
+ String nodeCreatePath = getMasterKeyZNodePathByDelegationKey(delegationKey);
+ LOG.debug("Storing RMDelegationKey_{}, ZkNodePath = {}.", delegationKey.getKeyId(),
+ nodeCreatePath);
+
+ // Write master key data to zk.
+ try(ByteArrayOutputStream os = new ByteArrayOutputStream();
+ DataOutputStream fsOut = new DataOutputStream(os)) {
+ delegationKey.write(fsOut);
+ put(nodeCreatePath, os.toByteArray(), false);
+ }
+
+ // Get the stored masterKey from zk.
+ RouterMasterKey masterKeyFromZK = getRouterMasterKeyFromZK(nodeCreatePath);
+ long end = clock.getTime();
+ opDurations.addStoreNewMasterKeyDuration(start, end);
+ return RouterMasterKeyResponse.newInstance(masterKeyFromZK);
}
+ /**
+ * ZookeeperFederationStateStore Supports Remove MasterKey.
+ *
+ * @param request The request contains RouterMasterKey, which is an abstraction for DelegationKey
+ * @return routerMasterKeyResponse, the response contains the RouterMasterKey.
+ * @throws YarnException if the call to the state store is unsuccessful.
+ * @throws IOException An IO Error occurred.
+ */
@Override
- public RouterMasterKeyResponse removeStoredMasterKey(RouterMasterKeyRequest request)
+ public synchronized RouterMasterKeyResponse removeStoredMasterKey(RouterMasterKeyRequest request)
throws YarnException, IOException {
- throw new NotImplementedException("Code is not implemented");
+
+ long start = clock.getTime();
+ // For the verification of the request, after passing the verification,
+ // the request and the internal objects will not be empty and can be used directly.
+ FederationRouterRMTokenInputValidator.validate(request);
+
+ try {
+ // Parse the delegationKey from the request and get the ZK storage path.
+ RouterMasterKey masterKey = request.getRouterMasterKey();
+ DelegationKey delegationKey = convertMasterKeyToDelegationKey(request);
+ String nodeRemovePath = getMasterKeyZNodePathByDelegationKey(delegationKey);
+ LOG.debug("Removing RMDelegationKey_{}, ZkNodePath = {}.", delegationKey.getKeyId(),
+ nodeRemovePath);
+
+ // Check if the path exists, Throws an exception if the path does not exist.
+ if (!exists(nodeRemovePath)) {
+ throw new YarnException("ZkNodePath = " + nodeRemovePath + " not exists!");
+ }
+
+ // try to remove masterKey.
+ zkManager.delete(nodeRemovePath);
+ long end = clock.getTime();
+ opDurations.removeStoredMasterKeyDuration(start, end);
+ return RouterMasterKeyResponse.newInstance(masterKey);
+ } catch (Exception e) {
+ throw new YarnException(e);
+ }
}
+ /**
+ * ZookeeperFederationStateStore Supports Remove MasterKey.
+ *
+ * @param request The request contains RouterMasterKey, which is an abstraction for DelegationKey
+ * @return routerMasterKeyResponse, the response contains the RouterMasterKey.
+ * @throws YarnException if the call to the state store is unsuccessful.
+ * @throws IOException An IO Error occurred.
+ */
@Override
public RouterMasterKeyResponse getMasterKeyByDelegationKey(RouterMasterKeyRequest request)
throws YarnException, IOException {
- throw new NotImplementedException("Code is not implemented");
+
+ long start = clock.getTime();
+ // For the verification of the request, after passing the verification,
+ // the request and the internal objects will not be empty and can be used directly.
+ FederationRouterRMTokenInputValidator.validate(request);
+
+ try {
+
+ // Parse the delegationKey from the request and get the ZK storage path.
+ DelegationKey delegationKey = convertMasterKeyToDelegationKey(request);
+ String nodePath = getMasterKeyZNodePathByDelegationKey(delegationKey);
+
+ // Check if the path exists, Throws an exception if the path does not exist.
+ if (!exists(nodePath)) {
+ throw new YarnException("ZkNodePath = " + nodePath + " not exists!");
+ }
+
+ // Get the stored masterKey from zk.
+ RouterMasterKey routerMasterKey = getRouterMasterKeyFromZK(nodePath);
+ long end = clock.getTime();
+ opDurations.getMasterKeyByDelegationKeyDuration(start, end);
+ return RouterMasterKeyResponse.newInstance(routerMasterKey);
+ } catch (Exception e) {
+ throw new YarnException(e);
+ }
+ }
+
+ /**
+ * Get MasterKeyZNodePath based on DelegationKey.
+ *
+ * @param delegationKey delegationKey.
+ * @return masterKey ZNodePath.
+ */
+ private String getMasterKeyZNodePathByDelegationKey(DelegationKey delegationKey) {
+ return getMasterKeyZNodePathByKeyId(delegationKey.getKeyId());
+ }
+
+ /**
+ * Get MasterKeyZNodePath based on KeyId.
+ *
+ * @param keyId master key id.
+ * @return masterKey ZNodePath.
+ */
+ private String getMasterKeyZNodePathByKeyId(int keyId) {
+ String nodeName = ROUTER_RM_DELEGATION_KEY_PREFIX + keyId;
+ return getNodePath(routerRMDTMasterKeysRootPath, nodeName);
+ }
+
+ /**
+ * Get RouterMasterKey from ZK.
+ *
+ * @param nodePath The path where masterKey is stored in zk.
+ *
+ * @return RouterMasterKey.
+ * @throws IOException An IO Error occurred.
+ */
+ private RouterMasterKey getRouterMasterKeyFromZK(String nodePath)
+ throws IOException {
+ try {
+ byte[] data = get(nodePath);
+ if ((data == null) || (data.length == 0)) {
+ return null;
+ }
+
+ ByteArrayInputStream bin = new ByteArrayInputStream(data);
+ DataInputStream din = new DataInputStream(bin);
+ DelegationKey key = new DelegationKey();
+ key.readFields(din);
+
+ return RouterMasterKey.newInstance(key.getKeyId(),
+ ByteBuffer.wrap(key.getEncodedKey()), key.getExpiryDate());
+ } catch (Exception ex) {
+ LOG.error("No node in path {}.", nodePath);
+ throw new IOException(ex);
+ }
}
+ /**
+ * ZookeeperFederationStateStore Supports Store RMDelegationTokenIdentifier.
+ *
+ * The stored token method is a synchronized method
+ * used to ensure that storeNewToken is a thread-safe method.
+ *
+ * @param request The request contains RouterRMToken (RMDelegationTokenIdentifier and renewDate)
+ * @return routerRMTokenResponse, the response contains the RouterStoreToken.
+ * @throws YarnException if the call to the state store is unsuccessful.
+ * @throws IOException An IO Error occurred.
+ */
@Override
- public RouterRMTokenResponse storeNewToken(RouterRMTokenRequest request)
+ public synchronized RouterRMTokenResponse storeNewToken(RouterRMTokenRequest request)
Review Comment:
Thank you very much for your help in reviewing the code. In ZookeeperFederationStateStore, we have 7 methods that use the synchronized keyword
1. Group1 methods are mainly used to create or remove MasterKey
- storeNewMasterKey
- removeStoredMasterKey
2. Group2 methods are mainly used to create or update and remove Token
- storeNewToken
- updateStoredToken
- removeStoredToken
3. Group3 methods are used to increment DelegationTokenSeqNum and MasterKeyId
- incrementDelegationTokenSeqNum
- incrementCurrentKeyId
This part is mainly to protect the ZNode of ZK. We hope that the same Router has only one thread to write or update or delete the same ZNode at the same time.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-issues-help@hadoop.apache.org
[GitHub] [hadoop] hadoop-yetus commented on pull request #5131: YARN-11350. [Federation] Router Support DelegationToken With ZK.
Posted by GitBox <gi...@apache.org>.
hadoop-yetus commented on PR #5131:
URL: https://github.com/apache/hadoop/pull/5131#issuecomment-1325538547
:broken_heart: **-1 overall**
| Vote | Subsystem | Runtime | Logfile | Comment |
|:----:|----------:|--------:|:--------:|:-------:|
| +0 :ok: | reexec | 0m 49s | | Docker mode activated. |
|||| _ Prechecks _ |
| +1 :green_heart: | dupname | 0m 1s | | No case conflicting files found. |
| +0 :ok: | codespell | 0m 0s | | codespell was not available. |
| +0 :ok: | detsecrets | 0m 0s | | detect-secrets was not available. |
| +1 :green_heart: | @author | 0m 0s | | The patch does not contain any @author tags. |
| +1 :green_heart: | test4tests | 0m 0s | | The patch appears to include 1 new or modified test files. |
|||| _ trunk Compile Tests _ |
| +0 :ok: | mvndep | 16m 22s | | Maven dependency ordering for branch |
| +1 :green_heart: | mvninstall | 30m 20s | | trunk passed |
| +1 :green_heart: | compile | 5m 25s | | trunk passed with JDK Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 |
| +1 :green_heart: | compile | 4m 8s | | trunk passed with JDK Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | checkstyle | 1m 20s | | trunk passed |
| +1 :green_heart: | mvnsite | 2m 37s | | trunk passed |
| +1 :green_heart: | javadoc | 2m 37s | | trunk passed with JDK Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 |
| +1 :green_heart: | javadoc | 2m 31s | | trunk passed with JDK Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | spotbugs | 4m 51s | | trunk passed |
| +1 :green_heart: | shadedclient | 25m 38s | | branch has no errors when building and testing our client artifacts. |
| -0 :warning: | patch | 26m 6s | | Used diff version of patch file. Binary files and potentially other changes not applied. Please rebase and squash commits if necessary. |
|||| _ Patch Compile Tests _ |
| +0 :ok: | mvndep | 0m 34s | | Maven dependency ordering for patch |
| +1 :green_heart: | mvninstall | 2m 37s | | the patch passed |
| +1 :green_heart: | compile | 4m 14s | | the patch passed with JDK Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 |
| +1 :green_heart: | javac | 4m 14s | | the patch passed |
| +1 :green_heart: | compile | 3m 19s | | the patch passed with JDK Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | javac | 3m 19s | | the patch passed |
| +1 :green_heart: | blanks | 0m 0s | | The patch has no blanks issues. |
| +1 :green_heart: | checkstyle | 1m 7s | | the patch passed |
| +1 :green_heart: | mvnsite | 2m 23s | | the patch passed |
| -1 :x: | javadoc | 0m 35s | [/patch-javadoc-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-common-jdkUbuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/10/artifact/out/patch-javadoc-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-common-jdkUbuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04.txt) | hadoop-yarn-server-common in the patch failed with JDK Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04. |
| -1 :x: | javadoc | 0m 32s | [/patch-javadoc-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-common-jdkPrivateBuild-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/10/artifact/out/patch-javadoc-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-common-jdkPrivateBuild-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07.txt) | hadoop-yarn-server-common in the patch failed with JDK Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07. |
| -1 :x: | spotbugs | 1m 33s | [/new-spotbugs-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-common.html](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/10/artifact/out/new-spotbugs-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-common.html) | hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common generated 1 new + 0 unchanged - 0 fixed = 1 total (was 0) |
| +1 :green_heart: | shadedclient | 20m 26s | | patch has no errors when building and testing our client artifacts. |
|||| _ Other Tests _ |
| +1 :green_heart: | unit | 3m 15s | | hadoop-yarn-server-common in the patch passed. |
| +1 :green_heart: | unit | 99m 11s | | hadoop-yarn-server-resourcemanager in the patch passed. |
| +1 :green_heart: | unit | 0m 43s | | hadoop-yarn-server-router in the patch passed. |
| +1 :green_heart: | asflicense | 0m 50s | | The patch does not generate ASF License warnings. |
| | | 246m 9s | | |
| Reason | Tests |
|-------:|:------|
| SpotBugs | module:hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common |
| | org.apache.hadoop.yarn.server.federation.store.impl.MemoryFederationStateStore.getDelegationTokenSeqNum() is unsynchronized, org.apache.hadoop.yarn.server.federation.store.impl.MemoryFederationStateStore.setDelegationTokenSeqNum(int) is synchronized At MemoryFederationStateStore.java:synchronized At MemoryFederationStateStore.java:[line 549] |
| Subsystem | Report/Notes |
|----------:|:-------------|
| Docker | ClientAPI=1.41 ServerAPI=1.41 base: https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/10/artifact/out/Dockerfile |
| GITHUB PR | https://github.com/apache/hadoop/pull/5131 |
| Optional Tests | dupname asflicense compile javac javadoc mvninstall mvnsite unit shadedclient spotbugs checkstyle codespell detsecrets |
| uname | Linux 4fa2ae0b3fda 4.15.0-191-generic #202-Ubuntu SMP Thu Aug 4 01:49:29 UTC 2022 x86_64 x86_64 x86_64 GNU/Linux |
| Build tool | maven |
| Personality | dev-support/bin/hadoop.sh |
| git revision | trunk / 760f508a440be3ddcf7288d4d69a548992b11dd6 |
| Default Java | Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| Multi-JDK versions | /usr/lib/jvm/java-11-openjdk-amd64:Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 /usr/lib/jvm/java-8-openjdk-amd64:Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| Test Results | https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/10/testReport/ |
| Max. process+thread count | 978 (vs. ulimit of 5500) |
| modules | C: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router U: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server |
| Console output | https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/10/console |
| versions | git=2.25.1 maven=3.6.3 spotbugs=4.2.2 |
| Powered by | Apache Yetus 0.14.0 https://yetus.apache.org |
This message was automatically generated.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-issues-help@hadoop.apache.org
[GitHub] [hadoop] goiri commented on a diff in pull request #5131: YARN-11350. [Federation] Router Support DelegationToken With ZK.
Posted by GitBox <gi...@apache.org>.
goiri commented on code in PR #5131:
URL: https://github.com/apache/hadoop/pull/5131#discussion_r1036514318
##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterStoreToken.java:
##########
@@ -22,6 +22,7 @@
import org.apache.hadoop.yarn.security.client.YARNDelegationTokenIdentifier;
import org.apache.hadoop.yarn.util.Records;
+import java.io.DataInput;
Review Comment:
Shouldn't that be covered in the impl and not in the super class?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-issues-help@hadoop.apache.org
[GitHub] [hadoop] goiri commented on a diff in pull request #5131: YARN-11350. [Federation] Router Support DelegationToken With ZK.
Posted by GitBox <gi...@apache.org>.
goiri commented on code in PR #5131:
URL: https://github.com/apache/hadoop/pull/5131#discussion_r1035037274
##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterStoreToken.java:
##########
@@ -22,6 +22,7 @@
import org.apache.hadoop.yarn.security.client.YARNDelegationTokenIdentifier;
import org.apache.hadoop.yarn.util.Records;
+import java.io.DataInput;
Review Comment:
Why do we need this? We didn't add anything using DataInput to this class.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-issues-help@hadoop.apache.org
[GitHub] [hadoop] goiri commented on a diff in pull request #5131: YARN-11350. [Federation] Router Support DelegationToken With ZK.
Posted by GitBox <gi...@apache.org>.
goiri commented on code in PR #5131:
URL: https://github.com/apache/hadoop/pull/5131#discussion_r1039698060
##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/TestZookeeperFederationStateStore.java:
##########
@@ -171,38 +203,293 @@ public void testMetricsInited() throws Exception {
MetricsRecords.assertMetric(record, "UpdateReservationHomeSubClusterNumOps", expectOps);
}
- @Test(expected = NotImplementedException.class)
+ @Test
public void testStoreNewMasterKey() throws Exception {
- super.testStoreNewMasterKey();
+
+ // Manually create a DelegationKey,
+ // and call the interface storeNewMasterKey to write the data to zk.
+ DelegationKey key = new DelegationKey(1234, Time.now() + 60 * 60, "keyBytes".getBytes());
+ RouterMasterKey paramRouterMasterKey = RouterMasterKey.newInstance(key.getKeyId(),
+ ByteBuffer.wrap(key.getEncodedKey()), key.getExpiryDate());
+ FederationStateStore stateStore = this.getStateStore();
+
+ assertTrue(stateStore instanceof ZookeeperFederationStateStore);
+
+ // Compare the data returned by the storeNewMasterKey
+ // interface with the data queried by zk, and ensure that the data is consistent.
+ RouterMasterKeyRequest routerMasterKeyRequest =
+ RouterMasterKeyRequest.newInstance(paramRouterMasterKey);
+ RouterMasterKeyResponse response = stateStore.storeNewMasterKey(routerMasterKeyRequest);
+ assertNotNull(response);
+ RouterMasterKey respRouterMasterKey = response.getRouterMasterKey();
+ assertNotNull(respRouterMasterKey);
+
+ // Get Data From zk.
+ String nodeName = ROUTER_RM_DELEGATION_KEY_PREFIX + key.getKeyId();
+ String nodePath = ZNODE_MASTER_KEY_PREFIX + nodeName;
+ RouterMasterKey zkRouterMasterKey = getRouterMasterKeyFromZK(nodePath);
+
+ assertNotNull(zkRouterMasterKey);
+ assertEquals(paramRouterMasterKey, respRouterMasterKey);
+ assertEquals(paramRouterMasterKey, zkRouterMasterKey);
+ assertEquals(zkRouterMasterKey, respRouterMasterKey);
}
- @Test(expected = NotImplementedException.class)
+ @Test
public void testGetMasterKeyByDelegationKey() throws YarnException, IOException {
- super.testGetMasterKeyByDelegationKey();
+
+ // Manually create a DelegationKey,
+ // and call the interface storeNewMasterKey to write the data to zk.
+ DelegationKey key = new DelegationKey(5678, Time.now() + 60 * 60, "keyBytes".getBytes());
+ RouterMasterKey paramRouterMasterKey = RouterMasterKey.newInstance(key.getKeyId(),
+ ByteBuffer.wrap(key.getEncodedKey()), key.getExpiryDate());
+ FederationStateStore stateStore = this.getStateStore();
+
+ assertTrue(stateStore instanceof ZookeeperFederationStateStore);
+
+ // Call the getMasterKeyByDelegationKey interface of stateStore to get the MasterKey data.
+ RouterMasterKeyRequest routerMasterKeyRequest =
+ RouterMasterKeyRequest.newInstance(paramRouterMasterKey);
+ RouterMasterKeyResponse response = stateStore.storeNewMasterKey(routerMasterKeyRequest);
+ assertNotNull(response);
+
+ // Get Data From zk.
+ String nodeName = ROUTER_RM_DELEGATION_KEY_PREFIX + key.getKeyId();
+ String nodePath = ZNODE_MASTER_KEY_PREFIX + nodeName;
+ RouterMasterKey zkRouterMasterKey = getRouterMasterKeyFromZK(nodePath);
+
+ // Call the getMasterKeyByDelegationKey interface to get the returned result.
+ // The zk data should be consistent with the returned data.
+ RouterMasterKeyResponse response1 =
+ stateStore.getMasterKeyByDelegationKey(routerMasterKeyRequest);
+ assertNotNull(response1);
+ RouterMasterKey respRouterMasterKey = response1.getRouterMasterKey();
+ assertEquals(paramRouterMasterKey, respRouterMasterKey);
+ assertEquals(paramRouterMasterKey, zkRouterMasterKey);
+ assertEquals(zkRouterMasterKey, respRouterMasterKey);
}
- @Test(expected = NotImplementedException.class)
+ @Test
public void testRemoveStoredMasterKey() throws YarnException, IOException {
- super.testRemoveStoredMasterKey();
+
+ // Manually create a DelegationKey,
+ // and call the interface storeNewMasterKey to write the data to zk.
+ DelegationKey key = new DelegationKey(2345, Time.now() + 60 * 60, "keyBytes".getBytes());
+ RouterMasterKey paramRouterMasterKey = RouterMasterKey.newInstance(key.getKeyId(),
+ ByteBuffer.wrap(key.getEncodedKey()), key.getExpiryDate());
+ FederationStateStore stateStore = this.getStateStore();
+
+ assertTrue(stateStore instanceof ZookeeperFederationStateStore);
+
+ // We need to ensure that the returned result is not empty.
+ RouterMasterKeyRequest routerMasterKeyRequest =
+ RouterMasterKeyRequest.newInstance(paramRouterMasterKey);
+ RouterMasterKeyResponse response = stateStore.storeNewMasterKey(routerMasterKeyRequest);
+ assertNotNull(response);
+
+ // We will check if delegationToken exists in zk.
+ String nodeName = ROUTER_RM_DELEGATION_KEY_PREFIX + key.getKeyId();
+ String nodePath = ZNODE_MASTER_KEY_PREFIX + nodeName;
+ assertTrue(isExists(nodePath));
+
+ // Call removeStoredMasterKey to remove the MasterKey data in zk.
+ RouterMasterKeyResponse response1 = stateStore.removeStoredMasterKey(routerMasterKeyRequest);
+ assertNotNull(response1);
+ RouterMasterKey respRouterMasterKey = response1.getRouterMasterKey();
+ assertNotNull(respRouterMasterKey);
+ assertEquals(paramRouterMasterKey, respRouterMasterKey);
+
+ // We have removed the RouterMasterKey data from zk,
+ // the path should be empty at this point.
+ assertFalse(isExists(nodePath));
}
- @Test(expected = NotImplementedException.class)
- public void testStoreNewToken() throws IOException, YarnException {
- super.testStoreNewToken();
+ @Test
+ public void testStoreNewToken() throws YarnException, IOException {
+
+ // We manually generate the DelegationToken,
+ // and then call the StoreNewToken method to store the Token in zk.
+ RMDelegationTokenIdentifier identifier = new RMDelegationTokenIdentifier(
+ new Text("owner2"), new Text("renewer2"), new Text("realuser2"));
+ FederationStateStore stateStore = this.getStateStore();
+ int seqNum = stateStore.incrementDelegationTokenSeqNum();
+ identifier.setSequenceNumber(seqNum);
+ Long renewDate = Time.now();
+
+ // Store new rm-token
+ RouterStoreToken paramStoreToken = RouterStoreToken.newInstance(identifier, renewDate);
+ RouterRMTokenRequest request = RouterRMTokenRequest.newInstance(paramStoreToken);
+ RouterRMTokenResponse routerRMTokenResponse = stateStore.storeNewToken(request);
+ assertNotNull(routerRMTokenResponse);
+ RouterStoreToken respStoreToken = routerRMTokenResponse.getRouterStoreToken();
+ assertNotNull(respStoreToken);
+
+ // Get delegationToken Path
+ String nodeName = ROUTER_RM_DELEGATION_TOKEN_PREFIX + identifier.getSequenceNumber();
+ String nodePath = getNodePath(ZNODE_DT_PREFIX, nodeName);
+
+ // Check if the path exists, we expect the result to exist.
+ assertTrue(isExists(nodePath));
+
+ // Check whether the token (paramStoreToken)
+ // We generated is consistent with the data stored in zk.
+ // We expect data to be consistent.
+ RouterStoreToken zkRouterStoreToken = getStoreTokenFromZK(nodePath);
+ assertNotNull(zkRouterStoreToken);
+ assertEquals(paramStoreToken, zkRouterStoreToken);
+ assertEquals(respStoreToken, zkRouterStoreToken);
}
- @Test(expected = NotImplementedException.class)
- public void testUpdateStoredToken() throws IOException, YarnException {
- super.testUpdateStoredToken();
+ @Test
+ public void testUpdateStoredToken() throws YarnException, IOException {
+
+ // We manually generate the DelegationToken,
+ // and then call the StoreNewToken method to store the Token in zk.
+ RMDelegationTokenIdentifier identifier = new RMDelegationTokenIdentifier(
+ new Text("owner2"), new Text("renewer2"), new Text("realuser2"));
+ FederationStateStore stateStore = this.getStateStore();
+ int seqNum = stateStore.incrementDelegationTokenSeqNum();
+ identifier.setSequenceNumber(seqNum);
+ Long renewDate = Time.now();
+
+ // Store new rm-token()
+ RouterStoreToken storeToken = RouterStoreToken.newInstance(identifier, renewDate);
+ RouterRMTokenRequest request = RouterRMTokenRequest.newInstance(storeToken);
+ RouterRMTokenResponse routerRMTokenResponse = stateStore.storeNewToken(request);
+ Assert.assertNotNull(routerRMTokenResponse);
+
+ // We are ready to update some data renewDate2 & sequenceNumber2
+ Long renewDate2 = Time.now();
+ int sequenceNumber2 = stateStore.incrementDelegationTokenSeqNum();
+ identifier.setSequenceNumber(sequenceNumber2);
+
+ // Update rm-token
+ RouterStoreToken paramStoreToken = RouterStoreToken.newInstance(identifier, renewDate2);
+ RouterRMTokenRequest updateTokenRequest = RouterRMTokenRequest.newInstance(paramStoreToken);
+ RouterRMTokenResponse updateTokenResponse = stateStore.updateStoredToken(updateTokenRequest);
+ Assert.assertNotNull(updateTokenResponse);
+ RouterStoreToken updateTokenResp = updateTokenResponse.getRouterStoreToken();
+
+ // Get delegationToken Path
+ String nodeName = ROUTER_RM_DELEGATION_TOKEN_PREFIX + identifier.getSequenceNumber();
+ String nodePath = getNodePath(ZNODE_DT_PREFIX, nodeName);
+
+ // Check if the path exists, we expect the result to exist.
+ assertTrue(isExists(nodePath));
+
+ // Check whether the token (paramStoreToken)
+ // We generated is consistent with the data stored in zk.
+ // We expect data to be consistent.
+ RouterStoreToken zkRouterStoreToken = getStoreTokenFromZK(nodePath);
+ assertNotNull(zkRouterStoreToken);
+ assertEquals(paramStoreToken, zkRouterStoreToken);
+ assertEquals(updateTokenResp, zkRouterStoreToken);
}
- @Test(expected = NotImplementedException.class)
- public void testRemoveStoredToken() throws IOException, YarnException {
- super.testRemoveStoredToken();
+ @Test
+ public void testRemoveStoredToken() throws YarnException, IOException {
+
+ // We manually generate the DelegationToken,
+ // and then call the StoreNewToken method to store the Token in zk.
+ RMDelegationTokenIdentifier identifier = new RMDelegationTokenIdentifier(
+ new Text("owner2"), new Text("renewer2"), new Text("realuser2"));
+ FederationStateStore stateStore = this.getStateStore();
+ int seqNum = stateStore.incrementDelegationTokenSeqNum();
+ identifier.setSequenceNumber(seqNum);
+ Long renewDate = Time.now();
+
+ // Store new rm-token
+ RouterStoreToken storeToken = RouterStoreToken.newInstance(identifier, renewDate);
+ RouterRMTokenRequest request = RouterRMTokenRequest.newInstance(storeToken);
+ RouterRMTokenResponse routerRMTokenResponse = stateStore.storeNewToken(request);
+ Assert.assertNotNull(routerRMTokenResponse);
Review Comment:
As you statically imported assertTrue and others, import this too.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-issues-help@hadoop.apache.org
[GitHub] [hadoop] Hexiaoqiao commented on pull request #5131: YARN-11350. [Federation] Router Support DelegationToken With ZK.
Posted by GitBox <gi...@apache.org>.
Hexiaoqiao commented on PR #5131:
URL: https://github.com/apache/hadoop/pull/5131#issuecomment-1317074076
Thanks @slfan1989 , Great work here!
One concern out of this PR, we have HDFS RBF / YARN Federation / KMS both depend on zookeeper to store DelegationToken. From my view, both of their logic are similar, we should consider implement one common module about ZKDelegationTokenManager? Thus it will reduce overhead to implement at each sub-project side. WDYT? cc @goiri
NOTE: This is not blocker comments. Thanks.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-issues-help@hadoop.apache.org
[GitHub] [hadoop] slfan1989 commented on a diff in pull request #5131: YARN-11350. [Federation] Router Support DelegationToken With ZK.
Posted by GitBox <gi...@apache.org>.
slfan1989 commented on code in PR #5131:
URL: https://github.com/apache/hadoop/pull/5131#discussion_r1023521160
##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/ZookeeperFederationStateStore.java:
##########
@@ -174,21 +219,64 @@ public void init(Configuration conf) throws YarnException {
policiesZNode = getNodePath(baseZNode, ROOT_ZNODE_NAME_POLICY);
reservationsZNode = getNodePath(baseZNode, ROOT_ZNODE_NAME_RESERVATION);
+ // delegation token znodes
+ routerRMDTSecretManagerRoot = getNodePath(baseZNode, ROUTER_RM_DT_SECRET_MANAGER_ROOT);
+ routerRMDTMasterKeysRootPath = getNodePath(routerRMDTSecretManagerRoot,
+ ROUTER_RM_DT_MASTER_KEYS_ROOT_ZNODE_NAME);
+ routerRMDelegationTokensRootPath = getNodePath(routerRMDTSecretManagerRoot,
+ ROUTER_RM_DELEGATION_TOKENS_ROOT_ZNODE_NAME);
+ routerRMSequenceNumberPath = getNodePath(routerRMDTSecretManagerRoot,
+ ROUTER_RM_DT_SEQUENTIAL_NUMBER_ZNODE_NAME);
+
// Create base znode for each entity
try {
List<ACL> zkAcl = ZKCuratorManager.getZKAcls(conf);
zkManager.createRootDirRecursively(membershipZNode, zkAcl);
zkManager.createRootDirRecursively(appsZNode, zkAcl);
zkManager.createRootDirRecursively(policiesZNode, zkAcl);
zkManager.createRootDirRecursively(reservationsZNode, zkAcl);
+ zkManager.createRootDirRecursively(routerRMDTSecretManagerRoot, zkAcl);
+ zkManager.createRootDirRecursively(routerRMDTMasterKeysRootPath, zkAcl);
+ zkManager.createRootDirRecursively(routerRMDelegationTokensRootPath, zkAcl);
} catch (Exception e) {
String errMsg = "Cannot create base directories: " + e.getMessage();
FederationStateStoreUtils.logAndThrowStoreException(LOG, errMsg);
}
+
+ // Distributed sequenceNum.
+ try {
+ seqNumBatchSize = conf.getInt(ZK_DTSM_TOKEN_SEQNUM_BATCH_SIZE,
+ ZK_DTSM_TOKEN_SEQNUM_BATCH_SIZE_DEFAULT);
+
+ delTokSeqCounter = new SharedCount(zkManager.getCurator(), routerRMSequenceNumberPath, 0);
+
+ if (delTokSeqCounter != null) {
+ delTokSeqCounter.start();
+ }
+
+ // the first batch range should be allocated during this starting window
+ // by calling the incrSharedCount
+ currentSeqNum = incrSharedCount(delTokSeqCounter, seqNumBatchSize);
+ currentMaxSeqNum = currentSeqNum + seqNumBatchSize;
+
+ LOG.info("Fetched initial range of seq num, from {} to {} ",
+ currentSeqNum + 1, currentMaxSeqNum);
+ } catch (Exception e) {
+ throw new YarnException("Could not start Sequence Counter.", e);
+ }
}
@Override
public void close() throws Exception {
+
+ try {
+ if (delTokSeqCounter != null) {
+ delTokSeqCounter.close();
Review Comment:
Thanks for your suggestion, I will modify the code.
##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/ZookeeperFederationStateStore.java:
##########
@@ -886,45 +974,558 @@ public UpdateReservationHomeSubClusterResponse updateReservationHomeSubCluster(
return UpdateReservationHomeSubClusterResponse.newInstance();
}
+ /**
+ * ZookeeperFederationStateStore Supports Store NewMasterKey.
+ *
+ * @param request The request contains RouterMasterKey, which is an abstraction for DelegationKey
+ * @return routerMasterKeyResponse, the response contains the RouterMasterKey.
+ * @throws YarnException if the call to the state store is unsuccessful.
+ * @throws IOException An IO Error occurred.
+ */
@Override
- public RouterMasterKeyResponse storeNewMasterKey(RouterMasterKeyRequest request)
+ public synchronized RouterMasterKeyResponse storeNewMasterKey(RouterMasterKeyRequest request)
throws YarnException, IOException {
- throw new NotImplementedException("Code is not implemented");
+
+ long start = clock.getTime();
+ // For the verification of the request, after passing the verification,
+ // the request and the internal objects will not be empty and can be used directly.
+ FederationRouterRMTokenInputValidator.validate(request);
+
+ // Parse the delegationKey from the request and get the ZK storage path.
+ DelegationKey delegationKey = convertMasterKeyToDelegationKey(request);
+ String nodeCreatePath = getMasterKeyZNodePathByDelegationKey(delegationKey);
+ LOG.debug("Storing RMDelegationKey_{}, ZkNodePath = {}.", delegationKey.getKeyId(),
+ nodeCreatePath);
+
+ // Write master key data to zk.
+ try(ByteArrayOutputStream os = new ByteArrayOutputStream();
+ DataOutputStream fsOut = new DataOutputStream(os)) {
+ delegationKey.write(fsOut);
+ put(nodeCreatePath, os.toByteArray(), false);
+ }
+
+ // Get the stored masterKey from zk.
+ RouterMasterKey masterKeyFromZK = getRouterMasterKeyFromZK(nodeCreatePath, false);
+ long end = clock.getTime();
+ opDurations.addStoreNewMasterKeyDuration(start, end);
+ return RouterMasterKeyResponse.newInstance(masterKeyFromZK);
}
+ /**
+ * ZookeeperFederationStateStore Supports Remove MasterKey.
+ *
+ * @param request The request contains RouterMasterKey, which is an abstraction for DelegationKey
+ * @return routerMasterKeyResponse, the response contains the RouterMasterKey.
+ * @throws YarnException if the call to the state store is unsuccessful.
+ * @throws IOException An IO Error occurred.
+ */
@Override
- public RouterMasterKeyResponse removeStoredMasterKey(RouterMasterKeyRequest request)
+ public synchronized RouterMasterKeyResponse removeStoredMasterKey(RouterMasterKeyRequest request)
throws YarnException, IOException {
- throw new NotImplementedException("Code is not implemented");
+
+ long start = clock.getTime();
+ // For the verification of the request, after passing the verification,
+ // the request and the internal objects will not be empty and can be used directly.
+ FederationRouterRMTokenInputValidator.validate(request);
+
+ try {
+ // Parse the delegationKey from the request and get the ZK storage path.
+ RouterMasterKey masterKey = request.getRouterMasterKey();
+ DelegationKey delegationKey = convertMasterKeyToDelegationKey(request);
+ String nodeRemovePath = getMasterKeyZNodePathByDelegationKey(delegationKey);
+ LOG.debug("Removing RMDelegationKey_{}, ZkNodePath = {}.", delegationKey.getKeyId(),
+ nodeRemovePath);
+
+ // Check if the path exists, Throws an exception if the path does not exist.
+ if (!exists(nodeRemovePath)) {
+ throw new YarnException("ZkNodePath = " + nodeRemovePath + " not exists!");
+ }
+
+ // try to remove masterKey.
+ zkManager.delete(nodeRemovePath);
+ long end = clock.getTime();
+ opDurations.removeStoredMasterKeyDuration(start, end);
+ return RouterMasterKeyResponse.newInstance(masterKey);
+ } catch (Exception e) {
+ throw new YarnException(e);
+ }
}
+ /**
+ * ZookeeperFederationStateStore Supports Remove MasterKey.
+ *
+ * @param request The request contains RouterMasterKey, which is an abstraction for DelegationKey
+ * @return routerMasterKeyResponse, the response contains the RouterMasterKey.
+ * @throws YarnException if the call to the state store is unsuccessful.
+ * @throws IOException An IO Error occurred.
+ */
@Override
public RouterMasterKeyResponse getMasterKeyByDelegationKey(RouterMasterKeyRequest request)
throws YarnException, IOException {
- throw new NotImplementedException("Code is not implemented");
+
+ long start = clock.getTime();
+ // For the verification of the request, after passing the verification,
+ // the request and the internal objects will not be empty and can be used directly.
+ FederationRouterRMTokenInputValidator.validate(request);
+
+ try {
+
+ // Parse the delegationKey from the request and get the ZK storage path.
+ DelegationKey delegationKey = convertMasterKeyToDelegationKey(request);
+ String nodePath = getMasterKeyZNodePathByDelegationKey(delegationKey);
+
+ // Check if the path exists, Throws an exception if the path does not exist.
+ if (!exists(nodePath)) {
+ throw new YarnException("ZkNodePath = " + nodePath + " not exists!");
+ }
+
+ // Get the stored masterKey from zk.
+ RouterMasterKey routerMasterKey = getRouterMasterKeyFromZK(nodePath, false);
+ long end = clock.getTime();
+ opDurations.getMasterKeyByDelegationKeyDuration(start, end);
+ return RouterMasterKeyResponse.newInstance(routerMasterKey);
+ } catch (Exception e) {
+ throw new YarnException(e);
+ }
+ }
+
+ /**
+ * Get MasterKeyZNodePath based on DelegationKey.
+ *
+ * @param delegationKey delegationKey.
+ * @return masterKey ZNodePath.
+ */
+ private String getMasterKeyZNodePathByDelegationKey(DelegationKey delegationKey) {
+ return getMasterKeyZNodePathByKeyId(delegationKey.getKeyId());
+ }
+
+ /**
+ * Get MasterKeyZNodePath based on KeyId.
+ *
+ * @param keyId master key id.
+ * @return masterKey ZNodePath.
+ */
+ private String getMasterKeyZNodePathByKeyId(int keyId) {
+ String nodeName = ROUTER_RM_DELEGATION_KEY_PREFIX + keyId;
+ return getNodePath(routerRMDTMasterKeysRootPath, nodeName);
}
+ /**
+ * Get RouterMasterKey from ZK.
+ *
+ * @param nodePath The path where masterKey is stored in zk.
+ * @param quiet If true is silent mode, no error message is printed at this time,
+ * if false is non-silent mode, error message is printed at this time.
+ *
+ * @return
+ * @throws IOException
+ */
+ private RouterMasterKey getRouterMasterKeyFromZK(String nodePath, boolean quiet)
+ throws IOException {
+ try {
+ byte[] data = get(nodePath);
+ if ((data == null) || (data.length == 0)) {
+ return null;
+ }
+
+ ByteArrayInputStream bin = new ByteArrayInputStream(data);
+ DataInputStream din = new DataInputStream(bin);
+ DelegationKey key = new DelegationKey();
+ key.readFields(din);
+
+ return RouterMasterKey.newInstance(key.getKeyId(),
+ ByteBuffer.wrap(key.getEncodedKey()), key.getExpiryDate());
+ } catch (Exception ex) {
+ if (!quiet) {
+ LOG.error("No node in path [" + nodePath + "]");
Review Comment:
I will fix it.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-issues-help@hadoop.apache.org
[GitHub] [hadoop] slfan1989 commented on a diff in pull request #5131: YARN-11350. [Federation] Router Support DelegationToken With ZK.
Posted by GitBox <gi...@apache.org>.
slfan1989 commented on code in PR #5131:
URL: https://github.com/apache/hadoop/pull/5131#discussion_r1023964038
##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/ZookeeperFederationStateStore.java:
##########
@@ -886,45 +974,558 @@ public UpdateReservationHomeSubClusterResponse updateReservationHomeSubCluster(
return UpdateReservationHomeSubClusterResponse.newInstance();
}
+ /**
+ * ZookeeperFederationStateStore Supports Store NewMasterKey.
+ *
+ * @param request The request contains RouterMasterKey, which is an abstraction for DelegationKey
+ * @return routerMasterKeyResponse, the response contains the RouterMasterKey.
+ * @throws YarnException if the call to the state store is unsuccessful.
+ * @throws IOException An IO Error occurred.
+ */
@Override
- public RouterMasterKeyResponse storeNewMasterKey(RouterMasterKeyRequest request)
+ public synchronized RouterMasterKeyResponse storeNewMasterKey(RouterMasterKeyRequest request)
throws YarnException, IOException {
- throw new NotImplementedException("Code is not implemented");
+
+ long start = clock.getTime();
+ // For the verification of the request, after passing the verification,
+ // the request and the internal objects will not be empty and can be used directly.
+ FederationRouterRMTokenInputValidator.validate(request);
+
+ // Parse the delegationKey from the request and get the ZK storage path.
+ DelegationKey delegationKey = convertMasterKeyToDelegationKey(request);
+ String nodeCreatePath = getMasterKeyZNodePathByDelegationKey(delegationKey);
+ LOG.debug("Storing RMDelegationKey_{}, ZkNodePath = {}.", delegationKey.getKeyId(),
+ nodeCreatePath);
+
+ // Write master key data to zk.
+ try(ByteArrayOutputStream os = new ByteArrayOutputStream();
+ DataOutputStream fsOut = new DataOutputStream(os)) {
+ delegationKey.write(fsOut);
+ put(nodeCreatePath, os.toByteArray(), false);
+ }
+
+ // Get the stored masterKey from zk.
+ RouterMasterKey masterKeyFromZK = getRouterMasterKeyFromZK(nodeCreatePath, false);
+ long end = clock.getTime();
+ opDurations.addStoreNewMasterKeyDuration(start, end);
+ return RouterMasterKeyResponse.newInstance(masterKeyFromZK);
}
+ /**
+ * ZookeeperFederationStateStore Supports Remove MasterKey.
+ *
+ * @param request The request contains RouterMasterKey, which is an abstraction for DelegationKey
+ * @return routerMasterKeyResponse, the response contains the RouterMasterKey.
+ * @throws YarnException if the call to the state store is unsuccessful.
+ * @throws IOException An IO Error occurred.
+ */
@Override
- public RouterMasterKeyResponse removeStoredMasterKey(RouterMasterKeyRequest request)
+ public synchronized RouterMasterKeyResponse removeStoredMasterKey(RouterMasterKeyRequest request)
throws YarnException, IOException {
- throw new NotImplementedException("Code is not implemented");
+
+ long start = clock.getTime();
+ // For the verification of the request, after passing the verification,
+ // the request and the internal objects will not be empty and can be used directly.
+ FederationRouterRMTokenInputValidator.validate(request);
+
+ try {
+ // Parse the delegationKey from the request and get the ZK storage path.
+ RouterMasterKey masterKey = request.getRouterMasterKey();
+ DelegationKey delegationKey = convertMasterKeyToDelegationKey(request);
+ String nodeRemovePath = getMasterKeyZNodePathByDelegationKey(delegationKey);
+ LOG.debug("Removing RMDelegationKey_{}, ZkNodePath = {}.", delegationKey.getKeyId(),
+ nodeRemovePath);
+
+ // Check if the path exists, Throws an exception if the path does not exist.
+ if (!exists(nodeRemovePath)) {
+ throw new YarnException("ZkNodePath = " + nodeRemovePath + " not exists!");
+ }
+
+ // try to remove masterKey.
+ zkManager.delete(nodeRemovePath);
+ long end = clock.getTime();
+ opDurations.removeStoredMasterKeyDuration(start, end);
+ return RouterMasterKeyResponse.newInstance(masterKey);
+ } catch (Exception e) {
+ throw new YarnException(e);
+ }
}
+ /**
+ * ZookeeperFederationStateStore Supports Remove MasterKey.
+ *
+ * @param request The request contains RouterMasterKey, which is an abstraction for DelegationKey
+ * @return routerMasterKeyResponse, the response contains the RouterMasterKey.
+ * @throws YarnException if the call to the state store is unsuccessful.
+ * @throws IOException An IO Error occurred.
+ */
@Override
public RouterMasterKeyResponse getMasterKeyByDelegationKey(RouterMasterKeyRequest request)
throws YarnException, IOException {
- throw new NotImplementedException("Code is not implemented");
+
+ long start = clock.getTime();
+ // For the verification of the request, after passing the verification,
+ // the request and the internal objects will not be empty and can be used directly.
+ FederationRouterRMTokenInputValidator.validate(request);
+
+ try {
+
+ // Parse the delegationKey from the request and get the ZK storage path.
+ DelegationKey delegationKey = convertMasterKeyToDelegationKey(request);
+ String nodePath = getMasterKeyZNodePathByDelegationKey(delegationKey);
+
+ // Check if the path exists, Throws an exception if the path does not exist.
+ if (!exists(nodePath)) {
+ throw new YarnException("ZkNodePath = " + nodePath + " not exists!");
+ }
+
+ // Get the stored masterKey from zk.
+ RouterMasterKey routerMasterKey = getRouterMasterKeyFromZK(nodePath, false);
+ long end = clock.getTime();
+ opDurations.getMasterKeyByDelegationKeyDuration(start, end);
+ return RouterMasterKeyResponse.newInstance(routerMasterKey);
+ } catch (Exception e) {
+ throw new YarnException(e);
+ }
+ }
+
+ /**
+ * Get MasterKeyZNodePath based on DelegationKey.
+ *
+ * @param delegationKey delegationKey.
+ * @return masterKey ZNodePath.
+ */
+ private String getMasterKeyZNodePathByDelegationKey(DelegationKey delegationKey) {
+ return getMasterKeyZNodePathByKeyId(delegationKey.getKeyId());
+ }
+
+ /**
+ * Get MasterKeyZNodePath based on KeyId.
+ *
+ * @param keyId master key id.
+ * @return masterKey ZNodePath.
+ */
+ private String getMasterKeyZNodePathByKeyId(int keyId) {
+ String nodeName = ROUTER_RM_DELEGATION_KEY_PREFIX + keyId;
+ return getNodePath(routerRMDTMasterKeysRootPath, nodeName);
}
+ /**
+ * Get RouterMasterKey from ZK.
+ *
+ * @param nodePath The path where masterKey is stored in zk.
+ * @param quiet If true is silent mode, no error message is printed at this time,
+ * if false is non-silent mode, error message is printed at this time.
+ *
+ * @return
+ * @throws IOException
+ */
+ private RouterMasterKey getRouterMasterKeyFromZK(String nodePath, boolean quiet)
Review Comment:
I agree with you, I will remove the quiet option.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-issues-help@hadoop.apache.org
[GitHub] [hadoop] slfan1989 commented on a diff in pull request #5131: YARN-11350. [Federation] Router Support DelegationToken With ZK.
Posted by GitBox <gi...@apache.org>.
slfan1989 commented on code in PR #5131:
URL: https://github.com/apache/hadoop/pull/5131#discussion_r1029962161
##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/FederationStateStoreBaseTest.java:
##########
@@ -905,7 +905,7 @@ public void testGetMasterKeyByDelegationKey() throws YarnException, IOException
}
@Test
- public void testRemoveStoredMasterKey() throws YarnException, IOException {
+ public void testRemoveStoredMasterKey() throws IOException, YarnException {
Review Comment:
Thank you so much for reviewing the code, I will fix it.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-issues-help@hadoop.apache.org
[GitHub] [hadoop] hadoop-yetus commented on pull request #5131: YARN-11350. [Federation] Router Support DelegationToken With ZK.
Posted by GitBox <gi...@apache.org>.
hadoop-yetus commented on PR #5131:
URL: https://github.com/apache/hadoop/pull/5131#issuecomment-1325957793
:broken_heart: **-1 overall**
| Vote | Subsystem | Runtime | Logfile | Comment |
|:----:|----------:|--------:|:--------:|:-------:|
| +0 :ok: | reexec | 0m 48s | | Docker mode activated. |
|||| _ Prechecks _ |
| +1 :green_heart: | dupname | 0m 1s | | No case conflicting files found. |
| +0 :ok: | codespell | 0m 0s | | codespell was not available. |
| +0 :ok: | detsecrets | 0m 0s | | detect-secrets was not available. |
| +1 :green_heart: | @author | 0m 0s | | The patch does not contain any @author tags. |
| +1 :green_heart: | test4tests | 0m 0s | | The patch appears to include 1 new or modified test files. |
|||| _ trunk Compile Tests _ |
| +0 :ok: | mvndep | 15m 56s | | Maven dependency ordering for branch |
| +1 :green_heart: | mvninstall | 26m 1s | | trunk passed |
| +1 :green_heart: | compile | 4m 2s | | trunk passed with JDK Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 |
| +1 :green_heart: | compile | 3m 34s | | trunk passed with JDK Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | checkstyle | 1m 24s | | trunk passed |
| +1 :green_heart: | mvnsite | 2m 50s | | trunk passed |
| +1 :green_heart: | javadoc | 2m 34s | | trunk passed with JDK Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 |
| +1 :green_heart: | javadoc | 2m 16s | | trunk passed with JDK Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | spotbugs | 4m 48s | | trunk passed |
| +1 :green_heart: | shadedclient | 20m 42s | | branch has no errors when building and testing our client artifacts. |
| -0 :warning: | patch | 21m 8s | | Used diff version of patch file. Binary files and potentially other changes not applied. Please rebase and squash commits if necessary. |
|||| _ Patch Compile Tests _ |
| +0 :ok: | mvndep | 0m 26s | | Maven dependency ordering for patch |
| +1 :green_heart: | mvninstall | 2m 6s | | the patch passed |
| +1 :green_heart: | compile | 3m 53s | | the patch passed with JDK Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 |
| +1 :green_heart: | javac | 3m 53s | | the patch passed |
| +1 :green_heart: | compile | 3m 14s | | the patch passed with JDK Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | javac | 3m 14s | | the patch passed |
| +1 :green_heart: | blanks | 0m 0s | | The patch has no blanks issues. |
| +1 :green_heart: | checkstyle | 1m 8s | | the patch passed |
| +1 :green_heart: | mvnsite | 2m 12s | | the patch passed |
| -1 :x: | javadoc | 0m 35s | [/patch-javadoc-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-common-jdkUbuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/12/artifact/out/patch-javadoc-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-common-jdkUbuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04.txt) | hadoop-yarn-server-common in the patch failed with JDK Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04. |
| -1 :x: | javadoc | 0m 37s | [/patch-javadoc-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-common-jdkPrivateBuild-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/12/artifact/out/patch-javadoc-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-common-jdkPrivateBuild-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07.txt) | hadoop-yarn-server-common in the patch failed with JDK Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07. |
| +1 :green_heart: | spotbugs | 4m 32s | | the patch passed |
| +1 :green_heart: | shadedclient | 20m 27s | | patch has no errors when building and testing our client artifacts. |
|||| _ Other Tests _ |
| +1 :green_heart: | unit | 3m 14s | | hadoop-yarn-server-common in the patch passed. |
| +1 :green_heart: | unit | 99m 7s | | hadoop-yarn-server-resourcemanager in the patch passed. |
| +1 :green_heart: | unit | 0m 46s | | hadoop-yarn-server-router in the patch passed. |
| +1 :green_heart: | asflicense | 0m 47s | | The patch does not generate ASF License warnings. |
| | | 233m 13s | | |
| Subsystem | Report/Notes |
|----------:|:-------------|
| Docker | ClientAPI=1.41 ServerAPI=1.41 base: https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/12/artifact/out/Dockerfile |
| GITHUB PR | https://github.com/apache/hadoop/pull/5131 |
| Optional Tests | dupname asflicense compile javac javadoc mvninstall mvnsite unit shadedclient spotbugs checkstyle codespell detsecrets |
| uname | Linux e876a1e203d6 4.15.0-191-generic #202-Ubuntu SMP Thu Aug 4 01:49:29 UTC 2022 x86_64 x86_64 x86_64 GNU/Linux |
| Build tool | maven |
| Personality | dev-support/bin/hadoop.sh |
| git revision | trunk / c4e7dc7024af5293b6b0684bde7b7dcefe9272b2 |
| Default Java | Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| Multi-JDK versions | /usr/lib/jvm/java-11-openjdk-amd64:Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 /usr/lib/jvm/java-8-openjdk-amd64:Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| Test Results | https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/12/testReport/ |
| Max. process+thread count | 1003 (vs. ulimit of 5500) |
| modules | C: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router U: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server |
| Console output | https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/12/console |
| versions | git=2.25.1 maven=3.6.3 spotbugs=4.2.2 |
| Powered by | Apache Yetus 0.14.0 https://yetus.apache.org |
This message was automatically generated.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-issues-help@hadoop.apache.org
[GitHub] [hadoop] slfan1989 commented on a diff in pull request #5131: YARN-11350. [Federation] Router Support DelegationToken With ZK.
Posted by GitBox <gi...@apache.org>.
slfan1989 commented on code in PR #5131:
URL: https://github.com/apache/hadoop/pull/5131#discussion_r1034873671
##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/ZookeeperFederationStateStore.java:
##########
@@ -886,45 +975,554 @@ public UpdateReservationHomeSubClusterResponse updateReservationHomeSubCluster(
return UpdateReservationHomeSubClusterResponse.newInstance();
}
+ /**
+ * ZookeeperFederationStateStore Supports Store NewMasterKey.
+ *
+ * @param request The request contains RouterMasterKey, which is an abstraction for DelegationKey
+ * @return routerMasterKeyResponse, the response contains the RouterMasterKey.
+ * @throws YarnException if the call to the state store is unsuccessful.
+ * @throws IOException An IO Error occurred.
+ */
@Override
- public RouterMasterKeyResponse storeNewMasterKey(RouterMasterKeyRequest request)
+ public synchronized RouterMasterKeyResponse storeNewMasterKey(RouterMasterKeyRequest request)
throws YarnException, IOException {
- throw new NotImplementedException("Code is not implemented");
+
+ long start = clock.getTime();
+ // For the verification of the request, after passing the verification,
+ // the request and the internal objects will not be empty and can be used directly.
+ FederationRouterRMTokenInputValidator.validate(request);
+
+ // Parse the delegationKey from the request and get the ZK storage path.
+ DelegationKey delegationKey = convertMasterKeyToDelegationKey(request);
+ String nodeCreatePath = getMasterKeyZNodePathByDelegationKey(delegationKey);
+ LOG.debug("Storing RMDelegationKey_{}, ZkNodePath = {}.", delegationKey.getKeyId(),
+ nodeCreatePath);
+
+ // Write master key data to zk.
+ try(ByteArrayOutputStream os = new ByteArrayOutputStream();
+ DataOutputStream fsOut = new DataOutputStream(os)) {
+ delegationKey.write(fsOut);
+ put(nodeCreatePath, os.toByteArray(), false);
+ }
+
+ // Get the stored masterKey from zk.
+ RouterMasterKey masterKeyFromZK = getRouterMasterKeyFromZK(nodeCreatePath);
+ long end = clock.getTime();
+ opDurations.addStoreNewMasterKeyDuration(start, end);
+ return RouterMasterKeyResponse.newInstance(masterKeyFromZK);
}
+ /**
+ * ZookeeperFederationStateStore Supports Remove MasterKey.
+ *
+ * @param request The request contains RouterMasterKey, which is an abstraction for DelegationKey
+ * @return routerMasterKeyResponse, the response contains the RouterMasterKey.
+ * @throws YarnException if the call to the state store is unsuccessful.
+ * @throws IOException An IO Error occurred.
+ */
@Override
- public RouterMasterKeyResponse removeStoredMasterKey(RouterMasterKeyRequest request)
+ public synchronized RouterMasterKeyResponse removeStoredMasterKey(RouterMasterKeyRequest request)
throws YarnException, IOException {
- throw new NotImplementedException("Code is not implemented");
+
+ long start = clock.getTime();
+ // For the verification of the request, after passing the verification,
+ // the request and the internal objects will not be empty and can be used directly.
+ FederationRouterRMTokenInputValidator.validate(request);
+
+ try {
+ // Parse the delegationKey from the request and get the ZK storage path.
+ RouterMasterKey masterKey = request.getRouterMasterKey();
+ DelegationKey delegationKey = convertMasterKeyToDelegationKey(request);
+ String nodeRemovePath = getMasterKeyZNodePathByDelegationKey(delegationKey);
+ LOG.debug("Removing RMDelegationKey_{}, ZkNodePath = {}.", delegationKey.getKeyId(),
+ nodeRemovePath);
+
+ // Check if the path exists, Throws an exception if the path does not exist.
+ if (!exists(nodeRemovePath)) {
+ throw new YarnException("ZkNodePath = " + nodeRemovePath + " not exists!");
+ }
+
+ // try to remove masterKey.
+ zkManager.delete(nodeRemovePath);
+ long end = clock.getTime();
+ opDurations.removeStoredMasterKeyDuration(start, end);
+ return RouterMasterKeyResponse.newInstance(masterKey);
+ } catch (Exception e) {
+ throw new YarnException(e);
+ }
}
+ /**
+ * ZookeeperFederationStateStore Supports Remove MasterKey.
+ *
+ * @param request The request contains RouterMasterKey, which is an abstraction for DelegationKey
+ * @return routerMasterKeyResponse, the response contains the RouterMasterKey.
+ * @throws YarnException if the call to the state store is unsuccessful.
+ * @throws IOException An IO Error occurred.
+ */
@Override
public RouterMasterKeyResponse getMasterKeyByDelegationKey(RouterMasterKeyRequest request)
throws YarnException, IOException {
- throw new NotImplementedException("Code is not implemented");
+
+ long start = clock.getTime();
+ // For the verification of the request, after passing the verification,
+ // the request and the internal objects will not be empty and can be used directly.
+ FederationRouterRMTokenInputValidator.validate(request);
+
+ try {
+
+ // Parse the delegationKey from the request and get the ZK storage path.
+ DelegationKey delegationKey = convertMasterKeyToDelegationKey(request);
+ String nodePath = getMasterKeyZNodePathByDelegationKey(delegationKey);
+
+ // Check if the path exists, Throws an exception if the path does not exist.
+ if (!exists(nodePath)) {
+ throw new YarnException("ZkNodePath = " + nodePath + " not exists!");
+ }
+
+ // Get the stored masterKey from zk.
+ RouterMasterKey routerMasterKey = getRouterMasterKeyFromZK(nodePath);
+ long end = clock.getTime();
+ opDurations.getMasterKeyByDelegationKeyDuration(start, end);
+ return RouterMasterKeyResponse.newInstance(routerMasterKey);
+ } catch (Exception e) {
+ throw new YarnException(e);
+ }
+ }
+
+ /**
+ * Get MasterKeyZNodePath based on DelegationKey.
+ *
+ * @param delegationKey delegationKey.
+ * @return masterKey ZNodePath.
+ */
+ private String getMasterKeyZNodePathByDelegationKey(DelegationKey delegationKey) {
+ return getMasterKeyZNodePathByKeyId(delegationKey.getKeyId());
+ }
+
+ /**
+ * Get MasterKeyZNodePath based on KeyId.
+ *
+ * @param keyId master key id.
+ * @return masterKey ZNodePath.
+ */
+ private String getMasterKeyZNodePathByKeyId(int keyId) {
+ String nodeName = ROUTER_RM_DELEGATION_KEY_PREFIX + keyId;
+ return getNodePath(routerRMDTMasterKeysRootPath, nodeName);
+ }
+
+ /**
+ * Get RouterMasterKey from ZK.
+ *
+ * @param nodePath The path where masterKey is stored in zk.
+ *
+ * @return RouterMasterKey.
+ * @throws IOException An IO Error occurred.
+ */
+ private RouterMasterKey getRouterMasterKeyFromZK(String nodePath)
+ throws IOException {
+ try {
+ byte[] data = get(nodePath);
+ if ((data == null) || (data.length == 0)) {
+ return null;
+ }
+
+ ByteArrayInputStream bin = new ByteArrayInputStream(data);
+ DataInputStream din = new DataInputStream(bin);
+ DelegationKey key = new DelegationKey();
+ key.readFields(din);
+
+ return RouterMasterKey.newInstance(key.getKeyId(),
+ ByteBuffer.wrap(key.getEncodedKey()), key.getExpiryDate());
+ } catch (Exception ex) {
+ LOG.error("No node in path {}.", nodePath);
+ throw new IOException(ex);
+ }
}
+ /**
+ * ZookeeperFederationStateStore Supports Store RMDelegationTokenIdentifier.
+ *
+ * The stored token method is a synchronized method
+ * used to ensure that storeNewToken is a thread-safe method.
+ *
+ * @param request The request contains RouterRMToken (RMDelegationTokenIdentifier and renewDate)
+ * @return routerRMTokenResponse, the response contains the RouterStoreToken.
+ * @throws YarnException if the call to the state store is unsuccessful.
+ * @throws IOException An IO Error occurred.
+ */
@Override
- public RouterRMTokenResponse storeNewToken(RouterRMTokenRequest request)
+ public synchronized RouterRMTokenResponse storeNewToken(RouterRMTokenRequest request)
Review Comment:
> Can we use a finer grain lock. There are many things synchronized that can be done in parallel. Maybe a lock per group too. Could we even do a read write lock?
Thank you very much for your suggestion, your understanding is accurate, if we use synchronized keyword at method level, there is too much involved.
When completing the PR (#5169), JIRA: YARN-11349. [Federation] Router Support Delegation Token With SQL.
I carefully read the `AbstractDelegationTokenSecretManager` code, I think we should be able to remove the `synchronized` keyword in `ZookeeperFederationStateStore`.
I am worried that concurrent access may cause different threads to generate the same primary key data, so that errors will occur when storing, so in the methods of adding, deleting, and updating MasterKey and Token I used the `synchronized` method.
But I found out that this does not happen for generating the same primary key.
> Group1 methods are mainly used to create or remove MasterKey
The key information in `RouterMasterKey` is `DelegationKey`, and the primary key of `DelegationKey` is `keyId`.
We found that `DelegationKey` will be constructed in `AbstractDelegationTokenSecretManager#updateCurrentKey`.
- **AbstractDelegationTokenSecretManager#updateCurrentKey**
This method is protected by synchronized when generating the primary key keyId and storing the masterKey, so the uniqueness of the primary key can be guaranteed
```
private void updateCurrentKey() throws IOException {
LOG.info("Updating the current master key for generating delegation tokens");
int newCurrentId;
// 1. newCurrentId is keyId, which has been protected
synchronized (this) {
newCurrentId = incrementCurrentKeyId();
}
DelegationKey newKey = new DelegationKey(newCurrentId, System
.currentTimeMillis()
+ keyUpdateInterval + tokenMaxLifetime, generateSecret());
logUpdateMasterKey(newKey);
// 2. storeDelegationKey calls storeNewMasterKey, this part has been protected
synchronized (this) {
currentKey = newKey;
storeDelegationKey(currentKey);
}
}
protected void storeDelegationKey(DelegationKey key) throws IOException {
allKeys.put(key.getKeyId(), key);
storeNewMasterKey(key);
}
```
We can find that the method of `removeMasterKey` is also protected.
- **AbstractDelegationTokenSecretManager#removeExpiredKeys**
```
private synchronized void removeExpiredKeys() {
long now = Time.now();
for (Iterator<Map.Entry<Integer, DelegationKey>> it = allKeys.entrySet()
.iterator(); it.hasNext();) {
Map.Entry<Integer, DelegationKey> e = it.next();
if (e.getValue().getExpiryDate() < now) {
it.remove();
// ensure the tokens generated by this current key can be recovered
// with this current key after this current key is rolled
if(!e.getValue().equals(currentKey))
removeStoredMasterKey(e.getValue());
}
}
}
```
> Group2 methods are mainly used to create or update and remove Token
For `DelegationToken`, we can also find that the `add`, `update`, and `remove` methods are all protected.
- **AbstractDelegationTokenSecretManager#storeToken**
```
protected synchronized byte[] createPassword(TokenIdent identifier) {
.....
DelegationTokenInformation tokenInfo = new DelegationTokenInformation(now
+ tokenRenewInterval, password, getTrackingIdIfEnabled(identifier));
try {
// storeToken
METRICS.trackStoreToken(() -> storeToken(identifier, tokenInfo));
} catch (IOException ioe) {
LOG.error("Could not store token " + formatTokenId(identifier) + "!!",
ioe);
}
return password;
}
```
- **AbstractDelegationTokenSecretManager#renewToken**
```
public synchronized long renewToken(Token<TokenIdent> token,
String renewer) throws InvalidToken, IOException {
......
long renewTime = Math.min(id.getMaxDate(), now + tokenRenewInterval);
String trackingId = getTrackingIdIfEnabled(id);
DelegationTokenInformation info = new DelegationTokenInformation(renewTime,
password, trackingId);
if (getTokenInfo(id) == null) {
throw new InvalidToken("Renewal request for unknown token "
+ formatTokenId(id));
}
// updateToken
METRICS.trackUpdateToken(() -> updateToken(id, info));
return renewTime;
}
```
- **AbstractDelegationTokenSecretManager#cancelToken**
```
public synchronized TokenIdent cancelToken(Token<TokenIdent> token,
String canceller) throws IOException {
......
DelegationTokenInformation info = currentTokens.remove(id);
if (info == null) {
throw new InvalidToken("Token not found " + formatTokenId(id));
}
// cancelToken
METRICS.trackRemoveToken(() -> {
removeTokenForOwnerStats(id);
removeStoredToken(id);
});
return id;
}
```
> Group3 methods are used to increment `DelegationTokenSeqNum` and `MasterKeyId`
These two methods are also protected in the abstract class.
- **AbstractDelegationTokenSecretManager#incrementDelegationTokenSeqNum**
```
protected synchronized int incrementDelegationTokenSeqNum() {
return ++delegationTokenSequenceNumber;
}
```
- **AbstractDelegationTokenSecretManager#incrementCurrentKeyId**
```
protected synchronized int incrementCurrentKeyId() {
return ++currentId;
}
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-issues-help@hadoop.apache.org
[GitHub] [hadoop] goiri commented on a diff in pull request #5131: YARN-11350. [Federation] Router Support DelegationToken With ZK.
Posted by GitBox <gi...@apache.org>.
goiri commented on code in PR #5131:
URL: https://github.com/apache/hadoop/pull/5131#discussion_r1042388676
##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/ZookeeperFederationStateStore.java:
##########
@@ -886,45 +1000,599 @@ public UpdateReservationHomeSubClusterResponse updateReservationHomeSubCluster(
return UpdateReservationHomeSubClusterResponse.newInstance();
}
+ /**
+ * ZookeeperFederationStateStore Supports Store NewMasterKey.
+ *
+ * @param request The request contains RouterMasterKey, which is an abstraction for DelegationKey
+ * @return routerMasterKeyResponse, the response contains the RouterMasterKey.
+ * @throws YarnException if the call to the state store is unsuccessful.
+ * @throws IOException An IO Error occurred.
+ */
@Override
public RouterMasterKeyResponse storeNewMasterKey(RouterMasterKeyRequest request)
throws YarnException, IOException {
- throw new NotImplementedException("Code is not implemented");
+
+ long start = clock.getTime();
+ // For the verification of the request, after passing the verification,
+ // the request and the internal objects will not be empty and can be used directly.
+ FederationRouterRMTokenInputValidator.validate(request);
+
+ // Parse the delegationKey from the request and get the ZK storage path.
+ DelegationKey delegationKey = convertMasterKeyToDelegationKey(request);
+ String nodeCreatePath = getMasterKeyZNodePathByDelegationKey(delegationKey);
+ LOG.debug("Storing RMDelegationKey_{}, ZkNodePath = {}.", delegationKey.getKeyId(),
+ nodeCreatePath);
+
+ // Write master key data to zk.
+ try(ByteArrayOutputStream os = new ByteArrayOutputStream();
Review Comment:
Space after try.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-issues-help@hadoop.apache.org
[GitHub] [hadoop] hadoop-yetus commented on pull request #5131: YARN-11350. [Federation] Router Support DelegationToken With ZK.
Posted by GitBox <gi...@apache.org>.
hadoop-yetus commented on PR #5131:
URL: https://github.com/apache/hadoop/pull/5131#issuecomment-1345993411
:broken_heart: **-1 overall**
| Vote | Subsystem | Runtime | Logfile | Comment |
|:----:|----------:|--------:|:--------:|:-------:|
| +0 :ok: | reexec | 0m 45s | | Docker mode activated. |
|||| _ Prechecks _ |
| +1 :green_heart: | dupname | 0m 0s | | No case conflicting files found. |
| +0 :ok: | codespell | 0m 1s | | codespell was not available. |
| +0 :ok: | detsecrets | 0m 1s | | detect-secrets was not available. |
| +1 :green_heart: | @author | 0m 0s | | The patch does not contain any @author tags. |
| +1 :green_heart: | test4tests | 0m 0s | | The patch appears to include 4 new or modified test files. |
|||| _ trunk Compile Tests _ |
| +0 :ok: | mvndep | 15m 52s | | Maven dependency ordering for branch |
| +1 :green_heart: | mvninstall | 25m 41s | | trunk passed |
| +1 :green_heart: | compile | 3m 53s | | trunk passed with JDK Ubuntu-11.0.17+8-post-Ubuntu-1ubuntu220.04 |
| +1 :green_heart: | compile | 3m 20s | | trunk passed with JDK Private Build-1.8.0_352-8u352-ga-1~20.04-b08 |
| +1 :green_heart: | checkstyle | 1m 15s | | trunk passed |
| +1 :green_heart: | mvnsite | 2m 22s | | trunk passed |
| -1 :x: | javadoc | 0m 48s | [/branch-javadoc-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-resourcemanager-jdkUbuntu-11.0.17+8-post-Ubuntu-1ubuntu220.04.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/22/artifact/out/branch-javadoc-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-resourcemanager-jdkUbuntu-11.0.17+8-post-Ubuntu-1ubuntu220.04.txt) | hadoop-yarn-server-resourcemanager in trunk failed with JDK Ubuntu-11.0.17+8-post-Ubuntu-1ubuntu220.04. |
| +1 :green_heart: | javadoc | 1m 48s | | trunk passed with JDK Private Build-1.8.0_352-8u352-ga-1~20.04-b08 |
| +1 :green_heart: | spotbugs | 4m 17s | | trunk passed |
| +1 :green_heart: | shadedclient | 19m 55s | | branch has no errors when building and testing our client artifacts. |
| -0 :warning: | patch | 20m 15s | | Used diff version of patch file. Binary files and potentially other changes not applied. Please rebase and squash commits if necessary. |
|||| _ Patch Compile Tests _ |
| +0 :ok: | mvndep | 0m 29s | | Maven dependency ordering for patch |
| +1 :green_heart: | mvninstall | 1m 50s | | the patch passed |
| +1 :green_heart: | compile | 3m 44s | | the patch passed with JDK Ubuntu-11.0.17+8-post-Ubuntu-1ubuntu220.04 |
| +1 :green_heart: | javac | 3m 44s | | the patch passed |
| +1 :green_heart: | compile | 3m 8s | | the patch passed with JDK Private Build-1.8.0_352-8u352-ga-1~20.04-b08 |
| +1 :green_heart: | javac | 3m 8s | | the patch passed |
| +1 :green_heart: | blanks | 0m 0s | | The patch has no blanks issues. |
| +1 :green_heart: | checkstyle | 1m 2s | | the patch passed |
| +1 :green_heart: | mvnsite | 2m 1s | | the patch passed |
| -1 :x: | javadoc | 0m 41s | [/patch-javadoc-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-resourcemanager-jdkUbuntu-11.0.17+8-post-Ubuntu-1ubuntu220.04.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/22/artifact/out/patch-javadoc-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-resourcemanager-jdkUbuntu-11.0.17+8-post-Ubuntu-1ubuntu220.04.txt) | hadoop-yarn-server-resourcemanager in the patch failed with JDK Ubuntu-11.0.17+8-post-Ubuntu-1ubuntu220.04. |
| +1 :green_heart: | javadoc | 1m 29s | | the patch passed with JDK Private Build-1.8.0_352-8u352-ga-1~20.04-b08 |
| +1 :green_heart: | spotbugs | 4m 21s | | the patch passed |
| +1 :green_heart: | shadedclient | 19m 52s | | patch has no errors when building and testing our client artifacts. |
|||| _ Other Tests _ |
| +1 :green_heart: | unit | 3m 3s | | hadoop-yarn-server-common in the patch passed. |
| +1 :green_heart: | unit | 98m 38s | | hadoop-yarn-server-resourcemanager in the patch passed. |
| +1 :green_heart: | unit | 0m 33s | | hadoop-yarn-server-router in the patch passed. |
| +1 :green_heart: | asflicense | 0m 36s | | The patch does not generate ASF License warnings. |
| | | 225m 48s | | |
| Subsystem | Report/Notes |
|----------:|:-------------|
| Docker | ClientAPI=1.41 ServerAPI=1.41 base: https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/22/artifact/out/Dockerfile |
| GITHUB PR | https://github.com/apache/hadoop/pull/5131 |
| Optional Tests | dupname asflicense compile javac javadoc mvninstall mvnsite unit shadedclient spotbugs checkstyle codespell detsecrets |
| uname | Linux 719a574550d4 4.15.0-200-generic #211-Ubuntu SMP Thu Nov 24 18:16:04 UTC 2022 x86_64 x86_64 x86_64 GNU/Linux |
| Build tool | maven |
| Personality | dev-support/bin/hadoop.sh |
| git revision | trunk / fbe10e558e179bb475adbe21948a967c4906a302 |
| Default Java | Private Build-1.8.0_352-8u352-ga-1~20.04-b08 |
| Multi-JDK versions | /usr/lib/jvm/java-11-openjdk-amd64:Ubuntu-11.0.17+8-post-Ubuntu-1ubuntu220.04 /usr/lib/jvm/java-8-openjdk-amd64:Private Build-1.8.0_352-8u352-ga-1~20.04-b08 |
| Test Results | https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/22/testReport/ |
| Max. process+thread count | 985 (vs. ulimit of 5500) |
| modules | C: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router U: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server |
| Console output | https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/22/console |
| versions | git=2.25.1 maven=3.6.3 spotbugs=4.2.2 |
| Powered by | Apache Yetus 0.14.0 https://yetus.apache.org |
This message was automatically generated.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-issues-help@hadoop.apache.org
[GitHub] [hadoop] goiri commented on a diff in pull request #5131: YARN-11350. [Federation] Router Support DelegationToken With ZK.
Posted by GitBox <gi...@apache.org>.
goiri commented on code in PR #5131:
URL: https://github.com/apache/hadoop/pull/5131#discussion_r1030718575
##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/ZookeeperFederationStateStore.java:
##########
@@ -886,45 +975,554 @@ public UpdateReservationHomeSubClusterResponse updateReservationHomeSubCluster(
return UpdateReservationHomeSubClusterResponse.newInstance();
}
+ /**
+ * ZookeeperFederationStateStore Supports Store NewMasterKey.
+ *
+ * @param request The request contains RouterMasterKey, which is an abstraction for DelegationKey
+ * @return routerMasterKeyResponse, the response contains the RouterMasterKey.
+ * @throws YarnException if the call to the state store is unsuccessful.
+ * @throws IOException An IO Error occurred.
+ */
@Override
- public RouterMasterKeyResponse storeNewMasterKey(RouterMasterKeyRequest request)
+ public synchronized RouterMasterKeyResponse storeNewMasterKey(RouterMasterKeyRequest request)
throws YarnException, IOException {
- throw new NotImplementedException("Code is not implemented");
+
+ long start = clock.getTime();
+ // For the verification of the request, after passing the verification,
+ // the request and the internal objects will not be empty and can be used directly.
+ FederationRouterRMTokenInputValidator.validate(request);
+
+ // Parse the delegationKey from the request and get the ZK storage path.
+ DelegationKey delegationKey = convertMasterKeyToDelegationKey(request);
+ String nodeCreatePath = getMasterKeyZNodePathByDelegationKey(delegationKey);
+ LOG.debug("Storing RMDelegationKey_{}, ZkNodePath = {}.", delegationKey.getKeyId(),
+ nodeCreatePath);
+
+ // Write master key data to zk.
+ try(ByteArrayOutputStream os = new ByteArrayOutputStream();
+ DataOutputStream fsOut = new DataOutputStream(os)) {
+ delegationKey.write(fsOut);
+ put(nodeCreatePath, os.toByteArray(), false);
+ }
+
+ // Get the stored masterKey from zk.
+ RouterMasterKey masterKeyFromZK = getRouterMasterKeyFromZK(nodeCreatePath);
+ long end = clock.getTime();
+ opDurations.addStoreNewMasterKeyDuration(start, end);
+ return RouterMasterKeyResponse.newInstance(masterKeyFromZK);
}
+ /**
+ * ZookeeperFederationStateStore Supports Remove MasterKey.
+ *
+ * @param request The request contains RouterMasterKey, which is an abstraction for DelegationKey
+ * @return routerMasterKeyResponse, the response contains the RouterMasterKey.
+ * @throws YarnException if the call to the state store is unsuccessful.
+ * @throws IOException An IO Error occurred.
+ */
@Override
- public RouterMasterKeyResponse removeStoredMasterKey(RouterMasterKeyRequest request)
+ public synchronized RouterMasterKeyResponse removeStoredMasterKey(RouterMasterKeyRequest request)
throws YarnException, IOException {
- throw new NotImplementedException("Code is not implemented");
+
+ long start = clock.getTime();
+ // For the verification of the request, after passing the verification,
+ // the request and the internal objects will not be empty and can be used directly.
+ FederationRouterRMTokenInputValidator.validate(request);
+
+ try {
+ // Parse the delegationKey from the request and get the ZK storage path.
+ RouterMasterKey masterKey = request.getRouterMasterKey();
+ DelegationKey delegationKey = convertMasterKeyToDelegationKey(request);
+ String nodeRemovePath = getMasterKeyZNodePathByDelegationKey(delegationKey);
+ LOG.debug("Removing RMDelegationKey_{}, ZkNodePath = {}.", delegationKey.getKeyId(),
+ nodeRemovePath);
+
+ // Check if the path exists, Throws an exception if the path does not exist.
+ if (!exists(nodeRemovePath)) {
+ throw new YarnException("ZkNodePath = " + nodeRemovePath + " not exists!");
+ }
+
+ // try to remove masterKey.
+ zkManager.delete(nodeRemovePath);
+ long end = clock.getTime();
+ opDurations.removeStoredMasterKeyDuration(start, end);
+ return RouterMasterKeyResponse.newInstance(masterKey);
+ } catch (Exception e) {
+ throw new YarnException(e);
+ }
}
+ /**
+ * ZookeeperFederationStateStore Supports Remove MasterKey.
+ *
+ * @param request The request contains RouterMasterKey, which is an abstraction for DelegationKey
+ * @return routerMasterKeyResponse, the response contains the RouterMasterKey.
+ * @throws YarnException if the call to the state store is unsuccessful.
+ * @throws IOException An IO Error occurred.
+ */
@Override
public RouterMasterKeyResponse getMasterKeyByDelegationKey(RouterMasterKeyRequest request)
throws YarnException, IOException {
- throw new NotImplementedException("Code is not implemented");
+
+ long start = clock.getTime();
+ // For the verification of the request, after passing the verification,
+ // the request and the internal objects will not be empty and can be used directly.
+ FederationRouterRMTokenInputValidator.validate(request);
+
+ try {
+
+ // Parse the delegationKey from the request and get the ZK storage path.
+ DelegationKey delegationKey = convertMasterKeyToDelegationKey(request);
+ String nodePath = getMasterKeyZNodePathByDelegationKey(delegationKey);
+
+ // Check if the path exists, Throws an exception if the path does not exist.
+ if (!exists(nodePath)) {
+ throw new YarnException("ZkNodePath = " + nodePath + " not exists!");
+ }
+
+ // Get the stored masterKey from zk.
+ RouterMasterKey routerMasterKey = getRouterMasterKeyFromZK(nodePath);
+ long end = clock.getTime();
+ opDurations.getMasterKeyByDelegationKeyDuration(start, end);
+ return RouterMasterKeyResponse.newInstance(routerMasterKey);
+ } catch (Exception e) {
+ throw new YarnException(e);
+ }
+ }
+
+ /**
+ * Get MasterKeyZNodePath based on DelegationKey.
+ *
+ * @param delegationKey delegationKey.
+ * @return masterKey ZNodePath.
+ */
+ private String getMasterKeyZNodePathByDelegationKey(DelegationKey delegationKey) {
+ return getMasterKeyZNodePathByKeyId(delegationKey.getKeyId());
+ }
+
+ /**
+ * Get MasterKeyZNodePath based on KeyId.
+ *
+ * @param keyId master key id.
+ * @return masterKey ZNodePath.
+ */
+ private String getMasterKeyZNodePathByKeyId(int keyId) {
+ String nodeName = ROUTER_RM_DELEGATION_KEY_PREFIX + keyId;
+ return getNodePath(routerRMDTMasterKeysRootPath, nodeName);
+ }
+
+ /**
+ * Get RouterMasterKey from ZK.
+ *
+ * @param nodePath The path where masterKey is stored in zk.
+ *
+ * @return RouterMasterKey.
+ * @throws IOException An IO Error occurred.
+ */
+ private RouterMasterKey getRouterMasterKeyFromZK(String nodePath)
+ throws IOException {
+ try {
+ byte[] data = get(nodePath);
+ if ((data == null) || (data.length == 0)) {
+ return null;
+ }
+
+ ByteArrayInputStream bin = new ByteArrayInputStream(data);
+ DataInputStream din = new DataInputStream(bin);
+ DelegationKey key = new DelegationKey();
+ key.readFields(din);
+
+ return RouterMasterKey.newInstance(key.getKeyId(),
+ ByteBuffer.wrap(key.getEncodedKey()), key.getExpiryDate());
+ } catch (Exception ex) {
+ LOG.error("No node in path {}.", nodePath);
+ throw new IOException(ex);
+ }
}
+ /**
+ * ZookeeperFederationStateStore Supports Store RMDelegationTokenIdentifier.
+ *
+ * The stored token method is a synchronized method
+ * used to ensure that storeNewToken is a thread-safe method.
+ *
+ * @param request The request contains RouterRMToken (RMDelegationTokenIdentifier and renewDate)
+ * @return routerRMTokenResponse, the response contains the RouterStoreToken.
+ * @throws YarnException if the call to the state store is unsuccessful.
+ * @throws IOException An IO Error occurred.
+ */
@Override
- public RouterRMTokenResponse storeNewToken(RouterRMTokenRequest request)
+ public synchronized RouterRMTokenResponse storeNewToken(RouterRMTokenRequest request)
Review Comment:
Can we use a finer grain lock. There are many things synchronized that can be done in parallel. Maybe a lock per group too. Could we even do a read write lock?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-issues-help@hadoop.apache.org
[GitHub] [hadoop] goiri merged pull request #5131: YARN-11350. [Federation] Router Support DelegationToken With ZK.
Posted by GitBox <gi...@apache.org>.
goiri merged PR #5131:
URL: https://github.com/apache/hadoop/pull/5131
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-issues-help@hadoop.apache.org
[GitHub] [hadoop] slfan1989 commented on a diff in pull request #5131: YARN-11350. [Federation] Router Support DelegationToken With ZK.
Posted by GitBox <gi...@apache.org>.
slfan1989 commented on code in PR #5131:
URL: https://github.com/apache/hadoop/pull/5131#discussion_r1035905558
##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterStoreToken.java:
##########
@@ -22,6 +22,7 @@
import org.apache.hadoop.yarn.security.client.YARNDelegationTokenIdentifier;
import org.apache.hadoop.yarn.util.Records;
+import java.io.DataInput;
Review Comment:
We used the `readFields` method, which requires `DataInput` parameters.
RouterStoreToken#readFields
```
public abstract void readFields(DataInput in) throws IOException;
```
When we store `RouterStoreToken`, we convert token into a byte array, and then store it in `ZK`
- ZookeeperFederationStateStore#storeOrUpdateRouterRMDT
```
private void storeOrUpdateRouterRMDT(RouterRMTokenRequest request, boolean isUpdate)
throws Exception {
RouterStoreToken routerStoreToken = request.getRouterStoreToken();
String nodeCreatePath = getStoreTokenZNodePathByTokenRequest(request);
LOG.debug("nodeCreatePath = {}, isUpdate = {}", nodeCreatePath, isUpdate);
put(nodeCreatePath, routerStoreToken.toByteArray(), isUpdate);
}
```
When we need to read data, we convert the bytes array stored in `ZK` into `DataInputStream`, and then let `RouterStoreToken` parse the `required fields` from it.
- ZookeeperFederationStateStore#getStoreTokenFromZK
```
private RouterStoreToken getStoreTokenFromZK(String nodePath, boolean quiet)
throws IOException {
try {
byte[] data = get(nodePath);
if ((data == null) || (data.length == 0)) {
return null;
}
ByteArrayInputStream bin = new ByteArrayInputStream(data);
DataInputStream din = new DataInputStream(bin);
RouterStoreToken storeToken = Records.newRecord(RouterStoreToken.class);
storeToken.readFields(din);
return storeToken;
} catch (Exception ex) {
if (!quiet) {
LOG.error("No node in path [" + nodePath + "]");
}
throw new IOException(ex);
}
}
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-issues-help@hadoop.apache.org
[GitHub] [hadoop] goiri commented on a diff in pull request #5131: YARN-11350. [Federation] Router Support DelegationToken With ZK.
Posted by GitBox <gi...@apache.org>.
goiri commented on code in PR #5131:
URL: https://github.com/apache/hadoop/pull/5131#discussion_r1039696785
##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/TestZookeeperFederationStateStore.java:
##########
@@ -171,38 +203,293 @@ public void testMetricsInited() throws Exception {
MetricsRecords.assertMetric(record, "UpdateReservationHomeSubClusterNumOps", expectOps);
}
- @Test(expected = NotImplementedException.class)
+ @Test
public void testStoreNewMasterKey() throws Exception {
- super.testStoreNewMasterKey();
+
+ // Manually create a DelegationKey,
+ // and call the interface storeNewMasterKey to write the data to zk.
+ DelegationKey key = new DelegationKey(1234, Time.now() + 60 * 60, "keyBytes".getBytes());
+ RouterMasterKey paramRouterMasterKey = RouterMasterKey.newInstance(key.getKeyId(),
+ ByteBuffer.wrap(key.getEncodedKey()), key.getExpiryDate());
+ FederationStateStore stateStore = this.getStateStore();
+
+ assertTrue(stateStore instanceof ZookeeperFederationStateStore);
+
+ // Compare the data returned by the storeNewMasterKey
+ // interface with the data queried by zk, and ensure that the data is consistent.
+ RouterMasterKeyRequest routerMasterKeyRequest =
+ RouterMasterKeyRequest.newInstance(paramRouterMasterKey);
+ RouterMasterKeyResponse response = stateStore.storeNewMasterKey(routerMasterKeyRequest);
+ assertNotNull(response);
+ RouterMasterKey respRouterMasterKey = response.getRouterMasterKey();
+ assertNotNull(respRouterMasterKey);
+
+ // Get Data From zk.
+ String nodeName = ROUTER_RM_DELEGATION_KEY_PREFIX + key.getKeyId();
+ String nodePath = ZNODE_MASTER_KEY_PREFIX + nodeName;
+ RouterMasterKey zkRouterMasterKey = getRouterMasterKeyFromZK(nodePath);
+
+ assertNotNull(zkRouterMasterKey);
+ assertEquals(paramRouterMasterKey, respRouterMasterKey);
+ assertEquals(paramRouterMasterKey, zkRouterMasterKey);
+ assertEquals(zkRouterMasterKey, respRouterMasterKey);
}
- @Test(expected = NotImplementedException.class)
+ @Test
public void testGetMasterKeyByDelegationKey() throws YarnException, IOException {
- super.testGetMasterKeyByDelegationKey();
+
+ // Manually create a DelegationKey,
+ // and call the interface storeNewMasterKey to write the data to zk.
+ DelegationKey key = new DelegationKey(5678, Time.now() + 60 * 60, "keyBytes".getBytes());
+ RouterMasterKey paramRouterMasterKey = RouterMasterKey.newInstance(key.getKeyId(),
+ ByteBuffer.wrap(key.getEncodedKey()), key.getExpiryDate());
+ FederationStateStore stateStore = this.getStateStore();
+
+ assertTrue(stateStore instanceof ZookeeperFederationStateStore);
Review Comment:
Can we use the super method and have a function to check these things we do in every method?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-issues-help@hadoop.apache.org
[GitHub] [hadoop] slfan1989 commented on pull request #5131: YARN-11350. [Federation] Router Support DelegationToken With ZK.
Posted by GitBox <gi...@apache.org>.
slfan1989 commented on PR #5131:
URL: https://github.com/apache/hadoop/pull/5131#issuecomment-1337534927
@goiri Can you help review pr(#5182)? Thank you very much! I fixed Java Doc issue for `hadoop-yarn-server-common` module in this pr.
In the process of compiling YARN-11350, I encountered some java doc compilation errors.
We can see the compilation report as follows:
[patch-javadoc-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-common-jdkUbuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/15/artifact/out/patch-javadoc-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-common-jdkUbuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04.txt)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-issues-help@hadoop.apache.org
[GitHub] [hadoop] goiri commented on a diff in pull request #5131: YARN-11350. [Federation] Router Support DelegationToken With ZK.
Posted by GitBox <gi...@apache.org>.
goiri commented on code in PR #5131:
URL: https://github.com/apache/hadoop/pull/5131#discussion_r1041207295
##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/TestSQLFederationStateStore.java:
##########
@@ -592,4 +588,14 @@ public void testRemoveStoredToken() throws IOException, YarnException {
public void testGetTokenByRouterStoreToken() throws IOException, YarnException {
super.testGetTokenByRouterStoreToken();
}
+
+ @Override
+ protected void checkRouterMasterKey(DelegationKey delegationKey,
+ RouterMasterKey routerMasterKey) throws YarnException, IOException {
Review Comment:
Comment on why we do nothing
##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/TestSQLFederationStateStore.java:
##########
@@ -18,21 +18,17 @@
package org.apache.hadoop.yarn.server.federation.store.impl;
import org.apache.commons.lang3.NotImplementedException;
+import org.apache.hadoop.security.token.delegation.DelegationKey;
import org.apache.hadoop.test.LambdaTestUtils;
import org.apache.hadoop.util.Time;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ReservationId;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier;
import org.apache.hadoop.yarn.server.federation.store.FederationStateStore;
import org.apache.hadoop.yarn.server.federation.store.metrics.FederationStateStoreClientMetrics;
-import org.apache.hadoop.yarn.server.federation.store.records.SubClusterId;
-import org.apache.hadoop.yarn.server.federation.store.records.SubClusterInfo;
-import org.apache.hadoop.yarn.server.federation.store.records.SubClusterRegisterRequest;
-import org.apache.hadoop.yarn.server.federation.store.records.ReservationHomeSubCluster;
-import org.apache.hadoop.yarn.server.federation.store.records.AddReservationHomeSubClusterRequest;
-import org.apache.hadoop.yarn.server.federation.store.records.UpdateReservationHomeSubClusterRequest;
-import org.apache.hadoop.yarn.server.federation.store.records.DeleteReservationHomeSubClusterRequest;
+import org.apache.hadoop.yarn.server.federation.store.records.*;
Review Comment:
Avoid
##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/TestZookeeperFederationStateStore.java:
##########
@@ -171,38 +203,117 @@ public void testMetricsInited() throws Exception {
MetricsRecords.assertMetric(record, "UpdateReservationHomeSubClusterNumOps", expectOps);
}
- @Test(expected = NotImplementedException.class)
+ @Test
public void testStoreNewMasterKey() throws Exception {
super.testStoreNewMasterKey();
}
- @Test(expected = NotImplementedException.class)
+ @Test
Review Comment:
I think we can just inherit and no need to have them here.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-issues-help@hadoop.apache.org
[GitHub] [hadoop] hadoop-yetus commented on pull request #5131: YARN-11350. [Federation] Router Support DelegationToken With ZK.
Posted by GitBox <gi...@apache.org>.
hadoop-yetus commented on PR #5131:
URL: https://github.com/apache/hadoop/pull/5131#issuecomment-1317401510
:confetti_ball: **+1 overall**
| Vote | Subsystem | Runtime | Logfile | Comment |
|:----:|----------:|--------:|:--------:|:-------:|
| +0 :ok: | reexec | 0m 42s | | Docker mode activated. |
|||| _ Prechecks _ |
| +1 :green_heart: | dupname | 0m 0s | | No case conflicting files found. |
| +0 :ok: | codespell | 0m 0s | | codespell was not available. |
| +0 :ok: | detsecrets | 0m 0s | | detect-secrets was not available. |
| +1 :green_heart: | @author | 0m 0s | | The patch does not contain any @author tags. |
| +1 :green_heart: | test4tests | 0m 0s | | The patch appears to include 3 new or modified test files. |
|||| _ trunk Compile Tests _ |
| +0 :ok: | mvndep | 16m 22s | | Maven dependency ordering for branch |
| +1 :green_heart: | mvninstall | 25m 55s | | trunk passed |
| +1 :green_heart: | compile | 4m 3s | | trunk passed with JDK Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 |
| +1 :green_heart: | compile | 3m 35s | | trunk passed with JDK Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | checkstyle | 1m 23s | | trunk passed |
| +1 :green_heart: | mvnsite | 2m 58s | | trunk passed |
| +1 :green_heart: | javadoc | 2m 27s | | trunk passed with JDK Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 |
| +1 :green_heart: | javadoc | 2m 19s | | trunk passed with JDK Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | spotbugs | 4m 47s | | trunk passed |
| +1 :green_heart: | shadedclient | 20m 51s | | branch has no errors when building and testing our client artifacts. |
|||| _ Patch Compile Tests _ |
| +0 :ok: | mvndep | 0m 28s | | Maven dependency ordering for patch |
| +1 :green_heart: | mvninstall | 2m 6s | | the patch passed |
| +1 :green_heart: | compile | 3m 57s | | the patch passed with JDK Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 |
| +1 :green_heart: | javac | 3m 57s | | the patch passed |
| +1 :green_heart: | compile | 3m 31s | | the patch passed with JDK Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | javac | 3m 31s | | the patch passed |
| +1 :green_heart: | blanks | 0m 0s | | The patch has no blanks issues. |
| +1 :green_heart: | checkstyle | 1m 11s | | the patch passed |
| +1 :green_heart: | mvnsite | 2m 18s | | the patch passed |
| +1 :green_heart: | javadoc | 0m 40s | | hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-common-jdkUbuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 with JDK Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 generated 0 new + 47 unchanged - 1 fixed = 47 total (was 48) |
| +1 :green_heart: | javadoc | 0m 45s | | hadoop-yarn-server-resourcemanager in the patch passed with JDK Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04. |
| +1 :green_heart: | javadoc | 0m 33s | | hadoop-yarn-server-router in the patch passed with JDK Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04. |
| +1 :green_heart: | javadoc | 0m 34s | | hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-common-jdkPrivateBuild-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 with JDK Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 generated 0 new + 47 unchanged - 1 fixed = 47 total (was 48) |
| +1 :green_heart: | javadoc | 0m 46s | | hadoop-yarn-server-resourcemanager in the patch passed with JDK Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07. |
| +1 :green_heart: | javadoc | 0m 30s | | hadoop-yarn-server-router in the patch passed with JDK Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07. |
| +1 :green_heart: | spotbugs | 4m 32s | | the patch passed |
| +1 :green_heart: | shadedclient | 20m 26s | | patch has no errors when building and testing our client artifacts. |
|||| _ Other Tests _ |
| +1 :green_heart: | unit | 3m 18s | | hadoop-yarn-server-common in the patch passed. |
| +1 :green_heart: | unit | 98m 57s | | hadoop-yarn-server-resourcemanager in the patch passed. |
| +1 :green_heart: | unit | 5m 48s | | hadoop-yarn-server-router in the patch passed. |
| +1 :green_heart: | asflicense | 0m 43s | | The patch does not generate ASF License warnings. |
| | | 239m 14s | | |
| Subsystem | Report/Notes |
|----------:|:-------------|
| Docker | ClientAPI=1.41 ServerAPI=1.41 base: https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/6/artifact/out/Dockerfile |
| GITHUB PR | https://github.com/apache/hadoop/pull/5131 |
| Optional Tests | dupname asflicense compile javac javadoc mvninstall mvnsite unit shadedclient spotbugs checkstyle codespell detsecrets |
| uname | Linux 382151c64743 4.15.0-191-generic #202-Ubuntu SMP Thu Aug 4 01:49:29 UTC 2022 x86_64 x86_64 x86_64 GNU/Linux |
| Build tool | maven |
| Personality | dev-support/bin/hadoop.sh |
| git revision | trunk / 768f524908bbfdd63260d2e293abd8d309ea5250 |
| Default Java | Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| Multi-JDK versions | /usr/lib/jvm/java-11-openjdk-amd64:Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 /usr/lib/jvm/java-8-openjdk-amd64:Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| Test Results | https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/6/testReport/ |
| Max. process+thread count | 956 (vs. ulimit of 5500) |
| modules | C: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router U: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server |
| Console output | https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/6/console |
| versions | git=2.25.1 maven=3.6.3 spotbugs=4.2.2 |
| Powered by | Apache Yetus 0.14.0 https://yetus.apache.org |
This message was automatically generated.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-issues-help@hadoop.apache.org
[GitHub] [hadoop] goiri commented on a diff in pull request #5131: YARN-11350. [Federation] Router Support DelegationToken With ZK.
Posted by GitBox <gi...@apache.org>.
goiri commented on code in PR #5131:
URL: https://github.com/apache/hadoop/pull/5131#discussion_r1036592621
##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/ZookeeperFederationStateStore.java:
##########
@@ -886,45 +1000,608 @@ public UpdateReservationHomeSubClusterResponse updateReservationHomeSubCluster(
return UpdateReservationHomeSubClusterResponse.newInstance();
}
+ /**
+ * ZookeeperFederationStateStore Supports Store NewMasterKey.
+ *
+ * @param request The request contains RouterMasterKey, which is an abstraction for DelegationKey
+ * @return routerMasterKeyResponse, the response contains the RouterMasterKey.
+ * @throws YarnException if the call to the state store is unsuccessful.
+ * @throws IOException An IO Error occurred.
+ */
@Override
public RouterMasterKeyResponse storeNewMasterKey(RouterMasterKeyRequest request)
throws YarnException, IOException {
- throw new NotImplementedException("Code is not implemented");
+
+ long start = clock.getTime();
+ // For the verification of the request, after passing the verification,
+ // the request and the internal objects will not be empty and can be used directly.
+ FederationRouterRMTokenInputValidator.validate(request);
+
+ // Parse the delegationKey from the request and get the ZK storage path.
+ DelegationKey delegationKey = convertMasterKeyToDelegationKey(request);
+ String nodeCreatePath = getMasterKeyZNodePathByDelegationKey(delegationKey);
+ LOG.debug("Storing RMDelegationKey_{}, ZkNodePath = {}.", delegationKey.getKeyId(),
+ nodeCreatePath);
+
+ // Write master key data to zk.
+ try(ByteArrayOutputStream os = new ByteArrayOutputStream();
+ DataOutputStream fsOut = new DataOutputStream(os)) {
+ delegationKey.write(fsOut);
+ put(nodeCreatePath, os.toByteArray(), false);
+ }
+
+ // Get the stored masterKey from zk.
+ RouterMasterKey masterKeyFromZK = getRouterMasterKeyFromZK(nodeCreatePath);
+ long end = clock.getTime();
+ opDurations.addStoreNewMasterKeyDuration(start, end);
+ return RouterMasterKeyResponse.newInstance(masterKeyFromZK);
}
+ /**
+ * ZookeeperFederationStateStore Supports Remove MasterKey.
+ *
+ * @param request The request contains RouterMasterKey, which is an abstraction for DelegationKey
+ * @return routerMasterKeyResponse, the response contains the RouterMasterKey.
+ * @throws YarnException if the call to the state store is unsuccessful.
+ * @throws IOException An IO Error occurred.
+ */
@Override
public RouterMasterKeyResponse removeStoredMasterKey(RouterMasterKeyRequest request)
throws YarnException, IOException {
- throw new NotImplementedException("Code is not implemented");
+
+ long start = clock.getTime();
+ // For the verification of the request, after passing the verification,
+ // the request and the internal objects will not be empty and can be used directly.
+ FederationRouterRMTokenInputValidator.validate(request);
+
+ try {
+ // Parse the delegationKey from the request and get the ZK storage path.
+ RouterMasterKey masterKey = request.getRouterMasterKey();
+ DelegationKey delegationKey = convertMasterKeyToDelegationKey(request);
+ String nodeRemovePath = getMasterKeyZNodePathByDelegationKey(delegationKey);
+ LOG.debug("Removing RMDelegationKey_{}, ZkNodePath = {}.", delegationKey.getKeyId(),
+ nodeRemovePath);
+
+ // Check if the path exists, Throws an exception if the path does not exist.
+ if (!exists(nodeRemovePath)) {
+ throw new YarnException("ZkNodePath = " + nodeRemovePath + " not exists!");
+ }
+
+ // try to remove masterKey.
+ zkManager.delete(nodeRemovePath);
+ long end = clock.getTime();
+ opDurations.removeStoredMasterKeyDuration(start, end);
+ return RouterMasterKeyResponse.newInstance(masterKey);
+ } catch (Exception e) {
+ throw new YarnException(e);
+ }
}
+ /**
+ * ZookeeperFederationStateStore Supports Remove MasterKey.
+ *
+ * @param request The request contains RouterMasterKey, which is an abstraction for DelegationKey
+ * @return routerMasterKeyResponse, the response contains the RouterMasterKey.
+ * @throws YarnException if the call to the state store is unsuccessful.
+ * @throws IOException An IO Error occurred.
+ */
@Override
public RouterMasterKeyResponse getMasterKeyByDelegationKey(RouterMasterKeyRequest request)
throws YarnException, IOException {
- throw new NotImplementedException("Code is not implemented");
+
+ long start = clock.getTime();
+ // For the verification of the request, after passing the verification,
+ // the request and the internal objects will not be empty and can be used directly.
+ FederationRouterRMTokenInputValidator.validate(request);
+
+ try {
+
+ // Parse the delegationKey from the request and get the ZK storage path.
+ DelegationKey delegationKey = convertMasterKeyToDelegationKey(request);
+ String nodePath = getMasterKeyZNodePathByDelegationKey(delegationKey);
+
+ // Check if the path exists, Throws an exception if the path does not exist.
+ if (!exists(nodePath)) {
+ throw new YarnException("ZkNodePath = " + nodePath + " not exists!");
+ }
+
+ // Get the stored masterKey from zk.
+ RouterMasterKey routerMasterKey = getRouterMasterKeyFromZK(nodePath);
+ long end = clock.getTime();
+ opDurations.getMasterKeyByDelegationKeyDuration(start, end);
+ return RouterMasterKeyResponse.newInstance(routerMasterKey);
+ } catch (Exception e) {
+ throw new YarnException(e);
+ }
+ }
+
+ /**
+ * Get MasterKeyZNodePath based on DelegationKey.
+ *
+ * @param delegationKey delegationKey.
+ * @return masterKey ZNodePath.
+ */
+ private String getMasterKeyZNodePathByDelegationKey(DelegationKey delegationKey) {
+ return getMasterKeyZNodePathByKeyId(delegationKey.getKeyId());
}
+ /**
+ * Get MasterKeyZNodePath based on KeyId.
+ *
+ * @param keyId master key id.
+ * @return masterKey ZNodePath.
+ */
+ private String getMasterKeyZNodePathByKeyId(int keyId) {
+ String nodeName = ROUTER_RM_DELEGATION_KEY_PREFIX + keyId;
+ return getNodePath(routerRMDTMasterKeysRootPath, nodeName);
+ }
+
+ /**
+ * Get RouterMasterKey from ZK.
+ *
+ * @param nodePath The path where masterKey is stored in zk.
+ *
+ * @return RouterMasterKey.
+ * @throws IOException An IO Error occurred.
+ */
+ private RouterMasterKey getRouterMasterKeyFromZK(String nodePath)
+ throws IOException {
+ try {
+ byte[] data = get(nodePath);
+ if ((data == null) || (data.length == 0)) {
+ return null;
+ }
+
+ ByteArrayInputStream bin = new ByteArrayInputStream(data);
+ DataInputStream din = new DataInputStream(bin);
+ DelegationKey key = new DelegationKey();
+ key.readFields(din);
+
+ return RouterMasterKey.newInstance(key.getKeyId(),
+ ByteBuffer.wrap(key.getEncodedKey()), key.getExpiryDate());
+ } catch (Exception ex) {
+ LOG.error("No node in path {}.", nodePath);
+ throw new IOException(ex);
+ }
+ }
+
+ /**
+ * ZookeeperFederationStateStore Supports Store RMDelegationTokenIdentifier.
+ *
+ * The stored token method is a synchronized method
+ * used to ensure that storeNewToken is a thread-safe method.
+ *
+ * @param request The request contains RouterRMToken (RMDelegationTokenIdentifier and renewDate)
+ * @return routerRMTokenResponse, the response contains the RouterStoreToken.
+ * @throws YarnException if the call to the state store is unsuccessful.
+ * @throws IOException An IO Error occurred.
+ */
@Override
public RouterRMTokenResponse storeNewToken(RouterRMTokenRequest request)
throws YarnException, IOException {
- throw new NotImplementedException("Code is not implemented");
+
+ long start = clock.getTime();
+ // We verify the RouterRMTokenRequest to ensure that the request is not empty,
+ // and that the internal RouterStoreToken is not empty.
+ FederationRouterRMTokenInputValidator.validate(request);
+
+ try {
+
+ // add delegationToken
+ storeOrUpdateRouterRMDT(request, false);
+
+ // Get the stored delegationToken from ZK and return.
+ RouterStoreToken resultStoreToken = getStoreTokenFromZK(request, false);
+ long end = clock.getTime();
+ opDurations.getStoreNewTokenDuration(start, end);
+ return RouterRMTokenResponse.newInstance(resultStoreToken);
+ } catch (YarnException | IOException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new YarnException(e);
+ }
}
+ /**
+ * ZookeeperFederationStateStore Supports Update RMDelegationTokenIdentifier.
+ *
+ * The update stored token method is a synchronized method
+ * used to ensure that storeNewToken is a thread-safe method.
+ *
+ * @param request The request contains RouterRMToken (RMDelegationTokenIdentifier and renewDate)
+ * @return routerRMTokenResponse, the response contains the RouterStoreToken.
+ * @throws YarnException if the call to the state store is unsuccessful.
+ * @throws IOException An IO Error occurred.
+ */
@Override
public RouterRMTokenResponse updateStoredToken(RouterRMTokenRequest request)
throws YarnException, IOException {
- throw new NotImplementedException("Code is not implemented");
+
+ long start = clock.getTime();
+ // We verify the RouterRMTokenRequest to ensure that the request is not empty,
+ // and that the internal RouterStoreToken is not empty.
+ FederationRouterRMTokenInputValidator.validate(request);
+
+ try {
+
+ // get the Token storage path
+ String nodePath = getStoreTokenZNodePathByTokenRequest(request);
+
+ // updateStoredToken needs to determine whether the zkNode exists.
+ // If it exists, update the token data.
+ // If it does not exist, write the new token data directly.
+ boolean pathExists = true;
+ if (!exists(nodePath)) {
+ pathExists = false;
+ }
+
+ if (pathExists) {
+ // update delegationToken
+ storeOrUpdateRouterRMDT(request, true);
+ } else {
+ // add new delegationToken
+ storeNewToken(request);
+ }
+
+ // Get the stored delegationToken from ZK and return.
+ RouterStoreToken resultStoreToken = getStoreTokenFromZK(request, false);
+ long end = clock.getTime();
+ opDurations.updateStoredTokenDuration(start, end);
+ return RouterRMTokenResponse.newInstance(resultStoreToken);
+ } catch (YarnException | IOException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new YarnException(e);
+ }
}
+ /**
+ * ZookeeperFederationStateStore Supports Remove RMDelegationTokenIdentifier.
+ *
+ * The remove stored token method is a synchronized method
+ * used to ensure that storeNewToken is a thread-safe method.
+ *
+ * @param request The request contains RouterRMToken (RMDelegationTokenIdentifier and renewDate)
+ * @return routerRMTokenResponse, the response contains the RouterStoreToken.
+ * @throws YarnException if the call to the state store is unsuccessful.
+ * @throws IOException An IO Error occurred.
+ */
@Override
public RouterRMTokenResponse removeStoredToken(RouterRMTokenRequest request)
throws YarnException, IOException {
- throw new NotImplementedException("Code is not implemented");
+
+ long start = clock.getTime();
+ // We verify the RouterRMTokenRequest to ensure that the request is not empty,
+ // and that the internal RouterStoreToken is not empty.
+ FederationRouterRMTokenInputValidator.validate(request);
+
+ try {
+
+ // get the Token storage path
+ String nodePath = getStoreTokenZNodePathByTokenRequest(request);
+
+ // If the path to be deleted does not exist, throw an exception directly.
+ if (!exists(nodePath)) {
+ throw new YarnException("ZkNodePath = " + nodePath + " not exists!");
+ }
+
+ // Check again, first get the data from ZK,
+ // if the data is not empty, then delete it
+ RouterStoreToken storeToken = getStoreTokenFromZK(request, false);
+ if (storeToken != null) {
+ zkManager.delete(nodePath);
+ }
+
+ // return deleted token data.
+ long end = clock.getTime();
+ opDurations.removeStoredTokenDuration(start, end);
+ return RouterRMTokenResponse.newInstance(storeToken);
+ } catch (YarnException | IOException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new YarnException(e);
+ }
}
+ /**
+ * The Router Supports GetTokenByRouterStoreToken.
+ *
+ * @param request The request contains RouterRMToken (RMDelegationTokenIdentifier and renewDate)
+ * @return RouterRMTokenResponse.
+ * @throws YarnException if the call to the state store is unsuccessful
+ * @throws IOException An IO Error occurred
+ */
@Override
public RouterRMTokenResponse getTokenByRouterStoreToken(RouterRMTokenRequest request)
throws YarnException, IOException {
- throw new NotImplementedException("Code is not implemented");
+
+ long start = clock.getTime();
+ // We verify the RouterRMTokenRequest to ensure that the request is not empty,
+ // and that the internal RouterStoreToken is not empty.
+ FederationRouterRMTokenInputValidator.validate(request);
+
+ try {
+
+ // Before get the token,
+ // we need to determine whether the path where the token is stored exists.
+ // If it doesn't exist, we will throw an exception.
+ String nodePath = getStoreTokenZNodePathByTokenRequest(request);
+ if (!exists(nodePath)) {
+ throw new YarnException("ZkNodePath = " + nodePath + " not exists!");
+ }
+
+ // Get the stored delegationToken from ZK and return.
+ RouterStoreToken resultStoreToken = getStoreTokenFromZK(request, false);
+ // return deleted token data.
+ long end = clock.getTime();
+ opDurations.getTokenByRouterStoreTokenDuration(start, end);
+ return RouterRMTokenResponse.newInstance(resultStoreToken);
+ } catch (YarnException | IOException e) {
+ throw e;
+ } catch (Exception e) {
+ throw new YarnException(e);
+ }
+ }
+
+ /**
+ * Convert MasterKey to DelegationKey.
+ *
+ * Before using this function,
+ * please use FederationRouterRMTokenInputValidator to verify the request.
+ * By default, the request is not empty, and the internal object is not empty.
+ *
+ * @param request RouterMasterKeyRequest
+ * @return DelegationKey.
+ */
+ private DelegationKey convertMasterKeyToDelegationKey(RouterMasterKeyRequest request) {
+ RouterMasterKey masterKey = request.getRouterMasterKey();
+ return convertMasterKeyToDelegationKey(masterKey);
+ }
+
+ /**
+ * Convert MasterKey to DelegationKey.
+ *
+ * @param masterKey masterKey.
+ * @return DelegationKey.
+ */
+ private DelegationKey convertMasterKeyToDelegationKey(RouterMasterKey masterKey) {
+ ByteBuffer keyByteBuf = masterKey.getKeyBytes();
+ byte[] keyBytes = new byte[keyByteBuf.remaining()];
+ keyByteBuf.get(keyBytes);
+ return new DelegationKey(masterKey.getKeyId(), masterKey.getExpiryDate(), keyBytes);
+ }
+
+ /**
+ * Check if a path exists in zk.
+ *
+ * @param path Path to be checked.
+ * @return Returns true if the path exists, false if the path does not exist.
+ * @throws Exception When an exception to access zk occurs.
+ */
+ @VisibleForTesting
+ boolean exists(final String path) throws Exception {
+ return zkManager.exists(path);
+ }
+
+ /**
+ * Add or update delegationToken.
+ *
+ * Before using this function,
+ * please use FederationRouterRMTokenInputValidator to verify the request.
+ * By default, the request is not empty, and the internal object is not empty.
+ *
+ * @param request storeToken
+ * @param isUpdate true, update the token; false, create a new token.
+ * @throws Exception exception occurs.
+ */
+ private void storeOrUpdateRouterRMDT(RouterRMTokenRequest request, boolean isUpdate)
+ throws Exception {
+
+ RouterStoreToken routerStoreToken = request.getRouterStoreToken();
+ String nodeCreatePath = getStoreTokenZNodePathByTokenRequest(request);
+ LOG.debug("nodeCreatePath = {}, isUpdate = {}", nodeCreatePath, isUpdate);
+ put(nodeCreatePath, routerStoreToken.toByteArray(), isUpdate);
+ }
+
+ /**
+ * Get ZNode Path of StoreToken.
+ *
+ * Before using this method, we should use FederationRouterRMTokenInputValidator
+ * to verify the request,ensure that the request is not empty,
+ * and ensure that the object in the request is not empty.
+ *
+ * @param request RouterMasterKeyRequest.
+ * @return RouterRMToken ZNode Path.
+ * @throws IOException io exception occurs.
+ */
+ private String getStoreTokenZNodePathByTokenRequest(RouterRMTokenRequest request)
+ throws IOException {
+ RouterStoreToken routerStoreToken = request.getRouterStoreToken();
+ YARNDelegationTokenIdentifier identifier = routerStoreToken.getTokenIdentifier();
+ return getStoreTokenZNodePathByIdentifier(identifier);
+ }
+
+ /**
+ * Get ZNode Path of StoreToken.
+ *
+ * @param identifier YARNDelegationTokenIdentifier
+ * @return RouterRMToken ZNode Path.
+ */
+ private String getStoreTokenZNodePathByIdentifier(YARNDelegationTokenIdentifier identifier) {
+ String nodePath = getNodePath(routerRMDelegationTokensRootPath,
+ ROUTER_RM_DELEGATION_TOKEN_PREFIX + identifier.getSequenceNumber());
+ return nodePath;
+ }
+
+ /**
+ * Get RouterStoreToken from ZK.
+ *
+ * @param request RouterMasterKeyRequest.
+ * @param quiet If true is silent mode, no error message is printed at this time,
+ * if false is non-silent mode, error message is printed at this time.
+ * @return RouterStoreToken.
+ * @throws IOException io exception occurs.
+ */
+ private RouterStoreToken getStoreTokenFromZK(RouterRMTokenRequest request,
+ boolean quiet) throws IOException {
+ RouterStoreToken routerStoreToken = request.getRouterStoreToken();
+ YARNDelegationTokenIdentifier identifier = routerStoreToken.getTokenIdentifier();
+ return getStoreTokenFromZK(identifier, quiet);
+ }
+
+ /**
+ * Get RouterStoreToken from ZK.
+ *
+ * @param identifier YARN DelegationToken Identifier
+ * @param quiet Whether it is in quiet mode,
+ * if it is in quiet mode, no exception information will be output.
+ * @return RouterStoreToken.
+ * @throws IOException io exception occurs.
+ */
+ private RouterStoreToken getStoreTokenFromZK(YARNDelegationTokenIdentifier identifier,
+ boolean quiet) throws IOException {
+ // get the Token storage path
+ String nodePath = getStoreTokenZNodePathByIdentifier(identifier);
+ return getStoreTokenFromZK(nodePath, quiet);
+ }
+
+ /**
+ * Get RouterStoreToken from ZK.
+ *
+ * @param nodePath Znode location where data is stored.
+ * @param quiet Whether it is in quiet mode,
+ * if it is in quiet mode, no exception information will be output.
+ * @return RouterStoreToken.
+ * @throws IOException io exception occurs.
+ */
+ private RouterStoreToken getStoreTokenFromZK(String nodePath, boolean quiet)
+ throws IOException {
+ try {
+ byte[] data = get(nodePath);
+ if ((data == null) || (data.length == 0)) {
+ return null;
+ }
+ ByteArrayInputStream bin = new ByteArrayInputStream(data);
+ DataInputStream din = new DataInputStream(bin);
+ RouterStoreToken storeToken = Records.newRecord(RouterStoreToken.class);
+ storeToken.readFields(din);
+ return storeToken;
+ } catch (Exception ex) {
+ if (!quiet) {
+ LOG.error("No node in path [" + nodePath + "]");
Review Comment:
Use {}
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-issues-help@hadoop.apache.org
[GitHub] [hadoop] slfan1989 commented on a diff in pull request #5131: YARN-11350. [Federation] Router Support DelegationToken With ZK.
Posted by GitBox <gi...@apache.org>.
slfan1989 commented on code in PR #5131:
URL: https://github.com/apache/hadoop/pull/5131#discussion_r1036559681
##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterStoreToken.java:
##########
@@ -22,6 +22,7 @@
import org.apache.hadoop.yarn.security.client.YARNDelegationTokenIdentifier;
import org.apache.hadoop.yarn.util.Records;
+import java.io.DataInput;
Review Comment:
The method we completed in the impl defines the abstract method declaration in the abstract class.
`RouterStoreToken` is an abstract class, its impl class is `RouterStoreTokenPBImpl`, we have implemented these 2 methods.
RouterStoreToken#toByteArray and readFields
```
public abstract byte[] toByteArray() throws IOException;
public abstract void readFields(DataInput in) throws IOException;
```
RouterStoreTokenPBImpl#toByteArray and readFields
```
public byte[] toByteArray() throws IOException {
return builder.build().toByteArray();
}
public void readFields(DataInput in) throws IOException {
builder.mergeFrom((DataInputStream) in);
}
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-issues-help@hadoop.apache.org
[GitHub] [hadoop] slfan1989 commented on a diff in pull request #5131: YARN-11350. [Federation] Router Support DelegationToken With ZK.
Posted by GitBox <gi...@apache.org>.
slfan1989 commented on code in PR #5131:
URL: https://github.com/apache/hadoop/pull/5131#discussion_r1035905558
##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/records/RouterStoreToken.java:
##########
@@ -22,6 +22,7 @@
import org.apache.hadoop.yarn.security.client.YARNDelegationTokenIdentifier;
import org.apache.hadoop.yarn.util.Records;
+import java.io.DataInput;
Review Comment:
We used the `readFields` method, which requires `DataInput` parameters.
RouterStoreToken#readFields
```
public abstract void readFields(DataInput in) throws IOException;
```
When we store `RouterStoreToken`, we convert token into a byte array, and then store it in `ZK`
```
private void storeOrUpdateRouterRMDT(RouterRMTokenRequest request, boolean isUpdate)
throws Exception {
RouterStoreToken routerStoreToken = request.getRouterStoreToken();
String nodeCreatePath = getStoreTokenZNodePathByTokenRequest(request);
LOG.debug("nodeCreatePath = {}, isUpdate = {}", nodeCreatePath, isUpdate);
put(nodeCreatePath, routerStoreToken.toByteArray(), isUpdate);
}
```
When we need to read data, we convert the bytes array stored in `ZK` into `DataInputStream`, and then let `RouterStoreToken` parse the `required fields` from it.
```
private RouterStoreToken getStoreTokenFromZK(String nodePath, boolean quiet)
throws IOException {
try {
byte[] data = get(nodePath);
if ((data == null) || (data.length == 0)) {
return null;
}
ByteArrayInputStream bin = new ByteArrayInputStream(data);
DataInputStream din = new DataInputStream(bin);
RouterStoreToken storeToken = Records.newRecord(RouterStoreToken.class);
storeToken.readFields(din);
return storeToken;
} catch (Exception ex) {
if (!quiet) {
LOG.error("No node in path [" + nodePath + "]");
}
throw new IOException(ex);
}
}
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-issues-help@hadoop.apache.org
[GitHub] [hadoop] slfan1989 commented on a diff in pull request #5131: YARN-11350. [Federation] Router Support DelegationToken With ZK.
Posted by GitBox <gi...@apache.org>.
slfan1989 commented on code in PR #5131:
URL: https://github.com/apache/hadoop/pull/5131#discussion_r1030632183
##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/ZookeeperFederationStateStore.java:
##########
@@ -886,45 +975,554 @@ public UpdateReservationHomeSubClusterResponse updateReservationHomeSubCluster(
return UpdateReservationHomeSubClusterResponse.newInstance();
}
+ /**
+ * ZookeeperFederationStateStore Supports Store NewMasterKey.
+ *
+ * @param request The request contains RouterMasterKey, which is an abstraction for DelegationKey
+ * @return routerMasterKeyResponse, the response contains the RouterMasterKey.
+ * @throws YarnException if the call to the state store is unsuccessful.
+ * @throws IOException An IO Error occurred.
+ */
@Override
- public RouterMasterKeyResponse storeNewMasterKey(RouterMasterKeyRequest request)
+ public synchronized RouterMasterKeyResponse storeNewMasterKey(RouterMasterKeyRequest request)
throws YarnException, IOException {
- throw new NotImplementedException("Code is not implemented");
+
+ long start = clock.getTime();
+ // For the verification of the request, after passing the verification,
+ // the request and the internal objects will not be empty and can be used directly.
+ FederationRouterRMTokenInputValidator.validate(request);
+
+ // Parse the delegationKey from the request and get the ZK storage path.
+ DelegationKey delegationKey = convertMasterKeyToDelegationKey(request);
+ String nodeCreatePath = getMasterKeyZNodePathByDelegationKey(delegationKey);
+ LOG.debug("Storing RMDelegationKey_{}, ZkNodePath = {}.", delegationKey.getKeyId(),
+ nodeCreatePath);
+
+ // Write master key data to zk.
+ try(ByteArrayOutputStream os = new ByteArrayOutputStream();
+ DataOutputStream fsOut = new DataOutputStream(os)) {
+ delegationKey.write(fsOut);
+ put(nodeCreatePath, os.toByteArray(), false);
+ }
+
+ // Get the stored masterKey from zk.
+ RouterMasterKey masterKeyFromZK = getRouterMasterKeyFromZK(nodeCreatePath);
+ long end = clock.getTime();
+ opDurations.addStoreNewMasterKeyDuration(start, end);
+ return RouterMasterKeyResponse.newInstance(masterKeyFromZK);
}
+ /**
+ * ZookeeperFederationStateStore Supports Remove MasterKey.
+ *
+ * @param request The request contains RouterMasterKey, which is an abstraction for DelegationKey
+ * @return routerMasterKeyResponse, the response contains the RouterMasterKey.
+ * @throws YarnException if the call to the state store is unsuccessful.
+ * @throws IOException An IO Error occurred.
+ */
@Override
- public RouterMasterKeyResponse removeStoredMasterKey(RouterMasterKeyRequest request)
+ public synchronized RouterMasterKeyResponse removeStoredMasterKey(RouterMasterKeyRequest request)
throws YarnException, IOException {
- throw new NotImplementedException("Code is not implemented");
+
+ long start = clock.getTime();
+ // For the verification of the request, after passing the verification,
+ // the request and the internal objects will not be empty and can be used directly.
+ FederationRouterRMTokenInputValidator.validate(request);
+
+ try {
+ // Parse the delegationKey from the request and get the ZK storage path.
+ RouterMasterKey masterKey = request.getRouterMasterKey();
+ DelegationKey delegationKey = convertMasterKeyToDelegationKey(request);
+ String nodeRemovePath = getMasterKeyZNodePathByDelegationKey(delegationKey);
+ LOG.debug("Removing RMDelegationKey_{}, ZkNodePath = {}.", delegationKey.getKeyId(),
+ nodeRemovePath);
+
+ // Check if the path exists, Throws an exception if the path does not exist.
+ if (!exists(nodeRemovePath)) {
+ throw new YarnException("ZkNodePath = " + nodeRemovePath + " not exists!");
+ }
+
+ // try to remove masterKey.
+ zkManager.delete(nodeRemovePath);
+ long end = clock.getTime();
+ opDurations.removeStoredMasterKeyDuration(start, end);
+ return RouterMasterKeyResponse.newInstance(masterKey);
+ } catch (Exception e) {
+ throw new YarnException(e);
+ }
}
+ /**
+ * ZookeeperFederationStateStore Supports Remove MasterKey.
+ *
+ * @param request The request contains RouterMasterKey, which is an abstraction for DelegationKey
+ * @return routerMasterKeyResponse, the response contains the RouterMasterKey.
+ * @throws YarnException if the call to the state store is unsuccessful.
+ * @throws IOException An IO Error occurred.
+ */
@Override
public RouterMasterKeyResponse getMasterKeyByDelegationKey(RouterMasterKeyRequest request)
throws YarnException, IOException {
- throw new NotImplementedException("Code is not implemented");
+
+ long start = clock.getTime();
+ // For the verification of the request, after passing the verification,
+ // the request and the internal objects will not be empty and can be used directly.
+ FederationRouterRMTokenInputValidator.validate(request);
+
+ try {
+
+ // Parse the delegationKey from the request and get the ZK storage path.
+ DelegationKey delegationKey = convertMasterKeyToDelegationKey(request);
+ String nodePath = getMasterKeyZNodePathByDelegationKey(delegationKey);
+
+ // Check if the path exists, Throws an exception if the path does not exist.
+ if (!exists(nodePath)) {
+ throw new YarnException("ZkNodePath = " + nodePath + " not exists!");
+ }
+
+ // Get the stored masterKey from zk.
+ RouterMasterKey routerMasterKey = getRouterMasterKeyFromZK(nodePath);
+ long end = clock.getTime();
+ opDurations.getMasterKeyByDelegationKeyDuration(start, end);
+ return RouterMasterKeyResponse.newInstance(routerMasterKey);
+ } catch (Exception e) {
+ throw new YarnException(e);
+ }
+ }
+
+ /**
+ * Get MasterKeyZNodePath based on DelegationKey.
+ *
+ * @param delegationKey delegationKey.
+ * @return masterKey ZNodePath.
+ */
+ private String getMasterKeyZNodePathByDelegationKey(DelegationKey delegationKey) {
+ return getMasterKeyZNodePathByKeyId(delegationKey.getKeyId());
+ }
+
+ /**
+ * Get MasterKeyZNodePath based on KeyId.
+ *
+ * @param keyId master key id.
+ * @return masterKey ZNodePath.
+ */
+ private String getMasterKeyZNodePathByKeyId(int keyId) {
+ String nodeName = ROUTER_RM_DELEGATION_KEY_PREFIX + keyId;
+ return getNodePath(routerRMDTMasterKeysRootPath, nodeName);
+ }
+
+ /**
+ * Get RouterMasterKey from ZK.
+ *
+ * @param nodePath The path where masterKey is stored in zk.
+ *
+ * @return RouterMasterKey.
+ * @throws IOException An IO Error occurred.
+ */
+ private RouterMasterKey getRouterMasterKeyFromZK(String nodePath)
+ throws IOException {
+ try {
+ byte[] data = get(nodePath);
+ if ((data == null) || (data.length == 0)) {
+ return null;
+ }
+
+ ByteArrayInputStream bin = new ByteArrayInputStream(data);
+ DataInputStream din = new DataInputStream(bin);
+ DelegationKey key = new DelegationKey();
+ key.readFields(din);
+
+ return RouterMasterKey.newInstance(key.getKeyId(),
+ ByteBuffer.wrap(key.getEncodedKey()), key.getExpiryDate());
+ } catch (Exception ex) {
+ LOG.error("No node in path {}.", nodePath);
+ throw new IOException(ex);
+ }
}
+ /**
+ * ZookeeperFederationStateStore Supports Store RMDelegationTokenIdentifier.
+ *
+ * The stored token method is a synchronized method
+ * used to ensure that storeNewToken is a thread-safe method.
+ *
+ * @param request The request contains RouterRMToken (RMDelegationTokenIdentifier and renewDate)
+ * @return routerRMTokenResponse, the response contains the RouterStoreToken.
+ * @throws YarnException if the call to the state store is unsuccessful.
+ * @throws IOException An IO Error occurred.
+ */
@Override
- public RouterRMTokenResponse storeNewToken(RouterRMTokenRequest request)
+ public synchronized RouterRMTokenResponse storeNewToken(RouterRMTokenRequest request)
Review Comment:
Thank you very much for your help in reviewing the code. In ZookeeperFederationStateStore, we have 7 methods that use the synchronized keyword
1. Group1 methods are mainly used to create or remove MasterKey
- storeNewMasterKey
- removeStoredMasterKey
2. Group2 methods are mainly used to create or update and remove Token
- storeNewToken
- updateStoredToken
- removeStoredToken
3. Group3 methods are used to increment DelegationTokenSeqNum and MasterKeyId
- incrementDelegationTokenSeqNum
- incrementCurrentKeyId
This part is mainly to protect the ZNode of ZK. We hope that the same Router has only one thread to write, update and delete the same ZNode at the same time.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-issues-help@hadoop.apache.org
[GitHub] [hadoop] hadoop-yetus commented on pull request #5131: YARN-11350. [Federation] Router Support DelegationToken With ZK.
Posted by GitBox <gi...@apache.org>.
hadoop-yetus commented on PR #5131:
URL: https://github.com/apache/hadoop/pull/5131#issuecomment-1325605657
:broken_heart: **-1 overall**
| Vote | Subsystem | Runtime | Logfile | Comment |
|:----:|----------:|--------:|:--------:|:-------:|
| +0 :ok: | reexec | 0m 52s | | Docker mode activated. |
|||| _ Prechecks _ |
| +1 :green_heart: | dupname | 0m 0s | | No case conflicting files found. |
| +0 :ok: | codespell | 0m 0s | | codespell was not available. |
| +0 :ok: | detsecrets | 0m 0s | | detect-secrets was not available. |
| +1 :green_heart: | @author | 0m 0s | | The patch does not contain any @author tags. |
| +1 :green_heart: | test4tests | 0m 0s | | The patch appears to include 1 new or modified test files. |
|||| _ trunk Compile Tests _ |
| +0 :ok: | mvndep | 15m 29s | | Maven dependency ordering for branch |
| +1 :green_heart: | mvninstall | 26m 11s | | trunk passed |
| +1 :green_heart: | compile | 3m 59s | | trunk passed with JDK Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 |
| +1 :green_heart: | compile | 3m 32s | | trunk passed with JDK Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | checkstyle | 1m 23s | | trunk passed |
| +1 :green_heart: | mvnsite | 2m 54s | | trunk passed |
| +1 :green_heart: | javadoc | 2m 39s | | trunk passed with JDK Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 |
| +1 :green_heart: | javadoc | 2m 18s | | trunk passed with JDK Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | spotbugs | 4m 58s | | trunk passed |
| +1 :green_heart: | shadedclient | 20m 41s | | branch has no errors when building and testing our client artifacts. |
| -0 :warning: | patch | 21m 7s | | Used diff version of patch file. Binary files and potentially other changes not applied. Please rebase and squash commits if necessary. |
|||| _ Patch Compile Tests _ |
| +0 :ok: | mvndep | 0m 30s | | Maven dependency ordering for patch |
| +1 :green_heart: | mvninstall | 2m 5s | | the patch passed |
| +1 :green_heart: | compile | 3m 52s | | the patch passed with JDK Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 |
| +1 :green_heart: | javac | 3m 52s | | the patch passed |
| +1 :green_heart: | compile | 3m 16s | | the patch passed with JDK Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | javac | 3m 16s | | the patch passed |
| +1 :green_heart: | blanks | 0m 0s | | The patch has no blanks issues. |
| +1 :green_heart: | checkstyle | 1m 9s | | the patch passed |
| +1 :green_heart: | mvnsite | 2m 12s | | the patch passed |
| -1 :x: | javadoc | 0m 35s | [/patch-javadoc-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-common-jdkUbuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/11/artifact/out/patch-javadoc-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-common-jdkUbuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04.txt) | hadoop-yarn-server-common in the patch failed with JDK Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04. |
| -1 :x: | javadoc | 0m 39s | [/patch-javadoc-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-common-jdkPrivateBuild-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/11/artifact/out/patch-javadoc-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-common-jdkPrivateBuild-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07.txt) | hadoop-yarn-server-common in the patch failed with JDK Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07. |
| -1 :x: | spotbugs | 1m 39s | [/new-spotbugs-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-common.html](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/11/artifact/out/new-spotbugs-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-common.html) | hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common generated 1 new + 0 unchanged - 0 fixed = 1 total (was 0) |
| +1 :green_heart: | shadedclient | 20m 39s | | patch has no errors when building and testing our client artifacts. |
|||| _ Other Tests _ |
| +1 :green_heart: | unit | 3m 19s | | hadoop-yarn-server-common in the patch passed. |
| +1 :green_heart: | unit | 99m 47s | | hadoop-yarn-server-resourcemanager in the patch passed. |
| +1 :green_heart: | unit | 0m 39s | | hadoop-yarn-server-router in the patch passed. |
| +1 :green_heart: | asflicense | 0m 45s | | The patch does not generate ASF License warnings. |
| | | 234m 0s | | |
| Reason | Tests |
|-------:|:------|
| SpotBugs | module:hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common |
| | org.apache.hadoop.yarn.server.federation.store.impl.MemoryFederationStateStore.getDelegationTokenSeqNum() is unsynchronized, org.apache.hadoop.yarn.server.federation.store.impl.MemoryFederationStateStore.setDelegationTokenSeqNum(int) is synchronized At MemoryFederationStateStore.java:synchronized At MemoryFederationStateStore.java:[line 549] |
| Subsystem | Report/Notes |
|----------:|:-------------|
| Docker | ClientAPI=1.41 ServerAPI=1.41 base: https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/11/artifact/out/Dockerfile |
| GITHUB PR | https://github.com/apache/hadoop/pull/5131 |
| Optional Tests | dupname asflicense compile javac javadoc mvninstall mvnsite unit shadedclient spotbugs checkstyle codespell detsecrets |
| uname | Linux 6562f6da4fd2 4.15.0-191-generic #202-Ubuntu SMP Thu Aug 4 01:49:29 UTC 2022 x86_64 x86_64 x86_64 GNU/Linux |
| Build tool | maven |
| Personality | dev-support/bin/hadoop.sh |
| git revision | trunk / e33a2260ffc1f2d4314138287e21b5be8721e13c |
| Default Java | Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| Multi-JDK versions | /usr/lib/jvm/java-11-openjdk-amd64:Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 /usr/lib/jvm/java-8-openjdk-amd64:Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| Test Results | https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/11/testReport/ |
| Max. process+thread count | 941 (vs. ulimit of 5500) |
| modules | C: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router U: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server |
| Console output | https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/11/console |
| versions | git=2.25.1 maven=3.6.3 spotbugs=4.2.2 |
| Powered by | Apache Yetus 0.14.0 https://yetus.apache.org |
This message was automatically generated.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-issues-help@hadoop.apache.org
[GitHub] [hadoop] hadoop-yetus commented on pull request #5131: YARN-11350. [Federation] Router Support DelegationToken With ZK.
Posted by GitBox <gi...@apache.org>.
hadoop-yetus commented on PR #5131:
URL: https://github.com/apache/hadoop/pull/5131#issuecomment-1350390825
:broken_heart: **-1 overall**
| Vote | Subsystem | Runtime | Logfile | Comment |
|:----:|----------:|--------:|:--------:|:-------:|
| +0 :ok: | reexec | 0m 43s | | Docker mode activated. |
|||| _ Prechecks _ |
| +1 :green_heart: | dupname | 0m 1s | | No case conflicting files found. |
| +0 :ok: | codespell | 0m 0s | | codespell was not available. |
| +0 :ok: | detsecrets | 0m 0s | | detect-secrets was not available. |
| +1 :green_heart: | @author | 0m 0s | | The patch does not contain any @author tags. |
| +1 :green_heart: | test4tests | 0m 0s | | The patch appears to include 4 new or modified test files. |
|||| _ trunk Compile Tests _ |
| +0 :ok: | mvndep | 16m 21s | | Maven dependency ordering for branch |
| +1 :green_heart: | mvninstall | 26m 4s | | trunk passed |
| +1 :green_heart: | compile | 3m 51s | | trunk passed with JDK Ubuntu-11.0.17+8-post-Ubuntu-1ubuntu220.04 |
| +1 :green_heart: | compile | 3m 16s | | trunk passed with JDK Private Build-1.8.0_352-8u352-ga-1~20.04-b08 |
| +1 :green_heart: | checkstyle | 1m 17s | | trunk passed |
| +1 :green_heart: | mvnsite | 2m 23s | | trunk passed |
| -1 :x: | javadoc | 0m 47s | [/branch-javadoc-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-resourcemanager-jdkUbuntu-11.0.17+8-post-Ubuntu-1ubuntu220.04.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/23/artifact/out/branch-javadoc-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-resourcemanager-jdkUbuntu-11.0.17+8-post-Ubuntu-1ubuntu220.04.txt) | hadoop-yarn-server-resourcemanager in trunk failed with JDK Ubuntu-11.0.17+8-post-Ubuntu-1ubuntu220.04. |
| +1 :green_heart: | javadoc | 1m 45s | | trunk passed with JDK Private Build-1.8.0_352-8u352-ga-1~20.04-b08 |
| +1 :green_heart: | spotbugs | 4m 17s | | trunk passed |
| +1 :green_heart: | shadedclient | 20m 2s | | branch has no errors when building and testing our client artifacts. |
| -0 :warning: | patch | 20m 22s | | Used diff version of patch file. Binary files and potentially other changes not applied. Please rebase and squash commits if necessary. |
|||| _ Patch Compile Tests _ |
| +0 :ok: | mvndep | 0m 28s | | Maven dependency ordering for patch |
| +1 :green_heart: | mvninstall | 1m 52s | | the patch passed |
| +1 :green_heart: | compile | 3m 44s | | the patch passed with JDK Ubuntu-11.0.17+8-post-Ubuntu-1ubuntu220.04 |
| +1 :green_heart: | javac | 3m 44s | | the patch passed |
| +1 :green_heart: | compile | 3m 11s | | the patch passed with JDK Private Build-1.8.0_352-8u352-ga-1~20.04-b08 |
| +1 :green_heart: | javac | 3m 11s | | the patch passed |
| +1 :green_heart: | blanks | 0m 0s | | The patch has no blanks issues. |
| +1 :green_heart: | checkstyle | 1m 2s | | the patch passed |
| +1 :green_heart: | mvnsite | 1m 59s | | the patch passed |
| -1 :x: | javadoc | 0m 41s | [/patch-javadoc-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-resourcemanager-jdkUbuntu-11.0.17+8-post-Ubuntu-1ubuntu220.04.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/23/artifact/out/patch-javadoc-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-resourcemanager-jdkUbuntu-11.0.17+8-post-Ubuntu-1ubuntu220.04.txt) | hadoop-yarn-server-resourcemanager in the patch failed with JDK Ubuntu-11.0.17+8-post-Ubuntu-1ubuntu220.04. |
| +1 :green_heart: | javadoc | 1m 29s | | the patch passed with JDK Private Build-1.8.0_352-8u352-ga-1~20.04-b08 |
| +1 :green_heart: | spotbugs | 4m 18s | | the patch passed |
| +1 :green_heart: | shadedclient | 20m 5s | | patch has no errors when building and testing our client artifacts. |
|||| _ Other Tests _ |
| +1 :green_heart: | unit | 3m 4s | | hadoop-yarn-server-common in the patch passed. |
| +1 :green_heart: | unit | 98m 54s | | hadoop-yarn-server-resourcemanager in the patch passed. |
| +1 :green_heart: | unit | 0m 32s | | hadoop-yarn-server-router in the patch passed. |
| +1 :green_heart: | asflicense | 0m 38s | | The patch does not generate ASF License warnings. |
| | | 227m 32s | | |
| Subsystem | Report/Notes |
|----------:|:-------------|
| Docker | ClientAPI=1.41 ServerAPI=1.41 base: https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/23/artifact/out/Dockerfile |
| GITHUB PR | https://github.com/apache/hadoop/pull/5131 |
| Optional Tests | dupname asflicense compile javac javadoc mvninstall mvnsite unit shadedclient spotbugs checkstyle codespell detsecrets |
| uname | Linux 83c73a4b2cd3 4.15.0-200-generic #211-Ubuntu SMP Thu Nov 24 18:16:04 UTC 2022 x86_64 x86_64 x86_64 GNU/Linux |
| Build tool | maven |
| Personality | dev-support/bin/hadoop.sh |
| git revision | trunk / a4f40ddbcb508eb02fc0e1486ebcac7221e1db3a |
| Default Java | Private Build-1.8.0_352-8u352-ga-1~20.04-b08 |
| Multi-JDK versions | /usr/lib/jvm/java-11-openjdk-amd64:Ubuntu-11.0.17+8-post-Ubuntu-1ubuntu220.04 /usr/lib/jvm/java-8-openjdk-amd64:Private Build-1.8.0_352-8u352-ga-1~20.04-b08 |
| Test Results | https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/23/testReport/ |
| Max. process+thread count | 967 (vs. ulimit of 5500) |
| modules | C: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router U: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server |
| Console output | https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/23/console |
| versions | git=2.25.1 maven=3.6.3 spotbugs=4.2.2 |
| Powered by | Apache Yetus 0.14.0 https://yetus.apache.org |
This message was automatically generated.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-issues-help@hadoop.apache.org
[GitHub] [hadoop] hadoop-yetus commented on pull request #5131: YARN-11350. [Federation] Router Support DelegationToken With ZK.
Posted by GitBox <gi...@apache.org>.
hadoop-yetus commented on PR #5131:
URL: https://github.com/apache/hadoop/pull/5131#issuecomment-1313480954
:broken_heart: **-1 overall**
| Vote | Subsystem | Runtime | Logfile | Comment |
|:----:|----------:|--------:|:--------:|:-------:|
| +0 :ok: | reexec | 0m 41s | | Docker mode activated. |
|||| _ Prechecks _ |
| +1 :green_heart: | dupname | 0m 0s | | No case conflicting files found. |
| +0 :ok: | codespell | 0m 1s | | codespell was not available. |
| +0 :ok: | detsecrets | 0m 1s | | detect-secrets was not available. |
| +1 :green_heart: | @author | 0m 0s | | The patch does not contain any @author tags. |
| +1 :green_heart: | test4tests | 0m 0s | | The patch appears to include 3 new or modified test files. |
|||| _ trunk Compile Tests _ |
| +0 :ok: | mvndep | 15m 26s | | Maven dependency ordering for branch |
| +1 :green_heart: | mvninstall | 25m 56s | | trunk passed |
| +1 :green_heart: | compile | 4m 11s | | trunk passed with JDK Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 |
| +1 :green_heart: | compile | 3m 26s | | trunk passed with JDK Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | checkstyle | 1m 23s | | trunk passed |
| +1 :green_heart: | mvnsite | 2m 55s | | trunk passed |
| +1 :green_heart: | javadoc | 2m 34s | | trunk passed with JDK Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 |
| +1 :green_heart: | javadoc | 2m 11s | | trunk passed with JDK Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | spotbugs | 4m 56s | | trunk passed |
| +1 :green_heart: | shadedclient | 20m 53s | | branch has no errors when building and testing our client artifacts. |
|||| _ Patch Compile Tests _ |
| +0 :ok: | mvndep | 0m 34s | | Maven dependency ordering for patch |
| +1 :green_heart: | mvninstall | 2m 9s | | the patch passed |
| +1 :green_heart: | compile | 3m 48s | | the patch passed with JDK Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 |
| +1 :green_heart: | javac | 3m 48s | | the patch passed |
| +1 :green_heart: | compile | 3m 19s | | the patch passed with JDK Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | javac | 3m 19s | | the patch passed |
| +1 :green_heart: | blanks | 0m 0s | | The patch has no blanks issues. |
| +1 :green_heart: | checkstyle | 1m 11s | | the patch passed |
| +1 :green_heart: | mvnsite | 2m 14s | | the patch passed |
| -1 :x: | javadoc | 0m 56s | [/patch-javadoc-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-common-jdkUbuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/1/artifact/out/patch-javadoc-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-common-jdkUbuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04.txt) | hadoop-yarn-server-common in the patch failed with JDK Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04. |
| -1 :x: | javadoc | 0m 32s | [/patch-javadoc-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-common-jdkPrivateBuild-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/1/artifact/out/patch-javadoc-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-common-jdkPrivateBuild-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07.txt) | hadoop-yarn-server-common in the patch failed with JDK Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07. |
| +1 :green_heart: | spotbugs | 4m 41s | | the patch passed |
| +1 :green_heart: | shadedclient | 22m 35s | | patch has no errors when building and testing our client artifacts. |
|||| _ Other Tests _ |
| +1 :green_heart: | unit | 3m 19s | | hadoop-yarn-server-common in the patch passed. |
| +1 :green_heart: | unit | 99m 10s | | hadoop-yarn-server-resourcemanager in the patch passed. |
| +1 :green_heart: | unit | 5m 54s | | hadoop-yarn-server-router in the patch passed. |
| +1 :green_heart: | asflicense | 0m 43s | | The patch does not generate ASF License warnings. |
| | | 241m 2s | | |
| Subsystem | Report/Notes |
|----------:|:-------------|
| Docker | ClientAPI=1.41 ServerAPI=1.41 base: https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/1/artifact/out/Dockerfile |
| GITHUB PR | https://github.com/apache/hadoop/pull/5131 |
| Optional Tests | dupname asflicense compile javac javadoc mvninstall mvnsite unit shadedclient spotbugs checkstyle codespell detsecrets |
| uname | Linux a86ea8d2a4b8 4.15.0-191-generic #202-Ubuntu SMP Thu Aug 4 01:49:29 UTC 2022 x86_64 x86_64 x86_64 GNU/Linux |
| Build tool | maven |
| Personality | dev-support/bin/hadoop.sh |
| git revision | trunk / 1afa646ee8006c7f833ea1778dbb65d68bdb536d |
| Default Java | Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| Multi-JDK versions | /usr/lib/jvm/java-11-openjdk-amd64:Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 /usr/lib/jvm/java-8-openjdk-amd64:Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| Test Results | https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/1/testReport/ |
| Max. process+thread count | 964 (vs. ulimit of 5500) |
| modules | C: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router U: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server |
| Console output | https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/1/console |
| versions | git=2.25.1 maven=3.6.3 spotbugs=4.2.2 |
| Powered by | Apache Yetus 0.14.0 https://yetus.apache.org |
This message was automatically generated.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-issues-help@hadoop.apache.org
[GitHub] [hadoop] slfan1989 commented on pull request #5131: YARN-11350. [Federation] Router Support DelegationToken With ZK.
Posted by GitBox <gi...@apache.org>.
slfan1989 commented on PR #5131:
URL: https://github.com/apache/hadoop/pull/5131#issuecomment-1346671056
@goiri Can you help to merge this pr into the trunk branch? Thank you very much! We have fixed all java doc issues for `hadoop-yarn-server-common`, but for `hadoop-yarn-server-resourcemanager` this module is not affected by our changes. After this pr is completed, I will improve YARN-11349 as soon as possible.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-issues-help@hadoop.apache.org
[GitHub] [hadoop] hadoop-yetus commented on pull request #5131: YARN-11350. [Federation] Router Support DelegationToken With ZK.
Posted by GitBox <gi...@apache.org>.
hadoop-yetus commented on PR #5131:
URL: https://github.com/apache/hadoop/pull/5131#issuecomment-1321858433
:confetti_ball: **+1 overall**
| Vote | Subsystem | Runtime | Logfile | Comment |
|:----:|----------:|--------:|:--------:|:-------:|
| +0 :ok: | reexec | 0m 47s | | Docker mode activated. |
|||| _ Prechecks _ |
| +1 :green_heart: | dupname | 0m 0s | | No case conflicting files found. |
| +0 :ok: | codespell | 0m 0s | | codespell was not available. |
| +0 :ok: | detsecrets | 0m 0s | | detect-secrets was not available. |
| +1 :green_heart: | @author | 0m 0s | | The patch does not contain any @author tags. |
| +1 :green_heart: | test4tests | 0m 0s | | The patch appears to include 3 new or modified test files. |
|||| _ trunk Compile Tests _ |
| +0 :ok: | mvndep | 15m 31s | | Maven dependency ordering for branch |
| +1 :green_heart: | mvninstall | 26m 2s | | trunk passed |
| +1 :green_heart: | compile | 4m 7s | | trunk passed with JDK Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 |
| +1 :green_heart: | compile | 3m 26s | | trunk passed with JDK Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | checkstyle | 1m 29s | | trunk passed |
| +1 :green_heart: | mvnsite | 2m 48s | | trunk passed |
| +1 :green_heart: | javadoc | 2m 40s | | trunk passed with JDK Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 |
| +1 :green_heart: | javadoc | 2m 15s | | trunk passed with JDK Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | spotbugs | 4m 47s | | trunk passed |
| +1 :green_heart: | shadedclient | 20m 55s | | branch has no errors when building and testing our client artifacts. |
|||| _ Patch Compile Tests _ |
| +0 :ok: | mvndep | 0m 30s | | Maven dependency ordering for patch |
| +1 :green_heart: | mvninstall | 2m 4s | | the patch passed |
| +1 :green_heart: | compile | 4m 21s | | the patch passed with JDK Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 |
| +1 :green_heart: | javac | 4m 21s | | the patch passed |
| +1 :green_heart: | compile | 3m 32s | | the patch passed with JDK Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | javac | 3m 32s | | the patch passed |
| +1 :green_heart: | blanks | 0m 1s | | The patch has no blanks issues. |
| +1 :green_heart: | checkstyle | 1m 11s | | the patch passed |
| +1 :green_heart: | mvnsite | 2m 25s | | the patch passed |
| +1 :green_heart: | javadoc | 0m 38s | | hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-common-jdkUbuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 with JDK Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 generated 0 new + 47 unchanged - 1 fixed = 47 total (was 48) |
| +1 :green_heart: | javadoc | 0m 51s | | hadoop-yarn-server-resourcemanager in the patch passed with JDK Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04. |
| +1 :green_heart: | javadoc | 0m 31s | | hadoop-yarn-server-router in the patch passed with JDK Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04. |
| +1 :green_heart: | javadoc | 0m 36s | | hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-common-jdkPrivateBuild-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 with JDK Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 generated 0 new + 47 unchanged - 1 fixed = 47 total (was 48) |
| +1 :green_heart: | javadoc | 0m 46s | | hadoop-yarn-server-resourcemanager in the patch passed with JDK Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07. |
| +1 :green_heart: | javadoc | 0m 25s | | hadoop-yarn-server-router in the patch passed with JDK Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07. |
| +1 :green_heart: | spotbugs | 4m 57s | | the patch passed |
| +1 :green_heart: | shadedclient | 20m 40s | | patch has no errors when building and testing our client artifacts. |
|||| _ Other Tests _ |
| +1 :green_heart: | unit | 3m 13s | | hadoop-yarn-server-common in the patch passed. |
| +1 :green_heart: | unit | 99m 21s | | hadoop-yarn-server-resourcemanager in the patch passed. |
| +1 :green_heart: | unit | 0m 38s | | hadoop-yarn-server-router in the patch passed. |
| +1 :green_heart: | asflicense | 0m 51s | | The patch does not generate ASF License warnings. |
| | | 234m 43s | | |
| Subsystem | Report/Notes |
|----------:|:-------------|
| Docker | ClientAPI=1.41 ServerAPI=1.41 base: https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/8/artifact/out/Dockerfile |
| GITHUB PR | https://github.com/apache/hadoop/pull/5131 |
| Optional Tests | dupname asflicense compile javac javadoc mvninstall mvnsite unit shadedclient spotbugs checkstyle codespell detsecrets |
| uname | Linux aac36fedc798 4.15.0-191-generic #202-Ubuntu SMP Thu Aug 4 01:49:29 UTC 2022 x86_64 x86_64 x86_64 GNU/Linux |
| Build tool | maven |
| Personality | dev-support/bin/hadoop.sh |
| git revision | trunk / e0087be66fc36296035b26968c04be2046318636 |
| Default Java | Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| Multi-JDK versions | /usr/lib/jvm/java-11-openjdk-amd64:Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 /usr/lib/jvm/java-8-openjdk-amd64:Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| Test Results | https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/8/testReport/ |
| Max. process+thread count | 968 (vs. ulimit of 5500) |
| modules | C: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router U: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server |
| Console output | https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/8/console |
| versions | git=2.25.1 maven=3.6.3 spotbugs=4.2.2 |
| Powered by | Apache Yetus 0.14.0 https://yetus.apache.org |
This message was automatically generated.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-issues-help@hadoop.apache.org
[GitHub] [hadoop] hadoop-yetus commented on pull request #5131: YARN-11350. [Federation] Router Support DelegationToken With ZK.
Posted by GitBox <gi...@apache.org>.
hadoop-yetus commented on PR #5131:
URL: https://github.com/apache/hadoop/pull/5131#issuecomment-1334094867
:broken_heart: **-1 overall**
| Vote | Subsystem | Runtime | Logfile | Comment |
|:----:|----------:|--------:|:--------:|:-------:|
| +0 :ok: | reexec | 0m 37s | | Docker mode activated. |
|||| _ Prechecks _ |
| +1 :green_heart: | dupname | 0m 0s | | No case conflicting files found. |
| +0 :ok: | codespell | 0m 1s | | codespell was not available. |
| +0 :ok: | detsecrets | 0m 1s | | detect-secrets was not available. |
| +1 :green_heart: | @author | 0m 0s | | The patch does not contain any @author tags. |
| +1 :green_heart: | test4tests | 0m 0s | | The patch appears to include 1 new or modified test files. |
|||| _ trunk Compile Tests _ |
| +0 :ok: | mvndep | 15m 30s | | Maven dependency ordering for branch |
| +1 :green_heart: | mvninstall | 26m 9s | | trunk passed |
| +1 :green_heart: | compile | 4m 1s | | trunk passed with JDK Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 |
| +1 :green_heart: | compile | 3m 32s | | trunk passed with JDK Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | checkstyle | 1m 20s | | trunk passed |
| +1 :green_heart: | mvnsite | 2m 44s | | trunk passed |
| +1 :green_heart: | javadoc | 2m 38s | | trunk passed with JDK Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 |
| +1 :green_heart: | javadoc | 2m 15s | | trunk passed with JDK Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | spotbugs | 4m 48s | | trunk passed |
| +1 :green_heart: | shadedclient | 23m 41s | | branch has no errors when building and testing our client artifacts. |
| -0 :warning: | patch | 24m 6s | | Used diff version of patch file. Binary files and potentially other changes not applied. Please rebase and squash commits if necessary. |
|||| _ Patch Compile Tests _ |
| +0 :ok: | mvndep | 0m 28s | | Maven dependency ordering for patch |
| +1 :green_heart: | mvninstall | 2m 6s | | the patch passed |
| +1 :green_heart: | compile | 3m 59s | | the patch passed with JDK Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 |
| +1 :green_heart: | javac | 3m 59s | | the patch passed |
| +1 :green_heart: | compile | 3m 26s | | the patch passed with JDK Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| +1 :green_heart: | javac | 3m 26s | | the patch passed |
| +1 :green_heart: | blanks | 0m 1s | | The patch has no blanks issues. |
| -0 :warning: | checkstyle | 1m 10s | [/results-checkstyle-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/14/artifact/out/results-checkstyle-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server.txt) | hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server: The patch generated 1 new + 10 unchanged - 0 fixed = 11 total (was 10) |
| +1 :green_heart: | mvnsite | 2m 21s | | the patch passed |
| -1 :x: | javadoc | 0m 42s | [/patch-javadoc-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-common-jdkUbuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/14/artifact/out/patch-javadoc-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-common-jdkUbuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04.txt) | hadoop-yarn-server-common in the patch failed with JDK Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04. |
| -1 :x: | javadoc | 0m 38s | [/patch-javadoc-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-common-jdkPrivateBuild-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/14/artifact/out/patch-javadoc-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-common-jdkPrivateBuild-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07.txt) | hadoop-yarn-server-common in the patch failed with JDK Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07. |
| +1 :green_heart: | spotbugs | 4m 30s | | the patch passed |
| +1 :green_heart: | shadedclient | 24m 52s | | patch has no errors when building and testing our client artifacts. |
|||| _ Other Tests _ |
| +1 :green_heart: | unit | 3m 8s | | hadoop-yarn-server-common in the patch passed. |
| +1 :green_heart: | unit | 99m 4s | | hadoop-yarn-server-resourcemanager in the patch passed. |
| +1 :green_heart: | unit | 0m 36s | | hadoop-yarn-server-router in the patch passed. |
| +1 :green_heart: | asflicense | 0m 42s | | The patch does not generate ASF License warnings. |
| | | 239m 56s | | |
| Subsystem | Report/Notes |
|----------:|:-------------|
| Docker | ClientAPI=1.41 ServerAPI=1.41 base: https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/14/artifact/out/Dockerfile |
| GITHUB PR | https://github.com/apache/hadoop/pull/5131 |
| Optional Tests | dupname asflicense compile javac javadoc mvninstall mvnsite unit shadedclient spotbugs checkstyle codespell detsecrets |
| uname | Linux f78482bbcd55 4.15.0-191-generic #202-Ubuntu SMP Thu Aug 4 01:49:29 UTC 2022 x86_64 x86_64 x86_64 GNU/Linux |
| Build tool | maven |
| Personality | dev-support/bin/hadoop.sh |
| git revision | trunk / d7b5b3dae5cc89c1271686f07aff9830fbea8439 |
| Default Java | Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| Multi-JDK versions | /usr/lib/jvm/java-11-openjdk-amd64:Ubuntu-11.0.16+8-post-Ubuntu-0ubuntu120.04 /usr/lib/jvm/java-8-openjdk-amd64:Private Build-1.8.0_342-8u342-b07-0ubuntu1~20.04-b07 |
| Test Results | https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/14/testReport/ |
| Max. process+thread count | 959 (vs. ulimit of 5500) |
| modules | C: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router U: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server |
| Console output | https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/14/console |
| versions | git=2.25.1 maven=3.6.3 spotbugs=4.2.2 |
| Powered by | Apache Yetus 0.14.0 https://yetus.apache.org |
This message was automatically generated.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-issues-help@hadoop.apache.org
[GitHub] [hadoop] slfan1989 commented on a diff in pull request #5131: YARN-11350. [Federation] Router Support DelegationToken With ZK.
Posted by GitBox <gi...@apache.org>.
slfan1989 commented on code in PR #5131:
URL: https://github.com/apache/hadoop/pull/5131#discussion_r1023980706
##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/FederationStateStoreBaseTest.java:
##########
@@ -879,7 +879,7 @@ public void testStoreNewMasterKey() throws Exception {
}
@Test
- public void testGetMasterKeyByDelegationKey() throws YarnException, IOException {
+ public void testGetMasterKeyByDelegationKey() throws Exception {
Review Comment:
The exception thrown by the method signature, I will recover. In the process of writing JunitTest code, I used `curatorFramework.getData().forPath` method to get data from ZK. This method throws an `Exception`, so I modified the exception throwing code of the method signature.
```
/**
* Commit the currently building operation using the given path
*
* @param path the path
* @return operation result if any
* @throws Exception errors
*/
public T forPath(String path) throws Exception;
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-issues-help@hadoop.apache.org
[GitHub] [hadoop] hadoop-yetus commented on pull request #5131: YARN-11350. [Federation] Router Support DelegationToken With ZK.
Posted by GitBox <gi...@apache.org>.
hadoop-yetus commented on PR #5131:
URL: https://github.com/apache/hadoop/pull/5131#issuecomment-1337902240
:broken_heart: **-1 overall**
| Vote | Subsystem | Runtime | Logfile | Comment |
|:----:|----------:|--------:|:--------:|:-------:|
| +0 :ok: | reexec | 0m 36s | | Docker mode activated. |
|||| _ Prechecks _ |
| +1 :green_heart: | dupname | 0m 0s | | No case conflicting files found. |
| +0 :ok: | codespell | 0m 1s | | codespell was not available. |
| +0 :ok: | detsecrets | 0m 1s | | detect-secrets was not available. |
| +1 :green_heart: | @author | 0m 0s | | The patch does not contain any @author tags. |
| +1 :green_heart: | test4tests | 0m 0s | | The patch appears to include 1 new or modified test files. |
|||| _ trunk Compile Tests _ |
| +0 :ok: | mvndep | 17m 10s | | Maven dependency ordering for branch |
| +1 :green_heart: | mvninstall | 26m 54s | | trunk passed |
| +1 :green_heart: | compile | 4m 4s | | trunk passed with JDK Ubuntu-11.0.17+8-post-Ubuntu-1ubuntu220.04 |
| +1 :green_heart: | compile | 3m 28s | | trunk passed with JDK Private Build-1.8.0_352-8u352-ga-1~20.04-b08 |
| +1 :green_heart: | checkstyle | 1m 29s | | trunk passed |
| +1 :green_heart: | mvnsite | 2m 55s | | trunk passed |
| -1 :x: | javadoc | 0m 51s | [/branch-javadoc-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-common-jdkUbuntu-11.0.17+8-post-Ubuntu-1ubuntu220.04.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/16/artifact/out/branch-javadoc-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-common-jdkUbuntu-11.0.17+8-post-Ubuntu-1ubuntu220.04.txt) | hadoop-yarn-server-common in trunk failed with JDK Ubuntu-11.0.17+8-post-Ubuntu-1ubuntu220.04. |
| -1 :x: | javadoc | 0m 54s | [/branch-javadoc-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-resourcemanager-jdkUbuntu-11.0.17+8-post-Ubuntu-1ubuntu220.04.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/16/artifact/out/branch-javadoc-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-resourcemanager-jdkUbuntu-11.0.17+8-post-Ubuntu-1ubuntu220.04.txt) | hadoop-yarn-server-resourcemanager in trunk failed with JDK Ubuntu-11.0.17+8-post-Ubuntu-1ubuntu220.04. |
| +1 :green_heart: | javadoc | 2m 9s | | trunk passed with JDK Private Build-1.8.0_352-8u352-ga-1~20.04-b08 |
| +1 :green_heart: | spotbugs | 4m 35s | | trunk passed |
| +1 :green_heart: | shadedclient | 20m 33s | | branch has no errors when building and testing our client artifacts. |
| -0 :warning: | patch | 20m 56s | | Used diff version of patch file. Binary files and potentially other changes not applied. Please rebase and squash commits if necessary. |
|||| _ Patch Compile Tests _ |
| +0 :ok: | mvndep | 0m 27s | | Maven dependency ordering for patch |
| +1 :green_heart: | mvninstall | 2m 3s | | the patch passed |
| +1 :green_heart: | compile | 3m 47s | | the patch passed with JDK Ubuntu-11.0.17+8-post-Ubuntu-1ubuntu220.04 |
| +1 :green_heart: | javac | 3m 47s | | the patch passed |
| +1 :green_heart: | compile | 3m 21s | | the patch passed with JDK Private Build-1.8.0_352-8u352-ga-1~20.04-b08 |
| +1 :green_heart: | javac | 3m 21s | | the patch passed |
| +1 :green_heart: | blanks | 0m 0s | | The patch has no blanks issues. |
| -0 :warning: | checkstyle | 1m 7s | [/results-checkstyle-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/16/artifact/out/results-checkstyle-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server.txt) | hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server: The patch generated 1 new + 10 unchanged - 0 fixed = 11 total (was 10) |
| +1 :green_heart: | mvnsite | 2m 15s | | the patch passed |
| -1 :x: | javadoc | 0m 34s | [/patch-javadoc-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-common-jdkUbuntu-11.0.17+8-post-Ubuntu-1ubuntu220.04.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/16/artifact/out/patch-javadoc-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-common-jdkUbuntu-11.0.17+8-post-Ubuntu-1ubuntu220.04.txt) | hadoop-yarn-server-common in the patch failed with JDK Ubuntu-11.0.17+8-post-Ubuntu-1ubuntu220.04. |
| -1 :x: | javadoc | 0m 44s | [/patch-javadoc-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-resourcemanager-jdkUbuntu-11.0.17+8-post-Ubuntu-1ubuntu220.04.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/16/artifact/out/patch-javadoc-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-resourcemanager-jdkUbuntu-11.0.17+8-post-Ubuntu-1ubuntu220.04.txt) | hadoop-yarn-server-resourcemanager in the patch failed with JDK Ubuntu-11.0.17+8-post-Ubuntu-1ubuntu220.04. |
| -1 :x: | javadoc | 0m 35s | [/patch-javadoc-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-common-jdkPrivateBuild-1.8.0_352-8u352-ga-1~20.04-b08.txt](https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/16/artifact/out/patch-javadoc-hadoop-yarn-project_hadoop-yarn_hadoop-yarn-server_hadoop-yarn-server-common-jdkPrivateBuild-1.8.0_352-8u352-ga-1~20.04-b08.txt) | hadoop-yarn-server-common in the patch failed with JDK Private Build-1.8.0_352-8u352-ga-1~20.04-b08. |
| +1 :green_heart: | spotbugs | 4m 34s | | the patch passed |
| +1 :green_heart: | shadedclient | 20m 32s | | patch has no errors when building and testing our client artifacts. |
|||| _ Other Tests _ |
| +1 :green_heart: | unit | 3m 16s | | hadoop-yarn-server-common in the patch passed. |
| +1 :green_heart: | unit | 98m 57s | | hadoop-yarn-server-resourcemanager in the patch passed. |
| +1 :green_heart: | unit | 0m 36s | | hadoop-yarn-server-router in the patch passed. |
| +1 :green_heart: | asflicense | 0m 41s | | The patch does not generate ASF License warnings. |
| | | 234m 31s | | |
| Subsystem | Report/Notes |
|----------:|:-------------|
| Docker | ClientAPI=1.41 ServerAPI=1.41 base: https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/16/artifact/out/Dockerfile |
| GITHUB PR | https://github.com/apache/hadoop/pull/5131 |
| Optional Tests | dupname asflicense compile javac javadoc mvninstall mvnsite unit shadedclient spotbugs checkstyle codespell detsecrets |
| uname | Linux 1a8655149961 4.15.0-191-generic #202-Ubuntu SMP Thu Aug 4 01:49:29 UTC 2022 x86_64 x86_64 x86_64 GNU/Linux |
| Build tool | maven |
| Personality | dev-support/bin/hadoop.sh |
| git revision | trunk / 72e2cf51c705e5ca9746894b4f6ddc6f5a0d4ebd |
| Default Java | Private Build-1.8.0_352-8u352-ga-1~20.04-b08 |
| Multi-JDK versions | /usr/lib/jvm/java-11-openjdk-amd64:Ubuntu-11.0.17+8-post-Ubuntu-1ubuntu220.04 /usr/lib/jvm/java-8-openjdk-amd64:Private Build-1.8.0_352-8u352-ga-1~20.04-b08 |
| Test Results | https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/16/testReport/ |
| Max. process+thread count | 1007 (vs. ulimit of 5500) |
| modules | C: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-router U: hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server |
| Console output | https://ci-hadoop.apache.org/job/hadoop-multibranch/job/PR-5131/16/console |
| versions | git=2.25.1 maven=3.6.3 spotbugs=4.2.2 |
| Powered by | Apache Yetus 0.14.0 https://yetus.apache.org |
This message was automatically generated.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-issues-help@hadoop.apache.org
[GitHub] [hadoop] goiri commented on a diff in pull request #5131: YARN-11350. [Federation] Router Support DelegationToken With ZK.
Posted by GitBox <gi...@apache.org>.
goiri commented on code in PR #5131:
URL: https://github.com/apache/hadoop/pull/5131#discussion_r1023317011
##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/FederationStateStoreBaseTest.java:
##########
@@ -879,7 +879,7 @@ public void testStoreNewMasterKey() throws Exception {
}
@Test
- public void testGetMasterKeyByDelegationKey() throws YarnException, IOException {
+ public void testGetMasterKeyByDelegationKey() throws Exception {
Review Comment:
This is correct but do we need to add all this churn?
##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/ZookeeperFederationStateStore.java:
##########
@@ -121,6 +134,14 @@
* |--- RESERVATION
* | |----- RESERVATION1
* | |----- RESERVATION2
+ * |--- ROUTER_RM_DT_SECRET_MANAGER_ROOT
+ * | |----- ROUTER_RM_DELEGATION_TOKENS_ROOT
+ * | | |----- RM_DELEGATION_TOKEN_1
+ * | | |----- RM_DELEGATION_TOKEN_2
+ * | | |----- RM_DELEGATION_TOKEN_3
+ * | |----- ROUTER_RM_DT_MASTER_KEYS_ROOT
+ * | | |----- DELEGATION_KEY_1
+ * |--- |----- ROUTER_RM_DT_SEQUENTIAL_NUMBER
Review Comment:
```suggestion
* | |----- ROUTER_RM_DT_SEQUENTIAL_NUMBER
```
##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/TestZookeeperFederationStateStore.java:
##########
@@ -171,38 +202,270 @@ public void testMetricsInited() throws Exception {
MetricsRecords.assertMetric(record, "UpdateReservationHomeSubClusterNumOps", expectOps);
}
- @Test(expected = NotImplementedException.class)
public void testStoreNewMasterKey() throws Exception {
- super.testStoreNewMasterKey();
+
+ // Manually create a DelegationKey,
+ // and call the interface storeNewMasterKey to write the data to zk.
+ DelegationKey key = new DelegationKey(1234, Time.now() + 60 * 60, "keyBytes".getBytes());
+ RouterMasterKey paramRouterMasterKey = RouterMasterKey.newInstance(key.getKeyId(),
+ ByteBuffer.wrap(key.getEncodedKey()), key.getExpiryDate());
+ FederationStateStore stateStore = this.getStateStore();
+
+ assertTrue(stateStore instanceof ZookeeperFederationStateStore);
+
+ // Compare the data returned by the storeNewMasterKey
+ // interface with the data queried by zk, and ensure that the data is consistent.
+ RouterMasterKeyRequest routerMasterKeyRequest =
+ RouterMasterKeyRequest.newInstance(paramRouterMasterKey);
+ RouterMasterKeyResponse response = stateStore.storeNewMasterKey(routerMasterKeyRequest);
+ assertNotNull(response);
+ RouterMasterKey respRouterMasterKey = response.getRouterMasterKey();
+ assertNotNull(respRouterMasterKey);
+
+ // Get Data From zk.
+ String nodeName = ROUTER_RM_DELEGATION_KEY_PREFIX + key.getKeyId();
+ String nodePath = ZNODE_MASTER_KEY_PREFIX + nodeName;
+ RouterMasterKey zkRouterMasterKey = getRouterMasterKeyFromZK(nodePath);
+
+ assertNotNull(zkRouterMasterKey);
+ assertEquals(paramRouterMasterKey, respRouterMasterKey);
+ assertEquals(paramRouterMasterKey, zkRouterMasterKey);
+ assertEquals(zkRouterMasterKey, respRouterMasterKey);
}
- @Test(expected = NotImplementedException.class)
- public void testGetMasterKeyByDelegationKey() throws YarnException, IOException {
- super.testGetMasterKeyByDelegationKey();
+ public void testGetMasterKeyByDelegationKey() throws Exception {
+
+ // Manually create a DelegationKey,
+ // and call the interface storeNewMasterKey to write the data to zk.
+ DelegationKey key = new DelegationKey(5678, Time.now() + 60 * 60, "keyBytes".getBytes());
+ RouterMasterKey paramRouterMasterKey = RouterMasterKey.newInstance(key.getKeyId(),
+ ByteBuffer.wrap(key.getEncodedKey()), key.getExpiryDate());
+ FederationStateStore stateStore = this.getStateStore();
+
+ assertTrue(stateStore instanceof ZookeeperFederationStateStore);
+
+ // Call the getMasterKeyByDelegationKey interface of stateStore to get the MasterKey data.
+ RouterMasterKeyRequest routerMasterKeyRequest =
+ RouterMasterKeyRequest.newInstance(paramRouterMasterKey);
+ RouterMasterKeyResponse response = stateStore.storeNewMasterKey(routerMasterKeyRequest);
+ assertNotNull(response);
+
+ // Get Data From zk.
+ String nodeName = ROUTER_RM_DELEGATION_KEY_PREFIX + key.getKeyId();
+ String nodePath = ZNODE_MASTER_KEY_PREFIX + nodeName;
+ RouterMasterKey zkRouterMasterKey = getRouterMasterKeyFromZK(nodePath);
+
+ // Call the getMasterKeyByDelegationKey interface to get the returned result.
+ // The zk data should be consistent with the returned data.
+ RouterMasterKeyResponse response1 =
+ stateStore.getMasterKeyByDelegationKey(routerMasterKeyRequest);
+ assertNotNull(response1);
+ RouterMasterKey respRouterMasterKey = response1.getRouterMasterKey();
+ assertEquals(paramRouterMasterKey, respRouterMasterKey);
+ assertEquals(paramRouterMasterKey, zkRouterMasterKey);
+ assertEquals(zkRouterMasterKey, respRouterMasterKey);
+ }
+
+ public void testRemoveStoredMasterKey() throws Exception {
+
+ // Manually create a DelegationKey,
+ // and call the interface storeNewMasterKey to write the data to zk.
+ DelegationKey key = new DelegationKey(2345, Time.now() + 60 * 60, "keyBytes".getBytes());
+ RouterMasterKey paramRouterMasterKey = RouterMasterKey.newInstance(key.getKeyId(),
+ ByteBuffer.wrap(key.getEncodedKey()), key.getExpiryDate());
+ FederationStateStore stateStore = this.getStateStore();
+
+ assertTrue(stateStore instanceof ZookeeperFederationStateStore);
+
+ // We need to ensure that the returned result is not empty.
+ RouterMasterKeyRequest routerMasterKeyRequest =
+ RouterMasterKeyRequest.newInstance(paramRouterMasterKey);
+ RouterMasterKeyResponse response = stateStore.storeNewMasterKey(routerMasterKeyRequest);
+ assertNotNull(response);
+
+ // We will check if delegationToken exists in zk.
+ String nodeName = ROUTER_RM_DELEGATION_KEY_PREFIX + key.getKeyId();
+ String nodePath = ZNODE_MASTER_KEY_PREFIX + nodeName;
+ assertTrue(curatorFramework.checkExists().forPath(nodePath) != null);
+
+ // Call removeStoredMasterKey to remove the MasterKey data in zk.
+ RouterMasterKeyResponse response1 = stateStore.removeStoredMasterKey(routerMasterKeyRequest);
+ assertNotNull(response1);
+ RouterMasterKey respRouterMasterKey = response1.getRouterMasterKey();
+ assertNotNull(respRouterMasterKey);
+ assertEquals(paramRouterMasterKey, respRouterMasterKey);
+
+ // We have removed the RouterMasterKey data from zk,
+ // the path should be empty at this point.
+ assertTrue(curatorFramework.checkExists().forPath(nodePath) == null);
+ }
+
+ public void testStoreNewToken() throws Exception {
+
+ // We manually generate the DelegationToken,
+ // and then call the StoreNewToken method to store the Token in zk.
+ RMDelegationTokenIdentifier identifier = new RMDelegationTokenIdentifier(
+ new Text("owner2"), new Text("renewer2"), new Text("realuser2"));
+ FederationStateStore stateStore = this.getStateStore();
+ int seqNum = stateStore.incrementDelegationTokenSeqNum();
+ identifier.setSequenceNumber(seqNum);
+ Long renewDate = Time.now();
+
+ // Store new rm-token
+ RouterStoreToken paramStoreToken = RouterStoreToken.newInstance(identifier, renewDate);
+ RouterRMTokenRequest request = RouterRMTokenRequest.newInstance(paramStoreToken);
+ RouterRMTokenResponse routerRMTokenResponse = stateStore.storeNewToken(request);
+ assertNotNull(routerRMTokenResponse);
+ RouterStoreToken respStoreToken = routerRMTokenResponse.getRouterStoreToken();
+ assertNotNull(respStoreToken);
+
+ // Get delegationToken Path
+ String nodeName = ROUTER_RM_DELEGATION_TOKEN_PREFIX + identifier.getSequenceNumber();
+ String nodePath = getNodePath(ZNODE_DT_PREFIX, nodeName);
+
+ // Check if the path exists, we expect the result to exist.
+ assertTrue(curatorFramework.checkExists().forPath(nodePath) != null);
+
+ // Check whether the token (paramStoreToken)
+ // We generated is consistent with the data stored in zk.
+ // We expect data to be consistent.
+ RouterStoreToken zkRouterStoreToken = getStoreTokenFromZK(nodePath);
+ assertNotNull(zkRouterStoreToken);
+ assertEquals(paramStoreToken, zkRouterStoreToken);
+ assertEquals(respStoreToken, zkRouterStoreToken);
}
- @Test(expected = NotImplementedException.class)
- public void testRemoveStoredMasterKey() throws YarnException, IOException {
- super.testRemoveStoredMasterKey();
+ public void testUpdateStoredToken() throws Exception {
Review Comment:
Can we keep the Test annotation in all these methods?
##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/ZookeeperFederationStateStore.java:
##########
@@ -886,45 +974,558 @@ public UpdateReservationHomeSubClusterResponse updateReservationHomeSubCluster(
return UpdateReservationHomeSubClusterResponse.newInstance();
}
+ /**
+ * ZookeeperFederationStateStore Supports Store NewMasterKey.
+ *
+ * @param request The request contains RouterMasterKey, which is an abstraction for DelegationKey
+ * @return routerMasterKeyResponse, the response contains the RouterMasterKey.
+ * @throws YarnException if the call to the state store is unsuccessful.
+ * @throws IOException An IO Error occurred.
+ */
@Override
- public RouterMasterKeyResponse storeNewMasterKey(RouterMasterKeyRequest request)
+ public synchronized RouterMasterKeyResponse storeNewMasterKey(RouterMasterKeyRequest request)
throws YarnException, IOException {
- throw new NotImplementedException("Code is not implemented");
+
+ long start = clock.getTime();
+ // For the verification of the request, after passing the verification,
+ // the request and the internal objects will not be empty and can be used directly.
+ FederationRouterRMTokenInputValidator.validate(request);
+
+ // Parse the delegationKey from the request and get the ZK storage path.
+ DelegationKey delegationKey = convertMasterKeyToDelegationKey(request);
+ String nodeCreatePath = getMasterKeyZNodePathByDelegationKey(delegationKey);
+ LOG.debug("Storing RMDelegationKey_{}, ZkNodePath = {}.", delegationKey.getKeyId(),
+ nodeCreatePath);
+
+ // Write master key data to zk.
+ try(ByteArrayOutputStream os = new ByteArrayOutputStream();
+ DataOutputStream fsOut = new DataOutputStream(os)) {
+ delegationKey.write(fsOut);
+ put(nodeCreatePath, os.toByteArray(), false);
+ }
+
+ // Get the stored masterKey from zk.
+ RouterMasterKey masterKeyFromZK = getRouterMasterKeyFromZK(nodeCreatePath, false);
+ long end = clock.getTime();
+ opDurations.addStoreNewMasterKeyDuration(start, end);
+ return RouterMasterKeyResponse.newInstance(masterKeyFromZK);
}
+ /**
+ * ZookeeperFederationStateStore Supports Remove MasterKey.
+ *
+ * @param request The request contains RouterMasterKey, which is an abstraction for DelegationKey
+ * @return routerMasterKeyResponse, the response contains the RouterMasterKey.
+ * @throws YarnException if the call to the state store is unsuccessful.
+ * @throws IOException An IO Error occurred.
+ */
@Override
- public RouterMasterKeyResponse removeStoredMasterKey(RouterMasterKeyRequest request)
+ public synchronized RouterMasterKeyResponse removeStoredMasterKey(RouterMasterKeyRequest request)
throws YarnException, IOException {
- throw new NotImplementedException("Code is not implemented");
+
+ long start = clock.getTime();
+ // For the verification of the request, after passing the verification,
+ // the request and the internal objects will not be empty and can be used directly.
+ FederationRouterRMTokenInputValidator.validate(request);
+
+ try {
+ // Parse the delegationKey from the request and get the ZK storage path.
+ RouterMasterKey masterKey = request.getRouterMasterKey();
+ DelegationKey delegationKey = convertMasterKeyToDelegationKey(request);
+ String nodeRemovePath = getMasterKeyZNodePathByDelegationKey(delegationKey);
+ LOG.debug("Removing RMDelegationKey_{}, ZkNodePath = {}.", delegationKey.getKeyId(),
+ nodeRemovePath);
+
+ // Check if the path exists, Throws an exception if the path does not exist.
+ if (!exists(nodeRemovePath)) {
+ throw new YarnException("ZkNodePath = " + nodeRemovePath + " not exists!");
+ }
+
+ // try to remove masterKey.
+ zkManager.delete(nodeRemovePath);
+ long end = clock.getTime();
+ opDurations.removeStoredMasterKeyDuration(start, end);
+ return RouterMasterKeyResponse.newInstance(masterKey);
+ } catch (Exception e) {
+ throw new YarnException(e);
+ }
}
+ /**
+ * ZookeeperFederationStateStore Supports Remove MasterKey.
+ *
+ * @param request The request contains RouterMasterKey, which is an abstraction for DelegationKey
+ * @return routerMasterKeyResponse, the response contains the RouterMasterKey.
+ * @throws YarnException if the call to the state store is unsuccessful.
+ * @throws IOException An IO Error occurred.
+ */
@Override
public RouterMasterKeyResponse getMasterKeyByDelegationKey(RouterMasterKeyRequest request)
throws YarnException, IOException {
- throw new NotImplementedException("Code is not implemented");
+
+ long start = clock.getTime();
+ // For the verification of the request, after passing the verification,
+ // the request and the internal objects will not be empty and can be used directly.
+ FederationRouterRMTokenInputValidator.validate(request);
+
+ try {
+
+ // Parse the delegationKey from the request and get the ZK storage path.
+ DelegationKey delegationKey = convertMasterKeyToDelegationKey(request);
+ String nodePath = getMasterKeyZNodePathByDelegationKey(delegationKey);
+
+ // Check if the path exists, Throws an exception if the path does not exist.
+ if (!exists(nodePath)) {
+ throw new YarnException("ZkNodePath = " + nodePath + " not exists!");
+ }
+
+ // Get the stored masterKey from zk.
+ RouterMasterKey routerMasterKey = getRouterMasterKeyFromZK(nodePath, false);
+ long end = clock.getTime();
+ opDurations.getMasterKeyByDelegationKeyDuration(start, end);
+ return RouterMasterKeyResponse.newInstance(routerMasterKey);
+ } catch (Exception e) {
+ throw new YarnException(e);
+ }
+ }
+
+ /**
+ * Get MasterKeyZNodePath based on DelegationKey.
+ *
+ * @param delegationKey delegationKey.
+ * @return masterKey ZNodePath.
+ */
+ private String getMasterKeyZNodePathByDelegationKey(DelegationKey delegationKey) {
+ return getMasterKeyZNodePathByKeyId(delegationKey.getKeyId());
+ }
+
+ /**
+ * Get MasterKeyZNodePath based on KeyId.
+ *
+ * @param keyId master key id.
+ * @return masterKey ZNodePath.
+ */
+ private String getMasterKeyZNodePathByKeyId(int keyId) {
+ String nodeName = ROUTER_RM_DELEGATION_KEY_PREFIX + keyId;
+ return getNodePath(routerRMDTMasterKeysRootPath, nodeName);
}
+ /**
+ * Get RouterMasterKey from ZK.
+ *
+ * @param nodePath The path where masterKey is stored in zk.
+ * @param quiet If true is silent mode, no error message is printed at this time,
+ * if false is non-silent mode, error message is printed at this time.
+ *
+ * @return
+ * @throws IOException
+ */
+ private RouterMasterKey getRouterMasterKeyFromZK(String nodePath, boolean quiet)
+ throws IOException {
+ try {
+ byte[] data = get(nodePath);
+ if ((data == null) || (data.length == 0)) {
+ return null;
+ }
+
+ ByteArrayInputStream bin = new ByteArrayInputStream(data);
+ DataInputStream din = new DataInputStream(bin);
+ DelegationKey key = new DelegationKey();
+ key.readFields(din);
+
+ return RouterMasterKey.newInstance(key.getKeyId(),
+ ByteBuffer.wrap(key.getEncodedKey()), key.getExpiryDate());
+ } catch (Exception ex) {
+ if (!quiet) {
+ LOG.error("No node in path [" + nodePath + "]");
Review Comment:
Use {}
##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/ZookeeperFederationStateStore.java:
##########
@@ -174,21 +219,64 @@ public void init(Configuration conf) throws YarnException {
policiesZNode = getNodePath(baseZNode, ROOT_ZNODE_NAME_POLICY);
reservationsZNode = getNodePath(baseZNode, ROOT_ZNODE_NAME_RESERVATION);
+ // delegation token znodes
+ routerRMDTSecretManagerRoot = getNodePath(baseZNode, ROUTER_RM_DT_SECRET_MANAGER_ROOT);
+ routerRMDTMasterKeysRootPath = getNodePath(routerRMDTSecretManagerRoot,
+ ROUTER_RM_DT_MASTER_KEYS_ROOT_ZNODE_NAME);
+ routerRMDelegationTokensRootPath = getNodePath(routerRMDTSecretManagerRoot,
+ ROUTER_RM_DELEGATION_TOKENS_ROOT_ZNODE_NAME);
+ routerRMSequenceNumberPath = getNodePath(routerRMDTSecretManagerRoot,
+ ROUTER_RM_DT_SEQUENTIAL_NUMBER_ZNODE_NAME);
+
// Create base znode for each entity
try {
List<ACL> zkAcl = ZKCuratorManager.getZKAcls(conf);
zkManager.createRootDirRecursively(membershipZNode, zkAcl);
zkManager.createRootDirRecursively(appsZNode, zkAcl);
zkManager.createRootDirRecursively(policiesZNode, zkAcl);
zkManager.createRootDirRecursively(reservationsZNode, zkAcl);
+ zkManager.createRootDirRecursively(routerRMDTSecretManagerRoot, zkAcl);
+ zkManager.createRootDirRecursively(routerRMDTMasterKeysRootPath, zkAcl);
+ zkManager.createRootDirRecursively(routerRMDelegationTokensRootPath, zkAcl);
} catch (Exception e) {
String errMsg = "Cannot create base directories: " + e.getMessage();
FederationStateStoreUtils.logAndThrowStoreException(LOG, errMsg);
}
+
+ // Distributed sequenceNum.
+ try {
+ seqNumBatchSize = conf.getInt(ZK_DTSM_TOKEN_SEQNUM_BATCH_SIZE,
+ ZK_DTSM_TOKEN_SEQNUM_BATCH_SIZE_DEFAULT);
+
+ delTokSeqCounter = new SharedCount(zkManager.getCurator(), routerRMSequenceNumberPath, 0);
+
+ if (delTokSeqCounter != null) {
+ delTokSeqCounter.start();
+ }
+
+ // the first batch range should be allocated during this starting window
+ // by calling the incrSharedCount
+ currentSeqNum = incrSharedCount(delTokSeqCounter, seqNumBatchSize);
+ currentMaxSeqNum = currentSeqNum + seqNumBatchSize;
+
+ LOG.info("Fetched initial range of seq num, from {} to {} ",
+ currentSeqNum + 1, currentMaxSeqNum);
+ } catch (Exception e) {
+ throw new YarnException("Could not start Sequence Counter.", e);
+ }
}
@Override
public void close() throws Exception {
+
+ try {
+ if (delTokSeqCounter != null) {
+ delTokSeqCounter.close();
Review Comment:
Add
```
delTokSeqCounter = null;
```
?
##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/federation/store/impl/ZookeeperFederationStateStore.java:
##########
@@ -886,45 +974,558 @@ public UpdateReservationHomeSubClusterResponse updateReservationHomeSubCluster(
return UpdateReservationHomeSubClusterResponse.newInstance();
}
+ /**
+ * ZookeeperFederationStateStore Supports Store NewMasterKey.
+ *
+ * @param request The request contains RouterMasterKey, which is an abstraction for DelegationKey
+ * @return routerMasterKeyResponse, the response contains the RouterMasterKey.
+ * @throws YarnException if the call to the state store is unsuccessful.
+ * @throws IOException An IO Error occurred.
+ */
@Override
- public RouterMasterKeyResponse storeNewMasterKey(RouterMasterKeyRequest request)
+ public synchronized RouterMasterKeyResponse storeNewMasterKey(RouterMasterKeyRequest request)
throws YarnException, IOException {
- throw new NotImplementedException("Code is not implemented");
+
+ long start = clock.getTime();
+ // For the verification of the request, after passing the verification,
+ // the request and the internal objects will not be empty and can be used directly.
+ FederationRouterRMTokenInputValidator.validate(request);
+
+ // Parse the delegationKey from the request and get the ZK storage path.
+ DelegationKey delegationKey = convertMasterKeyToDelegationKey(request);
+ String nodeCreatePath = getMasterKeyZNodePathByDelegationKey(delegationKey);
+ LOG.debug("Storing RMDelegationKey_{}, ZkNodePath = {}.", delegationKey.getKeyId(),
+ nodeCreatePath);
+
+ // Write master key data to zk.
+ try(ByteArrayOutputStream os = new ByteArrayOutputStream();
+ DataOutputStream fsOut = new DataOutputStream(os)) {
+ delegationKey.write(fsOut);
+ put(nodeCreatePath, os.toByteArray(), false);
+ }
+
+ // Get the stored masterKey from zk.
+ RouterMasterKey masterKeyFromZK = getRouterMasterKeyFromZK(nodeCreatePath, false);
+ long end = clock.getTime();
+ opDurations.addStoreNewMasterKeyDuration(start, end);
+ return RouterMasterKeyResponse.newInstance(masterKeyFromZK);
}
+ /**
+ * ZookeeperFederationStateStore Supports Remove MasterKey.
+ *
+ * @param request The request contains RouterMasterKey, which is an abstraction for DelegationKey
+ * @return routerMasterKeyResponse, the response contains the RouterMasterKey.
+ * @throws YarnException if the call to the state store is unsuccessful.
+ * @throws IOException An IO Error occurred.
+ */
@Override
- public RouterMasterKeyResponse removeStoredMasterKey(RouterMasterKeyRequest request)
+ public synchronized RouterMasterKeyResponse removeStoredMasterKey(RouterMasterKeyRequest request)
throws YarnException, IOException {
- throw new NotImplementedException("Code is not implemented");
+
+ long start = clock.getTime();
+ // For the verification of the request, after passing the verification,
+ // the request and the internal objects will not be empty and can be used directly.
+ FederationRouterRMTokenInputValidator.validate(request);
+
+ try {
+ // Parse the delegationKey from the request and get the ZK storage path.
+ RouterMasterKey masterKey = request.getRouterMasterKey();
+ DelegationKey delegationKey = convertMasterKeyToDelegationKey(request);
+ String nodeRemovePath = getMasterKeyZNodePathByDelegationKey(delegationKey);
+ LOG.debug("Removing RMDelegationKey_{}, ZkNodePath = {}.", delegationKey.getKeyId(),
+ nodeRemovePath);
+
+ // Check if the path exists, Throws an exception if the path does not exist.
+ if (!exists(nodeRemovePath)) {
+ throw new YarnException("ZkNodePath = " + nodeRemovePath + " not exists!");
+ }
+
+ // try to remove masterKey.
+ zkManager.delete(nodeRemovePath);
+ long end = clock.getTime();
+ opDurations.removeStoredMasterKeyDuration(start, end);
+ return RouterMasterKeyResponse.newInstance(masterKey);
+ } catch (Exception e) {
+ throw new YarnException(e);
+ }
}
+ /**
+ * ZookeeperFederationStateStore Supports Remove MasterKey.
+ *
+ * @param request The request contains RouterMasterKey, which is an abstraction for DelegationKey
+ * @return routerMasterKeyResponse, the response contains the RouterMasterKey.
+ * @throws YarnException if the call to the state store is unsuccessful.
+ * @throws IOException An IO Error occurred.
+ */
@Override
public RouterMasterKeyResponse getMasterKeyByDelegationKey(RouterMasterKeyRequest request)
throws YarnException, IOException {
- throw new NotImplementedException("Code is not implemented");
+
+ long start = clock.getTime();
+ // For the verification of the request, after passing the verification,
+ // the request and the internal objects will not be empty and can be used directly.
+ FederationRouterRMTokenInputValidator.validate(request);
+
+ try {
+
+ // Parse the delegationKey from the request and get the ZK storage path.
+ DelegationKey delegationKey = convertMasterKeyToDelegationKey(request);
+ String nodePath = getMasterKeyZNodePathByDelegationKey(delegationKey);
+
+ // Check if the path exists, Throws an exception if the path does not exist.
+ if (!exists(nodePath)) {
+ throw new YarnException("ZkNodePath = " + nodePath + " not exists!");
+ }
+
+ // Get the stored masterKey from zk.
+ RouterMasterKey routerMasterKey = getRouterMasterKeyFromZK(nodePath, false);
+ long end = clock.getTime();
+ opDurations.getMasterKeyByDelegationKeyDuration(start, end);
+ return RouterMasterKeyResponse.newInstance(routerMasterKey);
+ } catch (Exception e) {
+ throw new YarnException(e);
+ }
+ }
+
+ /**
+ * Get MasterKeyZNodePath based on DelegationKey.
+ *
+ * @param delegationKey delegationKey.
+ * @return masterKey ZNodePath.
+ */
+ private String getMasterKeyZNodePathByDelegationKey(DelegationKey delegationKey) {
+ return getMasterKeyZNodePathByKeyId(delegationKey.getKeyId());
+ }
+
+ /**
+ * Get MasterKeyZNodePath based on KeyId.
+ *
+ * @param keyId master key id.
+ * @return masterKey ZNodePath.
+ */
+ private String getMasterKeyZNodePathByKeyId(int keyId) {
+ String nodeName = ROUTER_RM_DELEGATION_KEY_PREFIX + keyId;
+ return getNodePath(routerRMDTMasterKeysRootPath, nodeName);
}
+ /**
+ * Get RouterMasterKey from ZK.
+ *
+ * @param nodePath The path where masterKey is stored in zk.
+ * @param quiet If true is silent mode, no error message is printed at this time,
+ * if false is non-silent mode, error message is printed at this time.
+ *
+ * @return
+ * @throws IOException
+ */
+ private RouterMasterKey getRouterMasterKeyFromZK(String nodePath, boolean quiet)
Review Comment:
I don't see much value in this quiet option.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-issues-help@hadoop.apache.org
[GitHub] [hadoop] slfan1989 commented on pull request #5131: YARN-11350. [Federation] Router Support DelegationToken With ZK.
Posted by GitBox <gi...@apache.org>.
slfan1989 commented on PR #5131:
URL: https://github.com/apache/hadoop/pull/5131#issuecomment-1317903431
@goiri Can you help review this PR again? Thank you very much!
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-issues-help@hadoop.apache.org
[GitHub] [hadoop] slfan1989 commented on a diff in pull request #5131: YARN-11350. [Federation] Router Support DelegationToken With ZK.
Posted by GitBox <gi...@apache.org>.
slfan1989 commented on code in PR #5131:
URL: https://github.com/apache/hadoop/pull/5131#discussion_r1041589814
##########
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/test/java/org/apache/hadoop/yarn/server/federation/store/impl/TestZookeeperFederationStateStore.java:
##########
@@ -171,38 +203,117 @@ public void testMetricsInited() throws Exception {
MetricsRecords.assertMetric(record, "UpdateReservationHomeSubClusterNumOps", expectOps);
}
- @Test(expected = NotImplementedException.class)
+ @Test
public void testStoreNewMasterKey() throws Exception {
super.testStoreNewMasterKey();
}
- @Test(expected = NotImplementedException.class)
+ @Test
Review Comment:
I agree with you, I will remove this part of the code.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
---------------------------------------------------------------------
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-issues-help@hadoop.apache.org