You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@tez.apache.org by "abstractdog (via GitHub)" <gi...@apache.org> on 2025/09/06 12:12:52 UTC
[PR] TEZ-4007: Zookeeper based FrameworkServices and AmExtensions (3/3) - initial patch [tez]
abstractdog opened a new pull request, #427:
URL: https://github.com/apache/tez/pull/427
(no comment)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] TEZ-4007: Introduce AmExtensions and Zookeeper-based FrameworkServices [tez]
Posted by "abstractdog (via GitHub)" <gi...@apache.org>.
abstractdog commented on code in PR #427:
URL: https://github.com/apache/tez/pull/427#discussion_r2671834345
##########
tez-api/src/main/java/org/apache/tez/client/registry/AMRegistryClient.java:
##########
@@ -0,0 +1,118 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.client.registry;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Client-side interface for discovering Application Master (AM) instances
+ * registered in the AM registry.
+ *
+ * <p>Implementations are responsible for locating AM endpoints and returning
+ * their metadata. This API is used by client components to discover running
+ * Tez AMs.</p>
+ *
+ * <p>Listeners may be registered to receive notifications when AM records
+ * appear or are removed.</p>
+ */
+public abstract class AMRegistryClient implements Closeable {
Review Comment:
we can, but it doesn't make the code better: this class is usually used as field, not local variable in try-catch, this applies to production as well as testing code, so I think Closeable might be better here
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] TEZ-4007: Introduce AmExtensions and Zookeeper-based FrameworkServices [tez]
Posted by "abstractdog (via GitHub)" <gi...@apache.org>.
abstractdog commented on code in PR #427:
URL: https://github.com/apache/tez/pull/427#discussion_r2672066145
##########
tez-api/src/main/java/org/apache/tez/client/registry/zookeeper/ZkAMRegistryClient.java:
##########
@@ -0,0 +1,209 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.client.registry.zookeeper;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.imps.CuratorFrameworkState;
+import org.apache.curator.framework.recipes.cache.ChildData;
+import org.apache.curator.framework.recipes.cache.TreeCache;
+import org.apache.curator.framework.recipes.cache.TreeCacheEvent;
+import org.apache.curator.framework.recipes.cache.TreeCacheListener;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.registry.client.types.ServiceRecord;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.tez.client.registry.AMRecord;
+import org.apache.tez.client.registry.AMRegistryClient;
+import org.apache.tez.client.registry.AMRegistryUtils;
+import org.apache.tez.dag.api.TezConfiguration;
+
+import com.fasterxml.jackson.core.JsonParseException;
+import com.google.common.base.Preconditions;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Curator/Zookeeper implementation of {@link AMRegistryClient}.
+ */
+@InterfaceAudience.Public
+public final class ZkAMRegistryClient extends AMRegistryClient {
+ private static final Logger LOG = LoggerFactory.getLogger(ZkAMRegistryClient.class);
+ private static final Map<String, ZkAMRegistryClient> INSTANCES = new HashMap<>();
+
+ private final Configuration conf;
+ // Cache of known AMs
+ private final ConcurrentHashMap<ApplicationId, AMRecord> amRecordCache = new ConcurrentHashMap<>();
+
+ private CuratorFramework client;
+ private TreeCache cache;
+ private ZkRegistryListener listener;
+
+ private ZkAMRegistryClient(final Configuration conf) {
+ this.conf = conf;
+ }
+
+ public static synchronized ZkAMRegistryClient getClient(final Configuration conf) {
+ String namespace = conf.get(TezConfiguration.TEZ_AM_REGISTRY_NAMESPACE);
+ ZkAMRegistryClient registry = INSTANCES.get(namespace);
+ if (registry == null) {
+ registry = new ZkAMRegistryClient(conf);
+ INSTANCES.put(namespace, registry);
+ }
+ LOG.info("Returning tez AM registry ({}) for namespace '{}'", System.identityHashCode(registry), namespace);
+ return registry;
+ }
+
+ /**
+ * Deserializes a {@link ServiceRecord} from ZooKeeper data and converts it into an {@link AMRecord}
+ * for caching.
+ *
+ * @param childData the ZooKeeper node data containing a serialized {@link ServiceRecord}
+ * @return an {@link AMRecord} constructed from the deserialized {@link ServiceRecord}, or {@code null}
+ * if no data is present
+ * @throws IOException if the data cannot be deserialized into a {@link ServiceRecord}
+ */
+ public static AMRecord getAMRecord(final ChildData childData) throws IOException {
+ // Not a leaf path. Only leaf path contains AMRecord.
+ if (!childData.getPath().contains(ApplicationId.appIdStrPrefix)) {
+ return null;
+ }
+ byte[] data = childData.getData();
+ // only the path appeared, there is no data yet
+ if (data.length == 0) {
+ return null;
+ }
+ String value = new String(data, StandardCharsets.UTF_8);
+ try {
+ return AMRegistryUtils.jsonStringToRecord(value);
+ } catch (JsonParseException e) {
+ //Not a json AMRecord (SRV), could be some other data.
+ LOG.warn("Non-json data received while de-serializing AMRecord: {}. Ignoring...", value);
+ return null;
+ }
+ }
+
+ public void start() throws Exception {
+ ZkConfig zkConf = new ZkConfig(this.conf);
+ client = zkConf.createCuratorFramework();
+ cache = new TreeCache(client, zkConf.getZkNamespace());
+ client.start();
+ cache.start();
+ listener = new ZkRegistryListener();
+ cache.getListenable().addListener(listener);
+ }
+
+ @Override
+ public AMRecord getRecord(ApplicationId appId) {
+ AMRecord rec = amRecordCache.get(appId);
+ // Return a copy.
+ return rec == null ? null : new AMRecord(rec);
+ }
+
+ @Override
+ public List<AMRecord> getAllRecords() {
+ return amRecordCache.values().stream().map(AMRecord::new).collect(Collectors.toList());
+ }
+
+ @Override
+ public void close() {
+ IOUtils.closeQuietly(cache);
+ IOUtils.closeQuietly(client);
Review Comment:
oh, this is a good catch that reveals a lot about how vague the current implementation is to me
INSTANCE is a static collection, so its lifecycle is not really clear, other than a zookeeper registry client for a specific namespace is shared across the whole JVM, but when and how its closure is supposed to happen, is not clear, I need to investigate this (I mean, it works in the downstream distro, but I cannot tell how and why)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] [DRAFT] TEZ-4007: Zookeeper based FrameworkServices and AmExtensions (3/3) - initial patch [tez]
Posted by "okumin (via GitHub)" <gi...@apache.org>.
okumin commented on code in PR #427:
URL: https://github.com/apache/tez/pull/427#discussion_r2365962664
##########
tez-api/src/main/java/org/apache/tez/client/registry/zookeeper/ZkAMRegistryClient.java:
##########
@@ -0,0 +1,188 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.tez.client.registry.zookeeper;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
+
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.imps.CuratorFrameworkState;
+import org.apache.curator.framework.recipes.cache.ChildData;
+import org.apache.curator.framework.recipes.cache.PathChildrenCache;
+import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent;
+import org.apache.curator.framework.recipes.cache.PathChildrenCacheListener;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.registry.client.binding.RegistryUtils;
+import org.apache.hadoop.registry.client.types.ServiceRecord;
+import org.apache.tez.client.registry.AMRecord;
+import org.apache.tez.client.registry.AMRegistryClient;
+import org.apache.tez.client.registry.AMRegistryClientListener;
+import org.apache.tez.dag.api.TezConfiguration;
+
+import com.google.common.base.Preconditions;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Curator/Zookeeper impl of AMRegistryClient
+*/
+@InterfaceAudience.Public
+public class ZkAMRegistryClient extends AMRegistryClient {
Review Comment:
> be aware that these patches have been working for years in downstream distributions
Sounds excellent.
> can you please elaborate on how Trino's lack of fault tolerance comes to this picture?
Trino must have SPOF, but still attracts users. Hive on Tez (with LLAP) without fault tolerance is easier to set up than Hive on Tez with fault tolerance. I expect the easy quick start should attract users.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] TEZ-4007: Introduce AmExtensions and Zookeeper-based FrameworkServices [tez]
Posted by "abstractdog (via GitHub)" <gi...@apache.org>.
abstractdog commented on code in PR #427:
URL: https://github.com/apache/tez/pull/427#discussion_r2561657222
##########
tez-api/src/main/java/org/apache/tez/client/registry/zookeeper/ZkConfig.java:
##########
@@ -0,0 +1,128 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.client.registry.zookeeper;
+
+import org.apache.curator.RetryPolicy;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.CuratorFrameworkFactory;
+import org.apache.curator.retry.ExponentialBackoffRetry;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.tez.dag.api.TezConfiguration;
+
+import com.google.common.base.Preconditions;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class ZkConfig {
+ private static final Logger LOG = LoggerFactory.getLogger(ZkConfig.class);
+
+ // if namespace defined in config is 'foo' and COMPUTE_GROUP_NAME env is 'bar' then the zkpaths will be of format
+ // /tez-external-sessions/foo/bar
+ private final static String ZK_NAMESPACE_PREFIX = "/tez-external-sessions";
+ public final static String COMPUTE_GROUP_NAME_ENV = "COMPUTE_GROUP_NAME";
+ public final static String DEFAULT_COMPUTE_GROUP_NAME = "default-compute";
+
+ private final String zkQuorum;
+ private final String zkNamespace;
+ private final int curatorBackoffSleep;
+ private final int curatorMaxRetries;
+ private final int sessionTimeoutMs;
+ private final int connectionTimeoutMs;
+
+ public ZkConfig(Configuration conf) {
+ zkQuorum = conf.get(TezConfiguration.TEZ_AM_ZOOKEEPER_QUORUM);
+ Preconditions.checkNotNull(zkQuorum);
+
+ String fullZkNamespace = ZK_NAMESPACE_PREFIX;
+
+ String namespace = conf.get(TezConfiguration.TEZ_AM_REGISTRY_NAMESPACE,
+ TezConfiguration.TEZ_AM_REGISTRY_NAMESPACE_DEFAULT);
+ Preconditions.checkNotNull(namespace);
+ if (namespace.startsWith(Path.SEPARATOR)) {
+ fullZkNamespace += namespace;
+ } else {
+ fullZkNamespace = fullZkNamespace + Path.SEPARATOR + namespace;
+ }
+
+ boolean enableComputeGroups = conf.getBoolean(TezConfiguration.TEZ_AM_REGISTRY_ENABLE_COMPUTE_GROUPS,
+ TezConfiguration.TEZ_AM_REGISTRY_ENABLE_COMPUTE_GROUPS_DEFAULT);
+ if (enableComputeGroups) {
+ final String subNamespace = System.getenv(COMPUTE_GROUP_NAME_ENV);
+ if (subNamespace != null && !subNamespace.isEmpty()) {
+ if (subNamespace.startsWith(Path.SEPARATOR)) {
+ fullZkNamespace += subNamespace;
+ } else {
+ fullZkNamespace = fullZkNamespace + Path.SEPARATOR + subNamespace;
+ }
+ LOG.info("Compute groups enabled: subNamespace: {} fullZkNamespace: {}", subNamespace, fullZkNamespace);
+ }
+ } else {
+ LOG.info("Compute groups disabled: fullZkNamespace: {}", fullZkNamespace);
+ }
+ zkNamespace = fullZkNamespace;
+
+ curatorBackoffSleep = conf.getInt(TezConfiguration.TEZ_AM_CURATOR_BACKOFF_SLEEP,
+ TezConfiguration.TEZ_AM_CURATOR_BACKOFF_SLEEP_DEFAULT);
+ curatorMaxRetries = conf.getInt(TezConfiguration.TEZ_AM_CURATOR_MAX_RETRIES,
+ TezConfiguration.TEZ_AM_CURATOR_MAX_RETRIES_DEFAULT);
+ sessionTimeoutMs = conf.getInt(TezConfiguration.TEZ_AM_CURATOR_SESSION_TIMEOUT,
+ TezConfiguration.TEZ_AM_CURATOR_SESSION_TIMEOUT_DEFAULT);
+ connectionTimeoutMs = conf.getInt(TezConfiguration.TEZ_AM_CURATOR_CONNECTION_TIMEOUT,
+ TezConfiguration.TEZ_AM_CURATOR_CONNECTION_TIMEOUT_DEFAULT);
+ }
+
+ public String getZkQuorum() {
+ return zkQuorum;
+ }
+
+ public String getZkNamespace() {
+ return zkNamespace;
+ }
+
+ public int getCuratorBackoffSleep() {
+ return curatorBackoffSleep;
+ }
+
+ public int getCuratorMaxRetries() {
+ return curatorMaxRetries;
+ }
+
+ public int getSessionTimeoutMs() {
+ return sessionTimeoutMs;
+ }
+
+ public int getConnectionTimeoutMs() {
+ return connectionTimeoutMs;
+ }
+
+ public RetryPolicy getRetryPolicy() {
+ return new ExponentialBackoffRetry(getCuratorBackoffSleep(), getCuratorMaxRetries());
+ }
+
+ public CuratorFramework createCuratorFramework() {
Review Comment:
yeah, close is taken care of now in ZkAMRegistryClient.close()
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] TEZ-4007: Introduce AmExtensions and Zookeeper-based FrameworkServices [tez]
Posted by "ayushtkn (via GitHub)" <gi...@apache.org>.
ayushtkn commented on code in PR #427:
URL: https://github.com/apache/tez/pull/427#discussion_r2548926973
##########
tez-api/src/main/java/org/apache/tez/client/registry/AMRecord.java:
##########
@@ -54,17 +59,23 @@ public class AMRecord {
* Although this constructor may not be used directly within Tez internals,
* it is part of the public API for Tez clients that handle unmanaged sessions.
*
- * @param appId the {@link ApplicationId} of the Tez application
- * @param host the hostname where the Application Master is running
- * @param port the port number on which the Application Master is listening
- * @param id an opaque identifier for the record; if {@code null}, defaults to an empty string
+ * @param appId the {@link ApplicationId} of the Tez application
+ * @param hostName the hostname where the Application Master is running
+ * @param hostIp the IP address of the Application Master host
+ * @param port the RPC port number on which the Application Master is listening
+ * @param externalId an optional external identifier for the record; if {@code null}, defaults to an empty string
+ * @param computeName the compute group or cluster name; if {@code null},
+ * defaults to {@link ZkConfig#DEFAULT_COMPUTE_GROUP_NAME}
*/
- public AMRecord(ApplicationId appId, String host, int port, String id) {
+ public AMRecord(ApplicationId appId, String hostName, String hostIp, int port, String externalId,
+ String computeName) {
this.appId = appId;
- this.host = host;
+ this.hostName = hostName;
+ this.hostIp = hostIp;
this.port = port;
- //If id is not provided, convert to empty string
- this.id = (id == null) ? "" : id;
+ //externalId is optional, if not provided, convert to empty string
+ this.externalId = (externalId == null) ? "" : externalId;
+ this.computeName = (computeName == null) ? ZkConfig.DEFAULT_COMPUTE_GROUP_NAME : computeName;
Review Comment:
nit, can we do something like:
```
this.externalId = Optional.ofNullable(externalId).orElse("");
this.computeName = Optional.ofNullable(computeName).orElse(ZkConfig.DEFAULT_COMPUTE_GROUP_NAME);
```
##########
tez-api/src/main/java/org/apache/tez/client/TezClient.java:
##########
@@ -484,7 +484,9 @@ public synchronized TezClient getClient(ApplicationId appId) throws TezException
}
private void startFrameworkClient() {
- frameworkClient = createFrameworkClient();
+ if (frameworkClient == null) {
+ frameworkClient = createFrameworkClient();
+ }
Review Comment:
this is the only place ``frameworkClient`` is written, why do we do a `null` check, wouldn't this be always `null` & if is not `null` then that init & start would have also been executed, is there some other flow which create but not start & init?
##########
tez-api/src/main/java/org/apache/tez/client/registry/AMRecord.java:
##########
@@ -97,37 +110,54 @@ public AMRecord(AMRecord other) {
*/
public AMRecord(ServiceRecord serviceRecord) {
this.appId = ApplicationId.fromString(serviceRecord.get(APP_ID_RECORD_KEY));
- this.host = serviceRecord.get(HOST_RECORD_KEY);
+ this.hostName = serviceRecord.get(HOST_NAME_RECORD_KEY);
+ this.hostIp = serviceRecord.get(HOST_IP_RECORD_KEY);
this.port = Integer.parseInt(serviceRecord.get(PORT_RECORD_KEY));
- this.id = serviceRecord.get(OPAQUE_ID_KEY);
+ this.externalId = serviceRecord.get(EXTERNAL_ID_KEY);
+ this.computeName = serviceRecord.get(COMPUTE_GROUP_NAME_KEY);
}
public ApplicationId getApplicationId() {
return appId;
}
public String getHost() {
- return host;
+ return hostName;
+ }
+
+ public String getHostName() {
+ return hostName;
+ }
Review Comment:
this is same as `getHost()` and nobody uses it
##########
tez-api/src/main/java/org/apache/tez/client/registry/AMRecord.java:
##########
@@ -97,37 +110,54 @@ public AMRecord(AMRecord other) {
*/
public AMRecord(ServiceRecord serviceRecord) {
this.appId = ApplicationId.fromString(serviceRecord.get(APP_ID_RECORD_KEY));
- this.host = serviceRecord.get(HOST_RECORD_KEY);
+ this.hostName = serviceRecord.get(HOST_NAME_RECORD_KEY);
+ this.hostIp = serviceRecord.get(HOST_IP_RECORD_KEY);
this.port = Integer.parseInt(serviceRecord.get(PORT_RECORD_KEY));
- this.id = serviceRecord.get(OPAQUE_ID_KEY);
+ this.externalId = serviceRecord.get(EXTERNAL_ID_KEY);
+ this.computeName = serviceRecord.get(COMPUTE_GROUP_NAME_KEY);
}
public ApplicationId getApplicationId() {
return appId;
}
public String getHost() {
- return host;
+ return hostName;
+ }
+
+ public String getHostName() {
+ return hostName;
+ }
+
+ public String getHostIp() {
+ return hostIp;
}
public int getPort() {
return port;
}
- public String getId() {
- return id;
+ public String getExternalId() {
+ return externalId;
+ }
+
+ public String getComputeName() {
+ return computeName;
}
@Override
public boolean equals(Object other) {
if (this == other) {
return true;
}
- if (other instanceof AMRecord otherRecord) {
+ if (other instanceof AMRecord) {
+ AMRecord otherRecord = (AMRecord) other;
Review Comment:
it can be done this way
```
if (other instanceof AMRecord otherRecord) {
return appId.equals(otherRecord.appId)
&& hostName.equals(otherRecord.hostName)
&& hostIp.equals(otherRecord.hostIp)
&& port == otherRecord.port
&& externalId.equals(otherRecord.externalId)
&& computeName.equals(otherRecord.computeName);
}
```
##########
tez-api/src/main/java/org/apache/tez/client/registry/AMRecord.java:
##########
@@ -150,14 +180,21 @@ public boolean equals(Object other) {
public ServiceRecord toServiceRecord() {
ServiceRecord serviceRecord = new ServiceRecord();
serviceRecord.set(APP_ID_RECORD_KEY, appId);
- serviceRecord.set(HOST_RECORD_KEY, host);
+ serviceRecord.set(HOST_NAME_RECORD_KEY, hostName);
+ serviceRecord.set(HOST_IP_RECORD_KEY, hostIp);
serviceRecord.set(PORT_RECORD_KEY, port);
- serviceRecord.set(OPAQUE_ID_KEY, id);
+ serviceRecord.set(EXTERNAL_ID_KEY, externalId);
+ serviceRecord.set(COMPUTE_GROUP_NAME_KEY, computeName);
return serviceRecord;
}
+ @Override
+ public String toString() {
+ return toServiceRecord().attributes().toString();
+ }
Review Comment:
this looks unnecessarily costly, where do you use it?
##########
tez-api/src/main/java/org/apache/tez/client/registry/AMRegistryClient.java:
##########
@@ -0,0 +1,98 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.client.registry;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Client-side interface for discovering Application Master (AM) instances
+ * registered in the AM registry.
+ *
+ * <p>Implementations are responsible for locating AM endpoints and returning
+ * their metadata. This API is used by client components to discover running
+ * Tez AMs.</p>
+ *
+ * <p>Listeners may be registered to receive notifications when AM records
+ * appear or are removed.</p>
+ */
+public abstract class AMRegistryClient implements Closeable {
+
+ private final List<AMRegistryClientListener> listeners = new ArrayList<>();
+
+ /**
+ * Returns the current set of registered listeners.
+ *
+ * @return a mutable list of listeners
+ */
+ protected List<AMRegistryClientListener> getListeners() {
+ return listeners;
+ }
+
+ /**
+ * Lookup AM metadata for the given application ID.
+ *
+ * @param appId the application ID
+ * @return the AM record if found, otherwise {@code null}
+ * @throws IOException if the lookup fails
+ */
+ public abstract AMRecord getRecord(String appId) throws IOException;
+
+ /**
+ * Retrieve all AM records known in the registry.
+ *
+ * @return a list of AM records (possibly empty)
+ * @throws IOException if the fetch fails
+ */
+ public abstract List<AMRecord> getAllRecords() throws IOException;
+
+ /**
+ * Register a listener for AM registry events.
+ * The listener will be notified when AM records are added or removed.
+ *
+ * @param listener the listener to add
+ */
+ public synchronized void addListener(AMRegistryClientListener listener) {
+ listeners.add(listener);
+ }
+
+ /**
+ * Notify listeners of a newly added AM record.
+ *
+ * @param record the added AM record
+ */
+ protected synchronized void notifyOnAdded(AMRecord record) {
+ for (AMRegistryClientListener listener : listeners) {
+ listener.onAdd(record);
+ }
Review Comment:
This is under synchronized As a general practice it ain't very nice to do
* If a listener performs IO → registry client becomes blocked
* If a listener is buggy / slow → everything is blocked
* If listener internally calls addListener() → deadlock risk
If implementation uses ZK watchers (which call callbacks on ZK event threads), a slow listener can block ZK event pipeline
```
for (AMRegistryClientListener listener : listeners) {
try {
listener.onAdd(record);
} catch (Exception e) {
// Log, but never let exceptions escape
}
}
```
And change ``listeners`` ArrayList<> to a threadsafe impl
##########
tez-api/src/main/java/org/apache/tez/client/registry/AMRegistry.java:
##########
@@ -0,0 +1,104 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.client.registry;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.service.ServiceStateException;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Base class for {@code AMRegistry} implementations.
+ *
+ * <p>The specific implementation is configured via the
+ * {@code tez.am.registry.class} property.</p>
+ *
+ * <p>Implementations are expected to provide appropriate service lifecycle
+ * behavior, including:
+ * <ul>
+ * <li>{@code init}</li>
+ * <li>{@code serviceStart}</li>
+ * <li>{@code serviceStop}</li>
+ * </ul>
+ * </p>
+ *
+ * <p>{@code init} and {@code serviceStart} are invoked during
+ * {@code DAGAppMaster.serviceInit()}, while {@code serviceStop} is called
+ * when {@code DAGAppMaster} shuts down.</p>
+ */
+public abstract class AMRegistry extends AbstractService {
+
+ private static final Logger LOG = LoggerFactory.getLogger(AMRegistry.class);
+ private List<AMRecord> amRecords = new ArrayList<>();
+
+ @Override
+ public void init(Configuration conf) {
+ try {
+ this.serviceInit(conf);
+ } catch (Exception e) {
+ LOG.error("Failed to init AMRegistry: name={}, type={}", getName(), getClass().getName());
+ throw ServiceStateException.convert(e);
+ }
+ }
+
+ @Override
+ public void start() {
+ try {
+ this.serviceStart();
+ } catch(Exception e) {
+ LOG.error("Failed to start AMRegistry: name={}, type={}", getName(), getClass().getName());
+ throw ServiceStateException.convert(e);
+ }
+ }
+
+ /* Implementations should provide a public no-arg constructor */
+ protected AMRegistry(String name) {
+ super(name);
+ }
+
+ /* Under typical usage, add will be called once automatically with an AMRecord
+ for the DAGClientServer servicing an AM
+ */
+ public void add(AMRecord server) throws Exception {
+ amRecords.add(server);
+ }
+
+ public abstract void remove(AMRecord server) throws Exception;
+
+ public ApplicationId generateNewId() throws Exception {
+ return null;
+ }
+
+ public abstract AMRecord createAmRecord(ApplicationId appId, String hostName, String hostIp, int port,
+ String computeName);
+
+ @Override
+ public void serviceStop() throws Exception {
+ List<AMRecord> records = new ArrayList<>(amRecords);
+ for(AMRecord record : records) {
+ remove(record);
Review Comment:
if one remove fails, the subsequent won't be removed & won't be tried either. we should catch -> log and proceed
##########
tez-api/src/main/java/org/apache/tez/client/registry/AMRegistry.java:
##########
@@ -0,0 +1,104 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.client.registry;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.service.ServiceStateException;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Base class for {@code AMRegistry} implementations.
+ *
+ * <p>The specific implementation is configured via the
+ * {@code tez.am.registry.class} property.</p>
+ *
+ * <p>Implementations are expected to provide appropriate service lifecycle
+ * behavior, including:
+ * <ul>
+ * <li>{@code init}</li>
+ * <li>{@code serviceStart}</li>
+ * <li>{@code serviceStop}</li>
+ * </ul>
+ * </p>
+ *
+ * <p>{@code init} and {@code serviceStart} are invoked during
+ * {@code DAGAppMaster.serviceInit()}, while {@code serviceStop} is called
+ * when {@code DAGAppMaster} shuts down.</p>
+ */
+public abstract class AMRegistry extends AbstractService {
+
+ private static final Logger LOG = LoggerFactory.getLogger(AMRegistry.class);
+ private List<AMRecord> amRecords = new ArrayList<>();
+
+ @Override
+ public void init(Configuration conf) {
+ try {
+ this.serviceInit(conf);
+ } catch (Exception e) {
+ LOG.error("Failed to init AMRegistry: name={}, type={}", getName(), getClass().getName());
+ throw ServiceStateException.convert(e);
+ }
+ }
+
+ @Override
+ public void start() {
+ try {
+ this.serviceStart();
+ } catch(Exception e) {
+ LOG.error("Failed to start AMRegistry: name={}, type={}", getName(), getClass().getName());
+ throw ServiceStateException.convert(e);
+ }
+ }
Review Comment:
why are u overriding these?
AbstractService.init() does several things:
sets the state
checks transitions
notifies listeners
calls serviceInit(conf)
By overriding init() and directly calling serviceInit(conf) you are skipping:
enterState(STATE.INITED)
state transition verification
listener invocations
context setup
##########
tez-api/src/main/java/org/apache/tez/client/registry/AMRegistry.java:
##########
@@ -0,0 +1,104 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.client.registry;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.service.ServiceStateException;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Base class for {@code AMRegistry} implementations.
+ *
+ * <p>The specific implementation is configured via the
+ * {@code tez.am.registry.class} property.</p>
+ *
+ * <p>Implementations are expected to provide appropriate service lifecycle
+ * behavior, including:
+ * <ul>
+ * <li>{@code init}</li>
+ * <li>{@code serviceStart}</li>
+ * <li>{@code serviceStop}</li>
+ * </ul>
+ * </p>
+ *
+ * <p>{@code init} and {@code serviceStart} are invoked during
+ * {@code DAGAppMaster.serviceInit()}, while {@code serviceStop} is called
+ * when {@code DAGAppMaster} shuts down.</p>
+ */
+public abstract class AMRegistry extends AbstractService {
+
+ private static final Logger LOG = LoggerFactory.getLogger(AMRegistry.class);
+ private List<AMRecord> amRecords = new ArrayList<>();
+
+ @Override
+ public void init(Configuration conf) {
+ try {
+ this.serviceInit(conf);
+ } catch (Exception e) {
+ LOG.error("Failed to init AMRegistry: name={}, type={}", getName(), getClass().getName());
+ throw ServiceStateException.convert(e);
+ }
+ }
+
+ @Override
+ public void start() {
+ try {
+ this.serviceStart();
+ } catch(Exception e) {
+ LOG.error("Failed to start AMRegistry: name={}, type={}", getName(), getClass().getName());
+ throw ServiceStateException.convert(e);
+ }
+ }
+
+ /* Implementations should provide a public no-arg constructor */
+ protected AMRegistry(String name) {
+ super(name);
+ }
+
+ /* Under typical usage, add will be called once automatically with an AMRecord
+ for the DAGClientServer servicing an AM
+ */
+ public void add(AMRecord server) throws Exception {
+ amRecords.add(server);
+ }
+
+ public abstract void remove(AMRecord server) throws Exception;
+
+ public ApplicationId generateNewId() throws Exception {
+ return null;
+ }
Review Comment:
this `null` default doesn't look very safe, why don't we have this method as abstract?
##########
tez-api/src/main/java/org/apache/tez/client/registry/zookeeper/ZkAMRegistryClient.java:
##########
@@ -0,0 +1,205 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.client.registry.zookeeper;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
+
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.imps.CuratorFrameworkState;
+import org.apache.curator.framework.recipes.cache.ChildData;
+import org.apache.curator.framework.recipes.cache.TreeCache;
+import org.apache.curator.framework.recipes.cache.TreeCacheEvent;
+import org.apache.curator.framework.recipes.cache.TreeCacheListener;
+import org.apache.curator.shaded.com.google.common.base.Charsets;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.registry.client.binding.RegistryUtils;
+import org.apache.hadoop.registry.client.types.ServiceRecord;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.tez.client.registry.AMRecord;
+import org.apache.tez.client.registry.AMRegistryClient;
+import org.apache.tez.client.registry.AMRegistryClientListener;
+import org.apache.tez.dag.api.TezConfiguration;
+
+import com.fasterxml.jackson.core.JsonParseException;
+import com.google.common.base.Preconditions;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Curator/Zookeeper implementation of {@link AMRegistryClient}.
+ */
+@InterfaceAudience.Public
+public final class ZkAMRegistryClient extends AMRegistryClient {
+ private static final Logger LOG = LoggerFactory.getLogger(ZkAMRegistryClient.class);
+ private static final Map<String, ZkAMRegistryClient> INSTANCES = new HashMap<>();
+
+ private final Configuration conf;
+ //Cache of known AMs
+ private final ConcurrentHashMap<String, AMRecord> amRecordCache = new ConcurrentHashMap<>();
+ private CuratorFramework client;
+
+ private ZkAMRegistryClient(final Configuration conf) {
+ this.conf = conf;
+ }
+
+ public static synchronized ZkAMRegistryClient getClient(final Configuration conf) {
+ String namespace = conf.get(TezConfiguration.TEZ_AM_REGISTRY_NAMESPACE);
+ ZkAMRegistryClient registry = INSTANCES.get(namespace);
+ if (registry == null) {
+ registry = new ZkAMRegistryClient(conf);
+ INSTANCES.put(namespace, registry);
+ }
+ LOG.info("Returning tez AM registry ({}) for namespace '{}'", System.identityHashCode(registry), namespace);
+ return registry;
+ }
+
+ /**
+ * Deserializes a {@link ServiceRecord} from ZooKeeper data and converts it into an {@link AMRecord}
+ * for caching.
+ *
+ * @param childData the ZooKeeper node data containing a serialized {@link ServiceRecord}
+ * @return an {@link AMRecord} constructed from the deserialized {@link ServiceRecord}, or {@code null}
+ * if no data is present
+ * @throws IOException if the data cannot be deserialized into a {@link ServiceRecord}
+ */
+ public static AMRecord getAMRecord(final ChildData childData) throws IOException {
+ // not a leaf path. Only leaf path contains AMRecord
+ if (!childData.getPath().contains(ApplicationId.appIdStrPrefix)) {
+ return null;
+ }
+ byte[] data = childData.getData();
+ // only the path appeared, there is no data yet
+ if (data.length == 0) {
+ return null;
+ }
+ String value = new String(data, Charsets.UTF_8);
+ try {
+ RegistryUtils.ServiceRecordMarshal marshal = new RegistryUtils.ServiceRecordMarshal();
+ ServiceRecord serviceRecord = marshal.fromJson(value);
+ return new AMRecord(serviceRecord);
+ } catch (JsonParseException e) {
+ //Not a json AMRecord (SRV), could be some other data.
+ LOG.warn("Non-json data received while de-serializing AMRecord: {}. Ignoring...", value);
+ return null;
+ }
+ }
+
+ public void start() throws Exception {
+ ZkConfig zkConf = new ZkConfig(this.conf);
+ client = zkConf.createCuratorFramework();
+ final TreeCache cache = new TreeCache(client, zkConf.getZkNamespace());
+ client.start();
+ cache.start();
+ cache.getListenable().addListener(new ZkRegistryListener());
+ }
+
+ @Override
+ public AMRecord getRecord(String appId) {
+ if (amRecordCache.get(appId) == null) {
+ return null;
+ }
+ //Return a copy
+ return new AMRecord(amRecordCache.get(appId));
+ }
+
+ @Override
+ public List<AMRecord> getAllRecords() {
+ return amRecordCache.values().stream().map(AMRecord::new).collect(Collectors.toList());
+ }
+
+ @Override
+ public synchronized void addListener(AMRegistryClientListener listener) {
+ getListeners().add(listener);
+ }
+
+ @Override
+ public void close() {
+ client.close();
+ }
+
+ /**
+ * Callback listener for ZooKeeper events that updates the local cache
+ * when child nodes under the monitored path change.
+ */
+ private class ZkRegistryListener implements TreeCacheListener {
+
+ @Override
+ public void childEvent(final CuratorFramework clientParam, final TreeCacheEvent event) throws Exception {
+ Preconditions.checkArgument(clientParam != null && clientParam.getState() == CuratorFrameworkState.STARTED,
+ "Curator client is not started");
+
+ ChildData childData = event.getData();
+ switch (event.getType()) {
+ case NODE_ADDED:
+ if (isEmpty(childData)) {
+ LOG.info("AppId allocated: {}", childData.getPath());
+ } else {
+ AMRecord amRecord = getAMRecord(childData);
+ if (amRecord != null) {
+ LOG.info("AM registered with data: {}. Notifying {} listeners.", amRecord, getListeners().size());
+ amRecordCache.put(amRecord.getApplicationId().toString(), amRecord);
+ notifyOnAdded(amRecord);
+ }
+ }
+ break;
+ case NODE_UPDATED:
+ if (isEmpty(childData)) {
+ throw new RuntimeException("AM updated with empty data");
+ } else {
+ AMRecord amRecord = getAMRecord(childData);
+ if (amRecord != null) {
+ LOG.info("AM updated data: {}. Notifying {} listeners.", amRecord, getListeners().size());
+ amRecordCache.put(amRecord.getApplicationId().toString(), amRecord);
+ notifyOnAdded(amRecord);
+ }
+ }
+ break;
+ case NODE_REMOVED:
+ if (isEmpty(childData)) {
+ LOG.info("Unused AppId unregistered: {}", childData.getPath());
+ } else {
+ AMRecord amRecord = getAMRecord(childData);
+ if (amRecord != null) {
+ LOG.info("AM removed: {}. Notifying {} listeners.", amRecord, getListeners().size());
+ amRecordCache.remove(amRecord.getApplicationId().toString(), amRecord);
Review Comment:
This removes only if the old value equals the instance passed.
But you store a new instance on insert & Then on event removal, getAMRecord(childData) again creates another new AMRecord,
##########
tez-api/src/main/java/org/apache/tez/client/registry/zookeeper/ZkAMRegistryClient.java:
##########
@@ -0,0 +1,205 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.client.registry.zookeeper;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
+
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.imps.CuratorFrameworkState;
+import org.apache.curator.framework.recipes.cache.ChildData;
+import org.apache.curator.framework.recipes.cache.TreeCache;
+import org.apache.curator.framework.recipes.cache.TreeCacheEvent;
+import org.apache.curator.framework.recipes.cache.TreeCacheListener;
+import org.apache.curator.shaded.com.google.common.base.Charsets;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.registry.client.binding.RegistryUtils;
+import org.apache.hadoop.registry.client.types.ServiceRecord;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.tez.client.registry.AMRecord;
+import org.apache.tez.client.registry.AMRegistryClient;
+import org.apache.tez.client.registry.AMRegistryClientListener;
+import org.apache.tez.dag.api.TezConfiguration;
+
+import com.fasterxml.jackson.core.JsonParseException;
+import com.google.common.base.Preconditions;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Curator/Zookeeper implementation of {@link AMRegistryClient}.
+ */
+@InterfaceAudience.Public
+public final class ZkAMRegistryClient extends AMRegistryClient {
+ private static final Logger LOG = LoggerFactory.getLogger(ZkAMRegistryClient.class);
+ private static final Map<String, ZkAMRegistryClient> INSTANCES = new HashMap<>();
+
+ private final Configuration conf;
+ //Cache of known AMs
+ private final ConcurrentHashMap<String, AMRecord> amRecordCache = new ConcurrentHashMap<>();
+ private CuratorFramework client;
+
+ private ZkAMRegistryClient(final Configuration conf) {
+ this.conf = conf;
+ }
+
+ public static synchronized ZkAMRegistryClient getClient(final Configuration conf) {
+ String namespace = conf.get(TezConfiguration.TEZ_AM_REGISTRY_NAMESPACE);
+ ZkAMRegistryClient registry = INSTANCES.get(namespace);
+ if (registry == null) {
+ registry = new ZkAMRegistryClient(conf);
+ INSTANCES.put(namespace, registry);
+ }
+ LOG.info("Returning tez AM registry ({}) for namespace '{}'", System.identityHashCode(registry), namespace);
+ return registry;
+ }
+
+ /**
+ * Deserializes a {@link ServiceRecord} from ZooKeeper data and converts it into an {@link AMRecord}
+ * for caching.
+ *
+ * @param childData the ZooKeeper node data containing a serialized {@link ServiceRecord}
+ * @return an {@link AMRecord} constructed from the deserialized {@link ServiceRecord}, or {@code null}
+ * if no data is present
+ * @throws IOException if the data cannot be deserialized into a {@link ServiceRecord}
+ */
+ public static AMRecord getAMRecord(final ChildData childData) throws IOException {
+ // not a leaf path. Only leaf path contains AMRecord
+ if (!childData.getPath().contains(ApplicationId.appIdStrPrefix)) {
+ return null;
+ }
+ byte[] data = childData.getData();
+ // only the path appeared, there is no data yet
+ if (data.length == 0) {
+ return null;
+ }
+ String value = new String(data, Charsets.UTF_8);
+ try {
+ RegistryUtils.ServiceRecordMarshal marshal = new RegistryUtils.ServiceRecordMarshal();
+ ServiceRecord serviceRecord = marshal.fromJson(value);
+ return new AMRecord(serviceRecord);
+ } catch (JsonParseException e) {
+ //Not a json AMRecord (SRV), could be some other data.
+ LOG.warn("Non-json data received while de-serializing AMRecord: {}. Ignoring...", value);
+ return null;
+ }
+ }
+
+ public void start() throws Exception {
+ ZkConfig zkConf = new ZkConfig(this.conf);
+ client = zkConf.createCuratorFramework();
+ final TreeCache cache = new TreeCache(client, zkConf.getZkNamespace());
+ client.start();
+ cache.start();
+ cache.getListenable().addListener(new ZkRegistryListener());
+ }
+
+ @Override
+ public AMRecord getRecord(String appId) {
+ if (amRecordCache.get(appId) == null) {
+ return null;
+ }
+ //Return a copy
+ return new AMRecord(amRecordCache.get(appId));
+ }
+
+ @Override
+ public List<AMRecord> getAllRecords() {
+ return amRecordCache.values().stream().map(AMRecord::new).collect(Collectors.toList());
+ }
+
+ @Override
+ public synchronized void addListener(AMRegistryClientListener listener) {
+ getListeners().add(listener);
+ }
+
+ @Override
+ public void close() {
+ client.close();
+ }
+
+ /**
+ * Callback listener for ZooKeeper events that updates the local cache
+ * when child nodes under the monitored path change.
+ */
+ private class ZkRegistryListener implements TreeCacheListener {
+
+ @Override
+ public void childEvent(final CuratorFramework clientParam, final TreeCacheEvent event) throws Exception {
+ Preconditions.checkArgument(clientParam != null && clientParam.getState() == CuratorFrameworkState.STARTED,
+ "Curator client is not started");
+
+ ChildData childData = event.getData();
+ switch (event.getType()) {
+ case NODE_ADDED:
+ if (isEmpty(childData)) {
+ LOG.info("AppId allocated: {}", childData.getPath());
+ } else {
+ AMRecord amRecord = getAMRecord(childData);
+ if (amRecord != null) {
+ LOG.info("AM registered with data: {}. Notifying {} listeners.", amRecord, getListeners().size());
+ amRecordCache.put(amRecord.getApplicationId().toString(), amRecord);
+ notifyOnAdded(amRecord);
+ }
+ }
+ break;
+ case NODE_UPDATED:
+ if (isEmpty(childData)) {
+ throw new RuntimeException("AM updated with empty data");
+ } else {
+ AMRecord amRecord = getAMRecord(childData);
+ if (amRecord != null) {
+ LOG.info("AM updated data: {}. Notifying {} listeners.", amRecord, getListeners().size());
Review Comment:
I am not sure in this entire class, if the Logging is too verbose at INFO, maybe some costly ones should be moved to debug
##########
tez-api/src/main/java/org/apache/tez/client/registry/AMRegistryClient.java:
##########
@@ -0,0 +1,98 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.client.registry;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Client-side interface for discovering Application Master (AM) instances
+ * registered in the AM registry.
+ *
+ * <p>Implementations are responsible for locating AM endpoints and returning
+ * their metadata. This API is used by client components to discover running
+ * Tez AMs.</p>
+ *
+ * <p>Listeners may be registered to receive notifications when AM records
+ * appear or are removed.</p>
+ */
+public abstract class AMRegistryClient implements Closeable {
+
+ private final List<AMRegistryClientListener> listeners = new ArrayList<>();
+
+ /**
+ * Returns the current set of registered listeners.
+ *
+ * @return a mutable list of listeners
+ */
+ protected List<AMRegistryClientListener> getListeners() {
+ return listeners;
+ }
+
+ /**
+ * Lookup AM metadata for the given application ID.
+ *
+ * @param appId the application ID
+ * @return the AM record if found, otherwise {@code null}
+ * @throws IOException if the lookup fails
+ */
+ public abstract AMRecord getRecord(String appId) throws IOException;
+
+ /**
+ * Retrieve all AM records known in the registry.
+ *
+ * @return a list of AM records (possibly empty)
+ * @throws IOException if the fetch fails
+ */
+ public abstract List<AMRecord> getAllRecords() throws IOException;
+
+ /**
+ * Register a listener for AM registry events.
+ * The listener will be notified when AM records are added or removed.
+ *
+ * @param listener the listener to add
+ */
+ public synchronized void addListener(AMRegistryClientListener listener) {
+ listeners.add(listener);
+ }
+
+ /**
+ * Notify listeners of a newly added AM record.
+ *
+ * @param record the added AM record
+ */
+ protected synchronized void notifyOnAdded(AMRecord record) {
+ for (AMRegistryClientListener listener : listeners) {
+ listener.onAdd(record);
Review Comment:
if one listener throws, all others will be skipped
##########
tez-api/src/main/java/org/apache/tez/client/registry/zookeeper/ZkConfig.java:
##########
@@ -0,0 +1,128 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.client.registry.zookeeper;
+
+import org.apache.curator.RetryPolicy;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.CuratorFrameworkFactory;
+import org.apache.curator.retry.ExponentialBackoffRetry;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.tez.dag.api.TezConfiguration;
+
+import com.google.common.base.Preconditions;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class ZkConfig {
+ private static final Logger LOG = LoggerFactory.getLogger(ZkConfig.class);
+
+ // if namespace defined in config is 'foo' and COMPUTE_GROUP_NAME env is 'bar' then the zkpaths will be of format
+ // /tez-external-sessions/foo/bar
+ private final static String ZK_NAMESPACE_PREFIX = "/tez-external-sessions";
+ public final static String COMPUTE_GROUP_NAME_ENV = "COMPUTE_GROUP_NAME";
+ public final static String DEFAULT_COMPUTE_GROUP_NAME = "default-compute";
+
+ private final String zkQuorum;
+ private final String zkNamespace;
+ private final int curatorBackoffSleep;
+ private final int curatorMaxRetries;
+ private final int sessionTimeoutMs;
+ private final int connectionTimeoutMs;
+
+ public ZkConfig(Configuration conf) {
+ zkQuorum = conf.get(TezConfiguration.TEZ_AM_ZOOKEEPER_QUORUM);
+ Preconditions.checkNotNull(zkQuorum);
+
+ String fullZkNamespace = ZK_NAMESPACE_PREFIX;
+
+ String namespace = conf.get(TezConfiguration.TEZ_AM_REGISTRY_NAMESPACE,
+ TezConfiguration.TEZ_AM_REGISTRY_NAMESPACE_DEFAULT);
+ Preconditions.checkNotNull(namespace);
+ if (namespace.startsWith(Path.SEPARATOR)) {
+ fullZkNamespace += namespace;
+ } else {
+ fullZkNamespace = fullZkNamespace + Path.SEPARATOR + namespace;
+ }
+
+ boolean enableComputeGroups = conf.getBoolean(TezConfiguration.TEZ_AM_REGISTRY_ENABLE_COMPUTE_GROUPS,
+ TezConfiguration.TEZ_AM_REGISTRY_ENABLE_COMPUTE_GROUPS_DEFAULT);
+ if (enableComputeGroups) {
+ final String subNamespace = System.getenv(COMPUTE_GROUP_NAME_ENV);
+ if (subNamespace != null && !subNamespace.isEmpty()) {
+ if (subNamespace.startsWith(Path.SEPARATOR)) {
+ fullZkNamespace += subNamespace;
+ } else {
+ fullZkNamespace = fullZkNamespace + Path.SEPARATOR + subNamespace;
+ }
+ LOG.info("Compute groups enabled: subNamespace: {} fullZkNamespace: {}", subNamespace, fullZkNamespace);
+ }
+ } else {
+ LOG.info("Compute groups disabled: fullZkNamespace: {}", fullZkNamespace);
+ }
+ zkNamespace = fullZkNamespace;
+
+ curatorBackoffSleep = conf.getInt(TezConfiguration.TEZ_AM_CURATOR_BACKOFF_SLEEP,
+ TezConfiguration.TEZ_AM_CURATOR_BACKOFF_SLEEP_DEFAULT);
+ curatorMaxRetries = conf.getInt(TezConfiguration.TEZ_AM_CURATOR_MAX_RETRIES,
+ TezConfiguration.TEZ_AM_CURATOR_MAX_RETRIES_DEFAULT);
+ sessionTimeoutMs = conf.getInt(TezConfiguration.TEZ_AM_CURATOR_SESSION_TIMEOUT,
+ TezConfiguration.TEZ_AM_CURATOR_SESSION_TIMEOUT_DEFAULT);
+ connectionTimeoutMs = conf.getInt(TezConfiguration.TEZ_AM_CURATOR_CONNECTION_TIMEOUT,
+ TezConfiguration.TEZ_AM_CURATOR_CONNECTION_TIMEOUT_DEFAULT);
+ }
+
+ public String getZkQuorum() {
+ return zkQuorum;
+ }
+
+ public String getZkNamespace() {
+ return zkNamespace;
+ }
+
+ public int getCuratorBackoffSleep() {
+ return curatorBackoffSleep;
+ }
+
+ public int getCuratorMaxRetries() {
+ return curatorMaxRetries;
+ }
+
+ public int getSessionTimeoutMs() {
+ return sessionTimeoutMs;
+ }
+
+ public int getConnectionTimeoutMs() {
+ return connectionTimeoutMs;
+ }
+
+ public RetryPolicy getRetryPolicy() {
+ return new ExponentialBackoffRetry(getCuratorBackoffSleep(), getCuratorMaxRetries());
+ }
+
+ public CuratorFramework createCuratorFramework() {
Review Comment:
we do have create here, but no close, sounds like by design if we have create we should have close
##########
tez-api/src/main/java/org/apache/tez/client/registry/zookeeper/ZkConfig.java:
##########
@@ -0,0 +1,128 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.client.registry.zookeeper;
+
+import org.apache.curator.RetryPolicy;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.CuratorFrameworkFactory;
+import org.apache.curator.retry.ExponentialBackoffRetry;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.tez.dag.api.TezConfiguration;
+
+import com.google.common.base.Preconditions;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class ZkConfig {
+ private static final Logger LOG = LoggerFactory.getLogger(ZkConfig.class);
+
+ // if namespace defined in config is 'foo' and COMPUTE_GROUP_NAME env is 'bar' then the zkpaths will be of format
+ // /tez-external-sessions/foo/bar
+ private final static String ZK_NAMESPACE_PREFIX = "/tez-external-sessions";
+ public final static String COMPUTE_GROUP_NAME_ENV = "COMPUTE_GROUP_NAME";
+ public final static String DEFAULT_COMPUTE_GROUP_NAME = "default-compute";
+
+ private final String zkQuorum;
+ private final String zkNamespace;
+ private final int curatorBackoffSleep;
+ private final int curatorMaxRetries;
+ private final int sessionTimeoutMs;
+ private final int connectionTimeoutMs;
+
+ public ZkConfig(Configuration conf) {
+ zkQuorum = conf.get(TezConfiguration.TEZ_AM_ZOOKEEPER_QUORUM);
+ Preconditions.checkNotNull(zkQuorum);
Review Comment:
we should block for empty string as well
##########
tez-api/src/main/java/org/apache/tez/client/registry/zookeeper/ZkConfig.java:
##########
@@ -0,0 +1,128 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.client.registry.zookeeper;
+
+import org.apache.curator.RetryPolicy;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.CuratorFrameworkFactory;
+import org.apache.curator.retry.ExponentialBackoffRetry;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.tez.dag.api.TezConfiguration;
+
+import com.google.common.base.Preconditions;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class ZkConfig {
+ private static final Logger LOG = LoggerFactory.getLogger(ZkConfig.class);
+
+ // if namespace defined in config is 'foo' and COMPUTE_GROUP_NAME env is 'bar' then the zkpaths will be of format
+ // /tez-external-sessions/foo/bar
+ private final static String ZK_NAMESPACE_PREFIX = "/tez-external-sessions";
+ public final static String COMPUTE_GROUP_NAME_ENV = "COMPUTE_GROUP_NAME";
+ public final static String DEFAULT_COMPUTE_GROUP_NAME = "default-compute";
+
+ private final String zkQuorum;
+ private final String zkNamespace;
+ private final int curatorBackoffSleep;
+ private final int curatorMaxRetries;
+ private final int sessionTimeoutMs;
+ private final int connectionTimeoutMs;
+
+ public ZkConfig(Configuration conf) {
+ zkQuorum = conf.get(TezConfiguration.TEZ_AM_ZOOKEEPER_QUORUM);
+ Preconditions.checkNotNull(zkQuorum);
+
+ String fullZkNamespace = ZK_NAMESPACE_PREFIX;
+
+ String namespace = conf.get(TezConfiguration.TEZ_AM_REGISTRY_NAMESPACE,
Review Comment:
should we normalize the namespace:
```
namespace = Path.SEPARATOR + namespace.replaceAll("^/+", "");
```
##########
tez-api/src/main/java/org/apache/tez/client/registry/zookeeper/ZkFrameworkClient.java:
##########
@@ -0,0 +1,164 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.client.registry.zookeeper;
+
+import java.io.IOException;
+
+import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.client.api.YarnClientApplication;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.util.Records;
+import org.apache.tez.client.FrameworkClient;
+import org.apache.tez.client.registry.AMRecord;
+import org.apache.tez.dag.api.TezConfiguration;
+
+public class ZkFrameworkClient extends FrameworkClient {
+
+ private AMRecord amRecord;
+ private ZkAMRegistryClient amRegistryClient = null;
+ private volatile boolean isRunning = false;
+ private String amHost;
+ private int amPort;
+
+ @Override
+ public synchronized void init(TezConfiguration tezConf) {
+ if (this.amRegistryClient == null) {
+ try {
+ this.amRegistryClient = ZkAMRegistryClient.getClient(tezConf);
+ this.isRunning = true;
Review Comment:
this is wrong, init doesn't means started, init sets the state to initialized and start sets it to running
##########
tez-api/src/main/java/org/apache/tez/client/registry/zookeeper/ZkConfig.java:
##########
@@ -0,0 +1,128 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.client.registry.zookeeper;
+
+import org.apache.curator.RetryPolicy;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.CuratorFrameworkFactory;
+import org.apache.curator.retry.ExponentialBackoffRetry;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.tez.dag.api.TezConfiguration;
+
+import com.google.common.base.Preconditions;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class ZkConfig {
+ private static final Logger LOG = LoggerFactory.getLogger(ZkConfig.class);
+
+ // if namespace defined in config is 'foo' and COMPUTE_GROUP_NAME env is 'bar' then the zkpaths will be of format
+ // /tez-external-sessions/foo/bar
+ private final static String ZK_NAMESPACE_PREFIX = "/tez-external-sessions";
+ public final static String COMPUTE_GROUP_NAME_ENV = "COMPUTE_GROUP_NAME";
+ public final static String DEFAULT_COMPUTE_GROUP_NAME = "default-compute";
+
+ private final String zkQuorum;
+ private final String zkNamespace;
+ private final int curatorBackoffSleep;
+ private final int curatorMaxRetries;
+ private final int sessionTimeoutMs;
+ private final int connectionTimeoutMs;
+
+ public ZkConfig(Configuration conf) {
+ zkQuorum = conf.get(TezConfiguration.TEZ_AM_ZOOKEEPER_QUORUM);
+ Preconditions.checkNotNull(zkQuorum);
+
+ String fullZkNamespace = ZK_NAMESPACE_PREFIX;
+
+ String namespace = conf.get(TezConfiguration.TEZ_AM_REGISTRY_NAMESPACE,
+ TezConfiguration.TEZ_AM_REGISTRY_NAMESPACE_DEFAULT);
+ Preconditions.checkNotNull(namespace);
+ if (namespace.startsWith(Path.SEPARATOR)) {
+ fullZkNamespace += namespace;
+ } else {
+ fullZkNamespace = fullZkNamespace + Path.SEPARATOR + namespace;
+ }
+
+ boolean enableComputeGroups = conf.getBoolean(TezConfiguration.TEZ_AM_REGISTRY_ENABLE_COMPUTE_GROUPS,
+ TezConfiguration.TEZ_AM_REGISTRY_ENABLE_COMPUTE_GROUPS_DEFAULT);
+ if (enableComputeGroups) {
+ final String subNamespace = System.getenv(COMPUTE_GROUP_NAME_ENV);
Review Comment:
I think we should normalize this like namespace
##########
tez-api/src/main/java/org/apache/tez/frameworkplugins/FrameworkUtils.java:
##########
@@ -0,0 +1,96 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tez.frameworkplugins;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.ServiceLoader;
+
+import javax.annotation.Nullable;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.tez.common.ReflectionUtils;
+import org.apache.tez.dag.api.TezConfiguration;
+import org.apache.tez.dag.api.TezConstants;
+import org.apache.tez.dag.api.TezReflectionException;
+
+public final class FrameworkUtils {
+
+ private FrameworkUtils() {}
+
+ /*
+ Searches for a FrameworkService provider which implements a target interface.
+ The interface should be either ClientFrameworkService or ServerFrameworkService.
+ Depending on which interface is used, either the client or server class of a
+ matching FrameworkMode will be used as the implementation.
+
+ NOTE: Layering of FrameworkServices in a decorator-style is currently not supported
+
+ An implementation is searched in the following order:
+ 1. If conf is not null and the parameter TEZ_FRAMEWORK_MODE is set:
+ the value of TEZ_FRAMEWORK_MODE from the conf will be used
+ 2. If conf is null or the parameter TEZ_FRAMEWORK_MODE is not set
+ and the environment var TEZ_FRAMEWORK_MODE is not empty:
+ the value of the environment var will be used
+ 3. Otherwise:
+ the default java.util.ServiceLoader behavior will be used,
+ i.e. the implementation classname should appear in a file on the classpath at the location
+ META-INF/services/org.apache.tez.frameworkplugins.ClientFrameworkService
+ or META-INF/services/org.apache.tez.frameworkplugins.ServerFrameworkService
+ */
+ public static <T extends FrameworkService> T get(Class<T> interfaze, @Nullable Configuration conf) {
+ try {
+ if ((conf != null) && (conf.get(TezConfiguration.TEZ_FRAMEWORK_MODE) != null)) {
+ return getByMode(interfaze, conf.get(TezConfiguration.TEZ_FRAMEWORK_MODE));
+ } else if (System.getenv(TezConstants.TEZ_FRAMEWORK_MODE) != null) {
+ return getByMode(interfaze, System.getenv(TezConstants.TEZ_FRAMEWORK_MODE));
+ } else {
+ return getByServiceLoader(interfaze);
+ }
+ } catch (TezReflectionException e) {
+ throw new RuntimeException("Failed to load framework service for interface: " + interfaze.getName(), e);
+ }
+ }
+
+ private static <T extends FrameworkService> T getByServiceLoader(Class<T> interfaze) {
+ List<T> services = new ArrayList<>();
+ ServiceLoader<T> frameworkService = ServiceLoader.load(interfaze);
+ for (T service : frameworkService) {
+ services.add(service);
+ }
+ if (services.isEmpty()) {
+ return null;
Review Comment:
in the last callers we are doing `null` checks and then doing `new YarnServerFrameworkService()``, why don't we just return that here itself and save those redundant `null` checks
##########
tez-api/src/main/java/org/apache/tez/frameworkplugins/FrameworkUtils.java:
##########
@@ -0,0 +1,96 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tez.frameworkplugins;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.ServiceLoader;
+
+import javax.annotation.Nullable;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.tez.common.ReflectionUtils;
+import org.apache.tez.dag.api.TezConfiguration;
+import org.apache.tez.dag.api.TezConstants;
+import org.apache.tez.dag.api.TezReflectionException;
+
+public final class FrameworkUtils {
+
+ private FrameworkUtils() {}
+
+ /*
+ Searches for a FrameworkService provider which implements a target interface.
+ The interface should be either ClientFrameworkService or ServerFrameworkService.
+ Depending on which interface is used, either the client or server class of a
+ matching FrameworkMode will be used as the implementation.
+
+ NOTE: Layering of FrameworkServices in a decorator-style is currently not supported
+
+ An implementation is searched in the following order:
+ 1. If conf is not null and the parameter TEZ_FRAMEWORK_MODE is set:
+ the value of TEZ_FRAMEWORK_MODE from the conf will be used
+ 2. If conf is null or the parameter TEZ_FRAMEWORK_MODE is not set
+ and the environment var TEZ_FRAMEWORK_MODE is not empty:
+ the value of the environment var will be used
+ 3. Otherwise:
+ the default java.util.ServiceLoader behavior will be used,
+ i.e. the implementation classname should appear in a file on the classpath at the location
+ META-INF/services/org.apache.tez.frameworkplugins.ClientFrameworkService
+ or META-INF/services/org.apache.tez.frameworkplugins.ServerFrameworkService
+ */
+ public static <T extends FrameworkService> T get(Class<T> interfaze, @Nullable Configuration conf) {
+ try {
+ if ((conf != null) && (conf.get(TezConfiguration.TEZ_FRAMEWORK_MODE) != null)) {
+ return getByMode(interfaze, conf.get(TezConfiguration.TEZ_FRAMEWORK_MODE));
+ } else if (System.getenv(TezConstants.TEZ_FRAMEWORK_MODE) != null) {
+ return getByMode(interfaze, System.getenv(TezConstants.TEZ_FRAMEWORK_MODE));
+ } else {
+ return getByServiceLoader(interfaze);
+ }
+ } catch (TezReflectionException e) {
+ throw new RuntimeException("Failed to load framework service for interface: " + interfaze.getName(), e);
+ }
+ }
+
+ private static <T extends FrameworkService> T getByServiceLoader(Class<T> interfaze) {
+ List<T> services = new ArrayList<>();
+ ServiceLoader<T> frameworkService = ServiceLoader.load(interfaze);
+ for (T service : frameworkService) {
+ services.add(service);
+ }
+ if (services.isEmpty()) {
+ return null;
+ } else if (services.size() > 1) {
+ throw new RuntimeException("Layering of multiple framework services is not supported."
+ + " Please provide only one implementation class in configuration.");
+ }
+ //services is guaranteed to have one element at this point
+ return services.getFirst();
+ }
+
+ private static <T> T getByMode(Class<T> interfaze, String mode) throws TezReflectionException {
+ mode = mode.toUpperCase();
+ String clazz;
+ if (interfaze == ClientFrameworkService.class) {
+ clazz = FrameworkMode.valueOf(mode).getClientClassName();
+ } else {
+ clazz = FrameworkMode.valueOf(mode).getServerClassName();
Review Comment:
it assumes if it is not client class, it would be server class, in case of accidental usage: `FrameworkUtils.get(SomeOtherInterface.class, conf)`, it would lead to wierd errors later
##########
tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java:
##########
@@ -2344,16 +2350,74 @@ static Set<String> getPropertySet() {
public static final String TEZ_TASK_ATTEMPT_HOOKS = TEZ_TASK_PREFIX + "attempt.hooks";
/**
- * Comma-separated list of additional hadoop config files to load from CLASSPATH in standalone mode.
+ * String value. ZooKeeper quorum connection string used when creating a CuratorFramework for the ZooKeeper registry.
*/
@ConfigurationScope(Scope.AM)
@ConfigurationProperty
- public static final String TEZ_AM_STANDALONE_CONFS = TEZ_AM_PREFIX + "standalone.confs";
+ public static final String TEZ_AM_ZOOKEEPER_QUORUM = TEZ_AM_PREFIX + "zookeeper.quorum";
/**
- * String value. The class to be used for the AM registry.
- */
+ * String value. Namespace in ZooKeeper registry for the Application Master.
+ */
+ @ConfigurationScope(Scope.AM)
+ @ConfigurationProperty
+ public static final String TEZ_AM_REGISTRY_NAMESPACE = TEZ_AM_PREFIX + "registry.namespace";
+ public static final String TEZ_AM_REGISTRY_NAMESPACE_DEFAULT = "/tez_am/server";
+
+ /**
+ * Boolean value. Whether to enable compute groups, see further details in ZkConfig.
+ */
+ @ConfigurationScope(Scope.AM)
+ @ConfigurationProperty
+ public static final String TEZ_AM_REGISTRY_ENABLE_COMPUTE_GROUPS = TEZ_AM_PREFIX + "registry.enable.compute.groups";
+ public static final boolean TEZ_AM_REGISTRY_ENABLE_COMPUTE_GROUPS_DEFAULT = false;
+
+
+ /**
+ * Integer value. Initial backoff sleep duration (in milliseconds) for Curator retries.
+ * It's used when creating a CuratorFramework for the ZooKeeper registry.
+ */
+ @ConfigurationScope(Scope.AM)
+ @ConfigurationProperty
+ public static final String TEZ_AM_CURATOR_BACKOFF_SLEEP = TEZ_AM_PREFIX + "curator.backoff.sleep";
+ public static final int TEZ_AM_CURATOR_BACKOFF_SLEEP_DEFAULT = 1000;
+
+ /**
+ * Integer value. Maximum number of retries for Curator operations.
+ * It's used when creating a CuratorFramework for the ZooKeeper registry.
+ */
+ @ConfigurationScope(Scope.AM)
+ @ConfigurationProperty
+ public static final String TEZ_AM_CURATOR_MAX_RETRIES = TEZ_AM_PREFIX + "curator.max.retries";
+ public static final int TEZ_AM_CURATOR_MAX_RETRIES_DEFAULT = 3;
+
+ /**
+ * Integer value. Session timeout (in milliseconds) for Curator framework.
+ * It's used when creating a CuratorFramework for the ZooKeeper registry.
+ */
+ @ConfigurationScope(Scope.AM)
+ @ConfigurationProperty
+ public static final String TEZ_AM_CURATOR_SESSION_TIMEOUT = TEZ_AM_PREFIX + "curator.session.timeout";
+ public static final int TEZ_AM_CURATOR_SESSION_TIMEOUT_DEFAULT = 150000;
Review Comment:
I think we should support time suffix like s ms and all for all these timeout configs
##########
tez-api/src/main/java/org/apache/tez/client/registry/zookeeper/ZkConfig.java:
##########
@@ -0,0 +1,128 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.client.registry.zookeeper;
+
+import org.apache.curator.RetryPolicy;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.CuratorFrameworkFactory;
+import org.apache.curator.retry.ExponentialBackoffRetry;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.tez.dag.api.TezConfiguration;
+
+import com.google.common.base.Preconditions;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class ZkConfig {
+ private static final Logger LOG = LoggerFactory.getLogger(ZkConfig.class);
+
+ // if namespace defined in config is 'foo' and COMPUTE_GROUP_NAME env is 'bar' then the zkpaths will be of format
+ // /tez-external-sessions/foo/bar
+ private final static String ZK_NAMESPACE_PREFIX = "/tez-external-sessions";
+ public final static String COMPUTE_GROUP_NAME_ENV = "COMPUTE_GROUP_NAME";
+ public final static String DEFAULT_COMPUTE_GROUP_NAME = "default-compute";
+
+ private final String zkQuorum;
+ private final String zkNamespace;
+ private final int curatorBackoffSleep;
+ private final int curatorMaxRetries;
+ private final int sessionTimeoutMs;
+ private final int connectionTimeoutMs;
+
+ public ZkConfig(Configuration conf) {
+ zkQuorum = conf.get(TezConfiguration.TEZ_AM_ZOOKEEPER_QUORUM);
+ Preconditions.checkNotNull(zkQuorum);
+
+ String fullZkNamespace = ZK_NAMESPACE_PREFIX;
+
+ String namespace = conf.get(TezConfiguration.TEZ_AM_REGISTRY_NAMESPACE,
+ TezConfiguration.TEZ_AM_REGISTRY_NAMESPACE_DEFAULT);
+ Preconditions.checkNotNull(namespace);
+ if (namespace.startsWith(Path.SEPARATOR)) {
+ fullZkNamespace += namespace;
+ } else {
+ fullZkNamespace = fullZkNamespace + Path.SEPARATOR + namespace;
+ }
+
+ boolean enableComputeGroups = conf.getBoolean(TezConfiguration.TEZ_AM_REGISTRY_ENABLE_COMPUTE_GROUPS,
+ TezConfiguration.TEZ_AM_REGISTRY_ENABLE_COMPUTE_GROUPS_DEFAULT);
+ if (enableComputeGroups) {
+ final String subNamespace = System.getenv(COMPUTE_GROUP_NAME_ENV);
+ if (subNamespace != null && !subNamespace.isEmpty()) {
+ if (subNamespace.startsWith(Path.SEPARATOR)) {
+ fullZkNamespace += subNamespace;
+ } else {
+ fullZkNamespace = fullZkNamespace + Path.SEPARATOR + subNamespace;
+ }
+ LOG.info("Compute groups enabled: subNamespace: {} fullZkNamespace: {}", subNamespace, fullZkNamespace);
+ }
+ } else {
+ LOG.info("Compute groups disabled: fullZkNamespace: {}", fullZkNamespace);
+ }
+ zkNamespace = fullZkNamespace;
Review Comment:
maybe we should log it somewhere, if not already
```
LOG.info("Using ZK namespace: {}", fullZkNamespace);
```
##########
tez-dag/src/main/java/org/apache/tez/dag/api/client/registry/zookeeper/ZkAMRegistry.java:
##########
@@ -0,0 +1,175 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.api.client.registry.zookeeper;
+
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.curator.RetryLoop;
+import org.apache.curator.RetryPolicy;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.registry.client.binding.RegistryUtils;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.tez.client.registry.AMRecord;
+import org.apache.tez.client.registry.AMRegistry;
+import org.apache.tez.client.registry.zookeeper.ZkConfig;
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.data.Stat;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Curator/Zookeeper impl of AMRegistry (for internal use only)
+ * Clients should use org.apache.tez.dag.api.client.registry.zookeeper.ZkAMRegistryClient instead.
+ */
+@InterfaceAudience.Private
+public class ZkAMRegistry extends AMRegistry {
+
+ private static final Logger LOG = LoggerFactory.getLogger(ZkAMRegistry.class);
+
+ private final List<AMRecord> amRecords = new ArrayList<>();
Review Comment:
this isn't thread safe, but can be mutated by both add & remove, are we sure no concurrent threads will bother them
##########
tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java:
##########
@@ -1612,19 +1613,14 @@ DAGState initializeDAG() {
addVertex(v);
}
- // check task resources, only check it in non-local mode
if (!appContext.isLocal()) {
- for (Vertex v : vertexMap.values()) {
- // TODO TEZ-2003 (post) TEZ-2624 Ideally, this should be per source.
- if (v.getTaskResource().compareTo(appContext.getClusterInfo().getMaxContainerCapability()) > 0) {
- String msg = "Vertex's TaskResource is beyond the cluster container capability," +
- "Vertex=" + v.getLogIdentifier() +", Requested TaskResource=" + v.getTaskResource()
- + ", Cluster MaxContainerCapability=" + appContext.getClusterInfo().getMaxContainerCapability();
- LOG.error(msg);
- addDiagnostic(msg);
- finished(DAGState.FAILED);
- return DAGState.FAILED;
- }
+ try {
+ appContext.getAmExtensions().checkTaskResources(vertexMap, appContext.getClusterInfo());
+ } catch (TaskResourceException e) {
Review Comment:
if it throws some RuntimeException, then we don't set the state failed?
##########
tez-api/src/test/java/org/apache/tez/client/registry/zookeeper/TestZkFrameworkClient.java:
##########
@@ -0,0 +1,243 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.client.registry.zookeeper;
+
+import static org.junit.Assert.*;
+
+import java.io.File;
+import java.nio.charset.StandardCharsets;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.test.TestingServer;
+import org.apache.hadoop.registry.client.binding.RegistryUtils;
+import org.apache.hadoop.registry.client.types.ServiceRecord;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.client.api.YarnClientApplication;
+import org.apache.tez.client.registry.AMRecord;
+import org.apache.tez.dag.api.TezConfiguration;
+import org.apache.zookeeper.CreateMode;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Unit tests for {@link ZkFrameworkClient}.
+ * <p>
+ * This test class validates the ZooKeeper-based framework client that discovers
+ * and communicates with Application Masters through ZooKeeper registry.
+ * </p>
+ */
+public class TestZkFrameworkClient {
+ private static final Logger LOG = LoggerFactory.getLogger(TestZkFrameworkClient.class);
+ private static final File TEST_DIR = new File(System.getProperty("test.build.data", "target"),
+ TestZkFrameworkClient.class.getName()).getAbsoluteFile();
+
+ private TestingServer zkServer;
+ private ZkFrameworkClient zkFrameworkClient;
+ private CuratorFramework curatorClient;
+
+ @Before
+ public void setup() throws Exception {
+ zkServer = new TestingServer(true);
+ LOG.info("Started ZooKeeper test server on port: {}", zkServer.getPort());
+ }
+
+ @After
+ public void teardown() throws Exception {
+ if (zkFrameworkClient != null) {
+ zkFrameworkClient.close();
+ }
+ IOUtils.closeQuietly(curatorClient);
+ IOUtils.closeQuietly(zkServer);
+ }
+
+ /**
+ * Tests initialization and lifecycle methods of ZkFrameworkClient.
+ */
+ @Test
+ public void testInitAndLifecycle() throws Exception {
+ TezConfiguration tezConf = createTezConf();
+
+ zkFrameworkClient = new ZkFrameworkClient();
+ zkFrameworkClient.init(tezConf);
+
+ assertTrue("Client should be running after init", zkFrameworkClient.isRunning());
+
+ zkFrameworkClient.start();
+ assertTrue("Client should be running after start", zkFrameworkClient.isRunning());
+
+ zkFrameworkClient.stop();
+ assertFalse("Client should not be running after stop", zkFrameworkClient.isRunning());
+ }
+
+ /**
+ * Tests retrieving application report when AM is registered in ZooKeeper.
+ */
+ @Test
+ public void testGetApplicationReportWithRegisteredAM() throws Exception {
+ TezConfiguration tezConf = createTezConf();
+
+ // Register a mock AM in ZooKeeper
+ ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(), 1);
+ String testHostName = "test-host";
+ String testHostIp = "127.0.0.1";
+ int testPort = 12345;
+ registerMockAM(tezConf, appId, testHostName, testHostIp, testPort);
+
+ zkFrameworkClient = new ZkFrameworkClient();
+ zkFrameworkClient.init(tezConf);
+ zkFrameworkClient.start();
+
+ // Give time for ZK registry to initialize
+ Thread.sleep(500);
Review Comment:
these sleeps usually makes tests flaky, is there something we can assert with LambdaTestUtils.waitFor
##########
tez-dag/src/main/java/org/apache/tez/dag/api/client/registry/zookeeper/ZkAMRegistry.java:
##########
@@ -0,0 +1,175 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.api.client.registry.zookeeper;
+
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.curator.RetryLoop;
+import org.apache.curator.RetryPolicy;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.registry.client.binding.RegistryUtils;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.tez.client.registry.AMRecord;
+import org.apache.tez.client.registry.AMRegistry;
+import org.apache.tez.client.registry.zookeeper.ZkConfig;
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.data.Stat;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Curator/Zookeeper impl of AMRegistry (for internal use only)
+ * Clients should use org.apache.tez.dag.api.client.registry.zookeeper.ZkAMRegistryClient instead.
+ */
+@InterfaceAudience.Private
+public class ZkAMRegistry extends AMRegistry {
+
+ private static final Logger LOG = LoggerFactory.getLogger(ZkAMRegistry.class);
+
+ private final List<AMRecord> amRecords = new ArrayList<>();
+ private final String externalId;
+
+ private CuratorFramework client = null;
+ private String namespace = null;
+ private ZkConfig zkConfig = null;
+ private boolean started = false;
+
+ public ZkAMRegistry(String externalId) {
+ super("ZkAMRegistry");
+ this.externalId = externalId;
+ }
+
+ @Override
+ public void serviceInit(Configuration conf) {
+ if (zkConfig == null) {
+ zkConfig = new ZkConfig(conf);
+ this.client = zkConfig.createCuratorFramework();
+ this.namespace = zkConfig.getZkNamespace();
+ LOG.info("AMRegistryZkImpl initialized");
+ }
+ }
+
+ @Override
+ public void serviceStart() throws Exception {
+ if (!started) {
+ client.start();
+ started = true;
+ LOG.info("AMRegistryZkImpl started");
+ }
+ }
+
+ //Deletes from Zookeeper AMRecords that were added by this instance
+ @Override
+ public void serviceStop() throws Exception {
+ List<AMRecord> records = new ArrayList<>(amRecords);
+ for (AMRecord amRecord : records) {
+ remove(amRecord);
+ }
+ client.close();
+ LOG.info("AMRegistryZkImpl shutdown");
+ }
+
+ //Serialize AMRecord to ServiceRecord and deliver the JSON bytes to
+ //zkNode at the path: <TEZ_AM_REGISTRY_NAMESPACE>/<appId>
+ @Override
+ public void add(AMRecord server) throws Exception {
+ RegistryUtils.ServiceRecordMarshal marshal = new RegistryUtils.ServiceRecordMarshal();
+ String json = marshal.toJson(server.toServiceRecord());
+ try {
+ final String path = namespace + "/" + server.getApplicationId().toString();
+ client.setData().forPath(path, json.getBytes(StandardCharsets.UTF_8));
+ LOG.info("Added AMRecord to zkpath {}", path);
+ } catch (KeeperException.NoNodeException nne) {
+ client.create().creatingParentContainersIfNeeded().withMode(CreateMode.EPHEMERAL)
+ .forPath(namespace + "/" + server.getApplicationId().toString(), json.getBytes(StandardCharsets.UTF_8));
+ }
+ amRecords.add(server);
+ }
+
+ @Override
+ public void remove(AMRecord server) throws Exception {
+ amRecords.remove(server);
+ final String path = namespace + "/" + server.getApplicationId().toString();
+ client.delete().forPath(path);
+ LOG.info("Deleted AMRecord from zkpath {}", path);
+ }
+
+ @Override
+ public ApplicationId generateNewId() throws Exception {
+ createNamespaceIfNotExists();
+ long namespaceCreationTime = getNamespaceCreationTime();
+
+ boolean success = false;
+ long startTime = System.currentTimeMillis();
+ RetryPolicy retryPolicy = zkConfig.getRetryPolicy();
+ int tryId = 0;
+ for (int i = 0; (i < zkConfig.getCuratorMaxRetries()) && !success; i++) {
+ List<String> children = client.getChildren().forPath(namespace);
+ if (children != null && !children.isEmpty()) {
+ children.sort(Collections.reverseOrder());
+ String last = children.getFirst();
+ ApplicationId lastAppId = ApplicationId.fromString(last);
+ tryId = lastAppId.getId() + 1;
Review Comment:
Sorting strings lexicographically, not numerically
"100" < "99" lexicographically.
Parallel clients reading the same children list → same new tryId
Retry logic increments tryId, but doesn't re-check children.
This is not safe for ID allocation.
##########
tez-dag/src/main/java/org/apache/tez/dag/api/client/registry/zookeeper/ZkAMRegistry.java:
##########
@@ -0,0 +1,175 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.api.client.registry.zookeeper;
+
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.curator.RetryLoop;
+import org.apache.curator.RetryPolicy;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.registry.client.binding.RegistryUtils;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.tez.client.registry.AMRecord;
+import org.apache.tez.client.registry.AMRegistry;
+import org.apache.tez.client.registry.zookeeper.ZkConfig;
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.data.Stat;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Curator/Zookeeper impl of AMRegistry (for internal use only)
+ * Clients should use org.apache.tez.dag.api.client.registry.zookeeper.ZkAMRegistryClient instead.
+ */
+@InterfaceAudience.Private
+public class ZkAMRegistry extends AMRegistry {
+
+ private static final Logger LOG = LoggerFactory.getLogger(ZkAMRegistry.class);
+
+ private final List<AMRecord> amRecords = new ArrayList<>();
+ private final String externalId;
+
+ private CuratorFramework client = null;
+ private String namespace = null;
+ private ZkConfig zkConfig = null;
+ private boolean started = false;
+
+ public ZkAMRegistry(String externalId) {
+ super("ZkAMRegistry");
+ this.externalId = externalId;
+ }
+
+ @Override
+ public void serviceInit(Configuration conf) {
+ if (zkConfig == null) {
+ zkConfig = new ZkConfig(conf);
+ this.client = zkConfig.createCuratorFramework();
+ this.namespace = zkConfig.getZkNamespace();
+ LOG.info("AMRegistryZkImpl initialized");
+ }
+ }
+
+ @Override
+ public void serviceStart() throws Exception {
+ if (!started) {
+ client.start();
+ started = true;
+ LOG.info("AMRegistryZkImpl started");
+ }
+ }
+
+ //Deletes from Zookeeper AMRecords that were added by this instance
+ @Override
+ public void serviceStop() throws Exception {
+ List<AMRecord> records = new ArrayList<>(amRecords);
+ for (AMRecord amRecord : records) {
+ remove(amRecord);
+ }
+ client.close();
+ LOG.info("AMRegistryZkImpl shutdown");
+ }
+
+ //Serialize AMRecord to ServiceRecord and deliver the JSON bytes to
+ //zkNode at the path: <TEZ_AM_REGISTRY_NAMESPACE>/<appId>
+ @Override
+ public void add(AMRecord server) throws Exception {
+ RegistryUtils.ServiceRecordMarshal marshal = new RegistryUtils.ServiceRecordMarshal();
+ String json = marshal.toJson(server.toServiceRecord());
+ try {
+ final String path = namespace + "/" + server.getApplicationId().toString();
+ client.setData().forPath(path, json.getBytes(StandardCharsets.UTF_8));
+ LOG.info("Added AMRecord to zkpath {}", path);
+ } catch (KeeperException.NoNodeException nne) {
+ client.create().creatingParentContainersIfNeeded().withMode(CreateMode.EPHEMERAL)
+ .forPath(namespace + "/" + server.getApplicationId().toString(), json.getBytes(StandardCharsets.UTF_8));
+ }
+ amRecords.add(server);
+ }
+
+ @Override
+ public void remove(AMRecord server) throws Exception {
+ amRecords.remove(server);
+ final String path = namespace + "/" + server.getApplicationId().toString();
+ client.delete().forPath(path);
+ LOG.info("Deleted AMRecord from zkpath {}", path);
+ }
+
+ @Override
+ public ApplicationId generateNewId() throws Exception {
+ createNamespaceIfNotExists();
+ long namespaceCreationTime = getNamespaceCreationTime();
+
+ boolean success = false;
+ long startTime = System.currentTimeMillis();
+ RetryPolicy retryPolicy = zkConfig.getRetryPolicy();
+ int tryId = 0;
+ for (int i = 0; (i < zkConfig.getCuratorMaxRetries()) && !success; i++) {
+ List<String> children = client.getChildren().forPath(namespace);
+ if (children != null && !children.isEmpty()) {
+ children.sort(Collections.reverseOrder());
+ String last = children.getFirst();
+ ApplicationId lastAppId = ApplicationId.fromString(last);
+ tryId = lastAppId.getId() + 1;
+ }
+ ApplicationId tryAppId = ApplicationId.newInstance(namespaceCreationTime, tryId);
+ try {
+ client
+ .create()
+ .withMode(CreateMode.EPHEMERAL)
Review Comment:
should it be ``create().withMode(PERSISTENT_SEQUENTIAL)``?
##########
tez-dag/src/main/java/org/apache/tez/frameworkplugins/yarn/YarnServerFrameworkService.java:
##########
@@ -0,0 +1,120 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tez.frameworkplugins.yarn;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.util.Map;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.util.ConverterUtils;
+import org.apache.tez.client.registry.AMRegistry;
+import org.apache.tez.common.TezUtilsInternal;
+import org.apache.tez.common.security.JobTokenIdentifier;
+import org.apache.tez.common.security.JobTokenSecretManager;
+import org.apache.tez.common.security.TokenCache;
+import org.apache.tez.dag.api.TezConstants;
+import org.apache.tez.dag.api.records.DAGProtos;
+import org.apache.tez.dag.app.ClusterInfo;
+import org.apache.tez.dag.app.dag.Vertex;
+import org.apache.tez.frameworkplugins.AMExtensions;
+import org.apache.tez.frameworkplugins.ServerFrameworkService;
+import org.apache.tez.frameworkplugins.TaskResourceException;
+
+/**
+ * YARN-based server framework service implementation.
+ * Provides default YARN framework server functionality with default implementations
+ * for all AmExtensions methods.
+ */
+public class YarnServerFrameworkService implements ServerFrameworkService {
+
+ private final YarnAMExtensions amExtensions = new YarnAMExtensions();
+
+ @Override
+ public AMRegistry getAMRegistry(Configuration conf) {
+ // YARN mode doesn't require a custom AM registry
+ return null;
+ }
+
+ @Override
+ public AMExtensions getAMExtensions() {
+ return amExtensions;
+ }
+
+ /**
+ * Default YARN implementation of AmExtensions.
+ * Provides sensible defaults for all methods.
+ */
+ public static class YarnAMExtensions implements AMExtensions {
+
+ @Override
+ public DAGProtos.ConfigurationProto loadConfigurationProto() throws IOException {
+ return TezUtilsInternal
+ .readUserSpecifiedTezConfiguration(System.getenv(ApplicationConstants.Environment.PWD.name()));
+ }
+
+ @Override
+ public ContainerId allocateContainerId(Configuration conf) {
+ String containerIdStr = System.getenv(ApplicationConstants.Environment.CONTAINER_ID.name());
+ return ConverterUtils.toContainerId(containerIdStr);
+ }
+
+ @Override
+ public void checkTaskResources(Map<String, Vertex> vertices, ClusterInfo clusterInfo) throws TaskResourceException {
+ Resource maxContainerCapability = clusterInfo.getMaxContainerCapability();
+ for (Vertex v : vertices.values()) {
+ // TODO TEZ-2003 (post) TEZ-2624 Ideally, this should be per source.
+ if (v.getTaskResource().compareTo(maxContainerCapability) > 0) {
+ String msg = "Vertex's TaskResource is beyond the cluster container capability," +
+ "Vertex=" + v.getLogIdentifier() +", Requested TaskResource=" + v.getTaskResource()
+ + ", Cluster MaxContainerCapability=" + maxContainerCapability;
+ throw new TaskResourceException(msg);
+ }
+ }
+ }
+
+ @Override
+ public Token<JobTokenIdentifier> getSessionToken(
+ ApplicationAttemptId appAttemptID,
+ JobTokenSecretManager jobTokenSecretManager,
+ Credentials amCredentials) {
+ return TokenCache.getSessionToken(amCredentials);
+ }
+
+ @Override
+ public DAGProtos.PlanLocalResourcesProto getAdditionalSessionResources(String workingDirectory) throws IOException {
+ FileInputStream sessionResourcesStream = null;
+ try {
Review Comment:
can we use try-with-resources
```
try (FileInputStream sessionResourcesStream = new FileInputStream(
new File(workingDirectory, TezConstants.TEZ_AM_LOCAL_RESOURCES_PB_FILE_NAME))) {
return DAGProtos.PlanLocalResourcesProto.parseDelimitedFrom(sessionResourcesStream);
}
```
##########
tez-dag/src/main/java/org/apache/tez/dag/api/client/registry/zookeeper/ZkAMRegistry.java:
##########
@@ -0,0 +1,175 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.api.client.registry.zookeeper;
+
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.curator.RetryLoop;
+import org.apache.curator.RetryPolicy;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.registry.client.binding.RegistryUtils;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.tez.client.registry.AMRecord;
+import org.apache.tez.client.registry.AMRegistry;
+import org.apache.tez.client.registry.zookeeper.ZkConfig;
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.data.Stat;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Curator/Zookeeper impl of AMRegistry (for internal use only)
+ * Clients should use org.apache.tez.dag.api.client.registry.zookeeper.ZkAMRegistryClient instead.
+ */
+@InterfaceAudience.Private
+public class ZkAMRegistry extends AMRegistry {
+
+ private static final Logger LOG = LoggerFactory.getLogger(ZkAMRegistry.class);
+
+ private final List<AMRecord> amRecords = new ArrayList<>();
+ private final String externalId;
+
+ private CuratorFramework client = null;
+ private String namespace = null;
+ private ZkConfig zkConfig = null;
+ private boolean started = false;
+
+ public ZkAMRegistry(String externalId) {
+ super("ZkAMRegistry");
+ this.externalId = externalId;
+ }
+
+ @Override
+ public void serviceInit(Configuration conf) {
+ if (zkConfig == null) {
+ zkConfig = new ZkConfig(conf);
+ this.client = zkConfig.createCuratorFramework();
+ this.namespace = zkConfig.getZkNamespace();
+ LOG.info("AMRegistryZkImpl initialized");
+ }
+ }
+
+ @Override
+ public void serviceStart() throws Exception {
+ if (!started) {
+ client.start();
+ started = true;
+ LOG.info("AMRegistryZkImpl started");
+ }
+ }
+
+ //Deletes from Zookeeper AMRecords that were added by this instance
+ @Override
+ public void serviceStop() throws Exception {
+ List<AMRecord> records = new ArrayList<>(amRecords);
+ for (AMRecord amRecord : records) {
+ remove(amRecord);
+ }
+ client.close();
+ LOG.info("AMRegistryZkImpl shutdown");
+ }
+
+ //Serialize AMRecord to ServiceRecord and deliver the JSON bytes to
+ //zkNode at the path: <TEZ_AM_REGISTRY_NAMESPACE>/<appId>
+ @Override
+ public void add(AMRecord server) throws Exception {
+ RegistryUtils.ServiceRecordMarshal marshal = new RegistryUtils.ServiceRecordMarshal();
+ String json = marshal.toJson(server.toServiceRecord());
+ try {
+ final String path = namespace + "/" + server.getApplicationId().toString();
+ client.setData().forPath(path, json.getBytes(StandardCharsets.UTF_8));
+ LOG.info("Added AMRecord to zkpath {}", path);
+ } catch (KeeperException.NoNodeException nne) {
Review Comment:
only (NoNode → create) is retried, but for CONNECTIONLOSS, SESSIONEXPIRED?
##########
tez-dag/src/main/java/org/apache/tez/frameworkplugins/zookeeper/ZkStandaloneAMExtensions.java:
##########
@@ -0,0 +1,90 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tez.frameworkplugins.zookeeper;
+
+import java.io.IOException;
+import java.util.Map;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.tez.client.registry.AMRegistry;
+import org.apache.tez.common.TezUtilsInternal;
+import org.apache.tez.common.security.JobTokenIdentifier;
+import org.apache.tez.common.security.JobTokenSecretManager;
+import org.apache.tez.common.security.TokenCache;
+import org.apache.tez.dag.api.records.DAGProtos;
+import org.apache.tez.dag.app.ClusterInfo;
+import org.apache.tez.dag.app.dag.Vertex;
+import org.apache.tez.frameworkplugins.AMExtensions;
+import org.apache.tez.frameworkplugins.ServerFrameworkService;
+
+public class ZkStandaloneAMExtensions implements AMExtensions {
+
+ private final ServerFrameworkService frameworkService;
+
+ public ZkStandaloneAMExtensions(ServerFrameworkService frameworkService) {
+ this.frameworkService = frameworkService;
+ }
+
+ @Override
+ public ContainerId allocateContainerId(Configuration conf) {
+ try {
+ AMRegistry amRegistry = frameworkService.getAMRegistry(conf);
+ if (amRegistry != null) {
+ ApplicationId appId = amRegistry.generateNewId();
+ // attemptId is set to 1 only then APP_LAUNCHED event gets triggered
+ ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.newInstance(appId, 1);
+ return ContainerId.newContainerId(applicationAttemptId, 0);
+ } else {
+ throw new RuntimeException("AMRegistry is required for ZkStandaloneAmExtensions");
Review Comment:
maybe
throw new IllegalStateException("AMRegistry must not be null for standalone AM mode");
##########
tez-dag/src/main/java/org/apache/tez/frameworkplugins/yarn/YarnServerFrameworkService.java:
##########
@@ -0,0 +1,120 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tez.frameworkplugins.yarn;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.util.Map;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.util.ConverterUtils;
+import org.apache.tez.client.registry.AMRegistry;
+import org.apache.tez.common.TezUtilsInternal;
+import org.apache.tez.common.security.JobTokenIdentifier;
+import org.apache.tez.common.security.JobTokenSecretManager;
+import org.apache.tez.common.security.TokenCache;
+import org.apache.tez.dag.api.TezConstants;
+import org.apache.tez.dag.api.records.DAGProtos;
+import org.apache.tez.dag.app.ClusterInfo;
+import org.apache.tez.dag.app.dag.Vertex;
+import org.apache.tez.frameworkplugins.AMExtensions;
+import org.apache.tez.frameworkplugins.ServerFrameworkService;
+import org.apache.tez.frameworkplugins.TaskResourceException;
+
+/**
+ * YARN-based server framework service implementation.
+ * Provides default YARN framework server functionality with default implementations
+ * for all AmExtensions methods.
+ */
+public class YarnServerFrameworkService implements ServerFrameworkService {
+
+ private final YarnAMExtensions amExtensions = new YarnAMExtensions();
+
+ @Override
+ public AMRegistry getAMRegistry(Configuration conf) {
+ // YARN mode doesn't require a custom AM registry
+ return null;
+ }
+
+ @Override
+ public AMExtensions getAMExtensions() {
+ return amExtensions;
+ }
+
+ /**
+ * Default YARN implementation of AmExtensions.
+ * Provides sensible defaults for all methods.
+ */
+ public static class YarnAMExtensions implements AMExtensions {
+
+ @Override
+ public DAGProtos.ConfigurationProto loadConfigurationProto() throws IOException {
+ return TezUtilsInternal
+ .readUserSpecifiedTezConfiguration(System.getenv(ApplicationConstants.Environment.PWD.name()));
+ }
+
+ @Override
+ public ContainerId allocateContainerId(Configuration conf) {
+ String containerIdStr = System.getenv(ApplicationConstants.Environment.CONTAINER_ID.name());
+ return ConverterUtils.toContainerId(containerIdStr);
Review Comment:
this is deprecated, use
```
return ContainerId.fromString(containerIdStr);
```
##########
tez-dag/src/main/java/org/apache/tez/frameworkplugins/zookeeper/ZkStandaloneServerFrameworkService.java:
##########
@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tez.frameworkplugins.zookeeper;
+
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.tez.client.registry.AMRegistry;
+import org.apache.tez.dag.api.TezConstants;
+import org.apache.tez.dag.api.client.registry.zookeeper.ZkAMRegistry;
+import org.apache.tez.frameworkplugins.AMExtensions;
+import org.apache.tez.frameworkplugins.ServerFrameworkService;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class ZkStandaloneServerFrameworkService implements ServerFrameworkService {
+ private static final Logger LOG = LoggerFactory.getLogger(ZkStandaloneServerFrameworkService.class);
+ private ZkAMRegistry amRegistry;
+
+ @Override
+ public synchronized AMRegistry getAMRegistry(Configuration conf) {
+ if (amRegistry == null) {
+ try {
+ final String externalID = System.getenv(TezConstants.TEZ_AM_EXTERNAL_ID);
+ amRegistry = new ZkAMRegistry(externalID);
+ amRegistry.init(conf);
+ amRegistry.start();
+ LOG.info("Created Zookeeper based AM Registry with externalID: {}", externalID);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+ return amRegistry;
+ }
+
+ @Override
+ public AMExtensions getAMExtensions() {
+ return new ZkStandaloneAMExtensions(this);
Review Comment:
we are creating new instance on every invocation? should we create once & then send back the same if not null?
##########
tez-dag/src/main/java/org/apache/tez/frameworkplugins/zookeeper/ZkStandaloneServerFrameworkService.java:
##########
@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tez.frameworkplugins.zookeeper;
+
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.tez.client.registry.AMRegistry;
+import org.apache.tez.dag.api.TezConstants;
+import org.apache.tez.dag.api.client.registry.zookeeper.ZkAMRegistry;
+import org.apache.tez.frameworkplugins.AMExtensions;
+import org.apache.tez.frameworkplugins.ServerFrameworkService;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class ZkStandaloneServerFrameworkService implements ServerFrameworkService {
+ private static final Logger LOG = LoggerFactory.getLogger(ZkStandaloneServerFrameworkService.class);
+ private ZkAMRegistry amRegistry;
+
+ @Override
+ public synchronized AMRegistry getAMRegistry(Configuration conf) {
+ if (amRegistry == null) {
+ try {
+ final String externalID = System.getenv(TezConstants.TEZ_AM_EXTERNAL_ID);
Review Comment:
if it lands up being `null`?
##########
tez-dag/src/main/java/org/apache/tez/frameworkplugins/zookeeper/ZkStandaloneServerFrameworkService.java:
##########
@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tez.frameworkplugins.zookeeper;
+
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.tez.client.registry.AMRegistry;
+import org.apache.tez.dag.api.TezConstants;
+import org.apache.tez.dag.api.client.registry.zookeeper.ZkAMRegistry;
+import org.apache.tez.frameworkplugins.AMExtensions;
+import org.apache.tez.frameworkplugins.ServerFrameworkService;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class ZkStandaloneServerFrameworkService implements ServerFrameworkService {
+ private static final Logger LOG = LoggerFactory.getLogger(ZkStandaloneServerFrameworkService.class);
+ private ZkAMRegistry amRegistry;
+
+ @Override
+ public synchronized AMRegistry getAMRegistry(Configuration conf) {
+ if (amRegistry == null) {
+ try {
+ final String externalID = System.getenv(TezConstants.TEZ_AM_EXTERNAL_ID);
+ amRegistry = new ZkAMRegistry(externalID);
+ amRegistry.init(conf);
+ amRegistry.start();
Review Comment:
who does stop & close?
##########
tez-dag/src/test/java/org/apache/tez/client/registry/zookeeper/TestZkAMRegistryClient.java:
##########
@@ -0,0 +1,197 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.client.registry.zookeeper;
+
+import static org.junit.Assert.*;
Review Comment:
nit
should avoid *
##########
tez-dag/src/main/java/org/apache/tez/frameworkplugins/zookeeper/ZkStandaloneServerFrameworkService.java:
##########
@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tez.frameworkplugins.zookeeper;
+
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.tez.client.registry.AMRegistry;
+import org.apache.tez.dag.api.TezConstants;
+import org.apache.tez.dag.api.client.registry.zookeeper.ZkAMRegistry;
+import org.apache.tez.frameworkplugins.AMExtensions;
+import org.apache.tez.frameworkplugins.ServerFrameworkService;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class ZkStandaloneServerFrameworkService implements ServerFrameworkService {
+ private static final Logger LOG = LoggerFactory.getLogger(ZkStandaloneServerFrameworkService.class);
+ private ZkAMRegistry amRegistry;
Review Comment:
should this be volatile?
##########
tez-dag/src/main/java/org/apache/tez/frameworkplugins/zookeeper/ZkStandaloneAMExtensions.java:
##########
@@ -0,0 +1,90 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tez.frameworkplugins.zookeeper;
+
+import java.io.IOException;
+import java.util.Map;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.tez.client.registry.AMRegistry;
+import org.apache.tez.common.TezUtilsInternal;
+import org.apache.tez.common.security.JobTokenIdentifier;
+import org.apache.tez.common.security.JobTokenSecretManager;
+import org.apache.tez.common.security.TokenCache;
+import org.apache.tez.dag.api.records.DAGProtos;
+import org.apache.tez.dag.app.ClusterInfo;
+import org.apache.tez.dag.app.dag.Vertex;
+import org.apache.tez.frameworkplugins.AMExtensions;
+import org.apache.tez.frameworkplugins.ServerFrameworkService;
+
+public class ZkStandaloneAMExtensions implements AMExtensions {
+
+ private final ServerFrameworkService frameworkService;
+
+ public ZkStandaloneAMExtensions(ServerFrameworkService frameworkService) {
+ this.frameworkService = frameworkService;
+ }
+
+ @Override
+ public ContainerId allocateContainerId(Configuration conf) {
+ try {
+ AMRegistry amRegistry = frameworkService.getAMRegistry(conf);
+ if (amRegistry != null) {
+ ApplicationId appId = amRegistry.generateNewId();
+ // attemptId is set to 1 only then APP_LAUNCHED event gets triggered
+ ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.newInstance(appId, 1);
+ return ContainerId.newContainerId(applicationAttemptId, 0);
+ } else {
+ throw new RuntimeException("AMRegistry is required for ZkStandaloneAmExtensions");
+ }
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void checkTaskResources(Map<String, Vertex> vertices, ClusterInfo clusterInfo) {
+ // no-op: Zookeeper-based framework current enforce task resources at the moment
+ }
+
+ @Override
+ public DAGProtos.ConfigurationProto loadConfigurationProto() throws IOException {
+ return TezUtilsInternal.loadConfProtoFromText();
+ }
+
+ @Override
+ public Token<JobTokenIdentifier> getSessionToken(ApplicationAttemptId appAttemptID,
+ JobTokenSecretManager jobTokenSecretManager, Credentials amCredentials) {
+ JobTokenIdentifier identifier = new JobTokenIdentifier(new Text(appAttemptID.getApplicationId().toString()));
Review Comment:
can using only ApplicationId cause collisions across sessions?
Should you include attemptID also?
##########
tez-api/src/main/java/org/apache/tez/client/registry/AMRegistry.java:
##########
@@ -0,0 +1,104 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.client.registry;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.service.ServiceStateException;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Base class for {@code AMRegistry} implementations.
+ *
+ * <p>The specific implementation is configured via the
+ * {@code tez.am.registry.class} property.</p>
+ *
+ * <p>Implementations are expected to provide appropriate service lifecycle
+ * behavior, including:
+ * <ul>
+ * <li>{@code init}</li>
+ * <li>{@code serviceStart}</li>
+ * <li>{@code serviceStop}</li>
+ * </ul>
+ * </p>
+ *
+ * <p>{@code init} and {@code serviceStart} are invoked during
+ * {@code DAGAppMaster.serviceInit()}, while {@code serviceStop} is called
+ * when {@code DAGAppMaster} shuts down.</p>
+ */
+public abstract class AMRegistry extends AbstractService {
+
+ private static final Logger LOG = LoggerFactory.getLogger(AMRegistry.class);
+ private List<AMRecord> amRecords = new ArrayList<>();
Review Comment:
should be final & this isn't thread safe, we are using it in `add` and `serviceStop` are you sure we won't land up in concurrentmodifcation ex
##########
tez-api/src/main/java/org/apache/tez/client/registry/zookeeper/ZkAMRegistryClient.java:
##########
@@ -0,0 +1,205 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.client.registry.zookeeper;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
+
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.imps.CuratorFrameworkState;
+import org.apache.curator.framework.recipes.cache.ChildData;
+import org.apache.curator.framework.recipes.cache.TreeCache;
+import org.apache.curator.framework.recipes.cache.TreeCacheEvent;
+import org.apache.curator.framework.recipes.cache.TreeCacheListener;
+import org.apache.curator.shaded.com.google.common.base.Charsets;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.registry.client.binding.RegistryUtils;
+import org.apache.hadoop.registry.client.types.ServiceRecord;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.tez.client.registry.AMRecord;
+import org.apache.tez.client.registry.AMRegistryClient;
+import org.apache.tez.client.registry.AMRegistryClientListener;
+import org.apache.tez.dag.api.TezConfiguration;
+
+import com.fasterxml.jackson.core.JsonParseException;
+import com.google.common.base.Preconditions;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Curator/Zookeeper implementation of {@link AMRegistryClient}.
+ */
+@InterfaceAudience.Public
+public final class ZkAMRegistryClient extends AMRegistryClient {
+ private static final Logger LOG = LoggerFactory.getLogger(ZkAMRegistryClient.class);
+ private static final Map<String, ZkAMRegistryClient> INSTANCES = new HashMap<>();
+
+ private final Configuration conf;
+ //Cache of known AMs
+ private final ConcurrentHashMap<String, AMRecord> amRecordCache = new ConcurrentHashMap<>();
+ private CuratorFramework client;
+
+ private ZkAMRegistryClient(final Configuration conf) {
+ this.conf = conf;
+ }
+
+ public static synchronized ZkAMRegistryClient getClient(final Configuration conf) {
+ String namespace = conf.get(TezConfiguration.TEZ_AM_REGISTRY_NAMESPACE);
+ ZkAMRegistryClient registry = INSTANCES.get(namespace);
+ if (registry == null) {
+ registry = new ZkAMRegistryClient(conf);
+ INSTANCES.put(namespace, registry);
+ }
+ LOG.info("Returning tez AM registry ({}) for namespace '{}'", System.identityHashCode(registry), namespace);
+ return registry;
+ }
+
+ /**
+ * Deserializes a {@link ServiceRecord} from ZooKeeper data and converts it into an {@link AMRecord}
+ * for caching.
+ *
+ * @param childData the ZooKeeper node data containing a serialized {@link ServiceRecord}
+ * @return an {@link AMRecord} constructed from the deserialized {@link ServiceRecord}, or {@code null}
+ * if no data is present
+ * @throws IOException if the data cannot be deserialized into a {@link ServiceRecord}
+ */
+ public static AMRecord getAMRecord(final ChildData childData) throws IOException {
+ // not a leaf path. Only leaf path contains AMRecord
+ if (!childData.getPath().contains(ApplicationId.appIdStrPrefix)) {
+ return null;
+ }
+ byte[] data = childData.getData();
+ // only the path appeared, there is no data yet
+ if (data.length == 0) {
+ return null;
+ }
+ String value = new String(data, Charsets.UTF_8);
+ try {
+ RegistryUtils.ServiceRecordMarshal marshal = new RegistryUtils.ServiceRecordMarshal();
+ ServiceRecord serviceRecord = marshal.fromJson(value);
+ return new AMRecord(serviceRecord);
+ } catch (JsonParseException e) {
+ //Not a json AMRecord (SRV), could be some other data.
+ LOG.warn("Non-json data received while de-serializing AMRecord: {}. Ignoring...", value);
+ return null;
+ }
+ }
+
+ public void start() throws Exception {
+ ZkConfig zkConf = new ZkConfig(this.conf);
+ client = zkConf.createCuratorFramework();
+ final TreeCache cache = new TreeCache(client, zkConf.getZkNamespace());
+ client.start();
+ cache.start();
+ cache.getListenable().addListener(new ZkRegistryListener());
+ }
+
+ @Override
+ public AMRecord getRecord(String appId) {
+ if (amRecordCache.get(appId) == null) {
+ return null;
+ }
+ //Return a copy
+ return new AMRecord(amRecordCache.get(appId));
+ }
+
+ @Override
+ public List<AMRecord> getAllRecords() {
+ return amRecordCache.values().stream().map(AMRecord::new).collect(Collectors.toList());
+ }
+
+ @Override
+ public synchronized void addListener(AMRegistryClientListener listener) {
+ getListeners().add(listener);
+ }
+
+ @Override
+ public void close() {
+ client.close();
+ }
+
+ /**
+ * Callback listener for ZooKeeper events that updates the local cache
+ * when child nodes under the monitored path change.
+ */
+ private class ZkRegistryListener implements TreeCacheListener {
+
+ @Override
+ public void childEvent(final CuratorFramework clientParam, final TreeCacheEvent event) throws Exception {
+ Preconditions.checkArgument(clientParam != null && clientParam.getState() == CuratorFrameworkState.STARTED,
+ "Curator client is not started");
+
+ ChildData childData = event.getData();
+ switch (event.getType()) {
+ case NODE_ADDED:
+ if (isEmpty(childData)) {
+ LOG.info("AppId allocated: {}", childData.getPath());
+ } else {
+ AMRecord amRecord = getAMRecord(childData);
+ if (amRecord != null) {
+ LOG.info("AM registered with data: {}. Notifying {} listeners.", amRecord, getListeners().size());
+ amRecordCache.put(amRecord.getApplicationId().toString(), amRecord);
+ notifyOnAdded(amRecord);
+ }
+ }
+ break;
+ case NODE_UPDATED:
+ if (isEmpty(childData)) {
+ throw new RuntimeException("AM updated with empty data");
+ } else {
+ AMRecord amRecord = getAMRecord(childData);
+ if (amRecord != null) {
+ LOG.info("AM updated data: {}. Notifying {} listeners.", amRecord, getListeners().size());
+ amRecordCache.put(amRecord.getApplicationId().toString(), amRecord);
+ notifyOnAdded(amRecord);
Review Comment:
on `NODE_UPDATED` we call `notifyOnAdded`, it should be some other `notifyOnUpdate`
##########
tez-api/src/main/java/org/apache/tez/client/registry/zookeeper/ZkAMRegistryClient.java:
##########
@@ -0,0 +1,205 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.client.registry.zookeeper;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
+
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.imps.CuratorFrameworkState;
+import org.apache.curator.framework.recipes.cache.ChildData;
+import org.apache.curator.framework.recipes.cache.TreeCache;
+import org.apache.curator.framework.recipes.cache.TreeCacheEvent;
+import org.apache.curator.framework.recipes.cache.TreeCacheListener;
+import org.apache.curator.shaded.com.google.common.base.Charsets;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.registry.client.binding.RegistryUtils;
+import org.apache.hadoop.registry.client.types.ServiceRecord;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.tez.client.registry.AMRecord;
+import org.apache.tez.client.registry.AMRegistryClient;
+import org.apache.tez.client.registry.AMRegistryClientListener;
+import org.apache.tez.dag.api.TezConfiguration;
+
+import com.fasterxml.jackson.core.JsonParseException;
+import com.google.common.base.Preconditions;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Curator/Zookeeper implementation of {@link AMRegistryClient}.
+ */
+@InterfaceAudience.Public
+public final class ZkAMRegistryClient extends AMRegistryClient {
+ private static final Logger LOG = LoggerFactory.getLogger(ZkAMRegistryClient.class);
+ private static final Map<String, ZkAMRegistryClient> INSTANCES = new HashMap<>();
+
+ private final Configuration conf;
+ //Cache of known AMs
+ private final ConcurrentHashMap<String, AMRecord> amRecordCache = new ConcurrentHashMap<>();
+ private CuratorFramework client;
+
+ private ZkAMRegistryClient(final Configuration conf) {
+ this.conf = conf;
+ }
+
+ public static synchronized ZkAMRegistryClient getClient(final Configuration conf) {
+ String namespace = conf.get(TezConfiguration.TEZ_AM_REGISTRY_NAMESPACE);
+ ZkAMRegistryClient registry = INSTANCES.get(namespace);
+ if (registry == null) {
+ registry = new ZkAMRegistryClient(conf);
+ INSTANCES.put(namespace, registry);
+ }
+ LOG.info("Returning tez AM registry ({}) for namespace '{}'", System.identityHashCode(registry), namespace);
+ return registry;
+ }
+
+ /**
+ * Deserializes a {@link ServiceRecord} from ZooKeeper data and converts it into an {@link AMRecord}
+ * for caching.
+ *
+ * @param childData the ZooKeeper node data containing a serialized {@link ServiceRecord}
+ * @return an {@link AMRecord} constructed from the deserialized {@link ServiceRecord}, or {@code null}
+ * if no data is present
+ * @throws IOException if the data cannot be deserialized into a {@link ServiceRecord}
+ */
+ public static AMRecord getAMRecord(final ChildData childData) throws IOException {
+ // not a leaf path. Only leaf path contains AMRecord
+ if (!childData.getPath().contains(ApplicationId.appIdStrPrefix)) {
+ return null;
+ }
+ byte[] data = childData.getData();
+ // only the path appeared, there is no data yet
+ if (data.length == 0) {
+ return null;
+ }
+ String value = new String(data, Charsets.UTF_8);
Review Comment:
use ``StandardCharsets.UTF_8``
##########
tez-api/src/main/java/org/apache/tez/client/registry/zookeeper/ZkAMRegistryClient.java:
##########
@@ -0,0 +1,205 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.client.registry.zookeeper;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
+
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.imps.CuratorFrameworkState;
+import org.apache.curator.framework.recipes.cache.ChildData;
+import org.apache.curator.framework.recipes.cache.TreeCache;
+import org.apache.curator.framework.recipes.cache.TreeCacheEvent;
+import org.apache.curator.framework.recipes.cache.TreeCacheListener;
+import org.apache.curator.shaded.com.google.common.base.Charsets;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.registry.client.binding.RegistryUtils;
+import org.apache.hadoop.registry.client.types.ServiceRecord;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.tez.client.registry.AMRecord;
+import org.apache.tez.client.registry.AMRegistryClient;
+import org.apache.tez.client.registry.AMRegistryClientListener;
+import org.apache.tez.dag.api.TezConfiguration;
+
+import com.fasterxml.jackson.core.JsonParseException;
+import com.google.common.base.Preconditions;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Curator/Zookeeper implementation of {@link AMRegistryClient}.
+ */
+@InterfaceAudience.Public
+public final class ZkAMRegistryClient extends AMRegistryClient {
+ private static final Logger LOG = LoggerFactory.getLogger(ZkAMRegistryClient.class);
+ private static final Map<String, ZkAMRegistryClient> INSTANCES = new HashMap<>();
+
+ private final Configuration conf;
+ //Cache of known AMs
+ private final ConcurrentHashMap<String, AMRecord> amRecordCache = new ConcurrentHashMap<>();
+ private CuratorFramework client;
+
+ private ZkAMRegistryClient(final Configuration conf) {
+ this.conf = conf;
+ }
+
+ public static synchronized ZkAMRegistryClient getClient(final Configuration conf) {
+ String namespace = conf.get(TezConfiguration.TEZ_AM_REGISTRY_NAMESPACE);
+ ZkAMRegistryClient registry = INSTANCES.get(namespace);
+ if (registry == null) {
+ registry = new ZkAMRegistryClient(conf);
+ INSTANCES.put(namespace, registry);
+ }
+ LOG.info("Returning tez AM registry ({}) for namespace '{}'", System.identityHashCode(registry), namespace);
+ return registry;
+ }
+
+ /**
+ * Deserializes a {@link ServiceRecord} from ZooKeeper data and converts it into an {@link AMRecord}
+ * for caching.
+ *
+ * @param childData the ZooKeeper node data containing a serialized {@link ServiceRecord}
+ * @return an {@link AMRecord} constructed from the deserialized {@link ServiceRecord}, or {@code null}
+ * if no data is present
+ * @throws IOException if the data cannot be deserialized into a {@link ServiceRecord}
+ */
+ public static AMRecord getAMRecord(final ChildData childData) throws IOException {
+ // not a leaf path. Only leaf path contains AMRecord
+ if (!childData.getPath().contains(ApplicationId.appIdStrPrefix)) {
+ return null;
+ }
+ byte[] data = childData.getData();
+ // only the path appeared, there is no data yet
+ if (data.length == 0) {
+ return null;
+ }
+ String value = new String(data, Charsets.UTF_8);
+ try {
+ RegistryUtils.ServiceRecordMarshal marshal = new RegistryUtils.ServiceRecordMarshal();
+ ServiceRecord serviceRecord = marshal.fromJson(value);
+ return new AMRecord(serviceRecord);
+ } catch (JsonParseException e) {
+ //Not a json AMRecord (SRV), could be some other data.
+ LOG.warn("Non-json data received while de-serializing AMRecord: {}. Ignoring...", value);
+ return null;
+ }
+ }
+
+ public void start() throws Exception {
+ ZkConfig zkConf = new ZkConfig(this.conf);
+ client = zkConf.createCuratorFramework();
+ final TreeCache cache = new TreeCache(client, zkConf.getZkNamespace());
+ client.start();
+ cache.start();
+ cache.getListenable().addListener(new ZkRegistryListener());
+ }
+
+ @Override
+ public AMRecord getRecord(String appId) {
+ if (amRecordCache.get(appId) == null) {
+ return null;
+ }
+ //Return a copy
+ return new AMRecord(amRecordCache.get(appId));
+ }
Review Comment:
it is 2 times get here:
```
AMRecord rec = amRecordCache.get(appId);
return rec == null ? null : new AMRecord(rec);
```
##########
tez-api/src/main/java/org/apache/tez/client/registry/AMRegistryClient.java:
##########
@@ -0,0 +1,98 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.client.registry;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Client-side interface for discovering Application Master (AM) instances
+ * registered in the AM registry.
+ *
+ * <p>Implementations are responsible for locating AM endpoints and returning
+ * their metadata. This API is used by client components to discover running
+ * Tez AMs.</p>
+ *
+ * <p>Listeners may be registered to receive notifications when AM records
+ * appear or are removed.</p>
+ */
+public abstract class AMRegistryClient implements Closeable {
+
+ private final List<AMRegistryClientListener> listeners = new ArrayList<>();
+
+ /**
+ * Returns the current set of registered listeners.
+ *
+ * @return a mutable list of listeners
+ */
+ protected List<AMRegistryClientListener> getListeners() {
+ return listeners;
+ }
+
+ /**
+ * Lookup AM metadata for the given application ID.
+ *
+ * @param appId the application ID
+ * @return the AM record if found, otherwise {@code null}
+ * @throws IOException if the lookup fails
+ */
+ public abstract AMRecord getRecord(String appId) throws IOException;
Review Comment:
Why are you using ``String appId`` instead of ``ApplicationId appId``, and ever caller id having `ApplicationId` and calling toString() on it & passing, we should directly take ``ApplicationId``
##########
tez-api/src/main/java/org/apache/tez/client/registry/AMRegistryClient.java:
##########
@@ -0,0 +1,98 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.client.registry;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Client-side interface for discovering Application Master (AM) instances
+ * registered in the AM registry.
+ *
+ * <p>Implementations are responsible for locating AM endpoints and returning
+ * their metadata. This API is used by client components to discover running
+ * Tez AMs.</p>
+ *
+ * <p>Listeners may be registered to receive notifications when AM records
+ * appear or are removed.</p>
+ */
+public abstract class AMRegistryClient implements Closeable {
+
+ private final List<AMRegistryClientListener> listeners = new ArrayList<>();
+
+ /**
+ * Returns the current set of registered listeners.
+ *
+ * @return a mutable list of listeners
+ */
+ protected List<AMRegistryClientListener> getListeners() {
+ return listeners;
+ }
Review Comment:
Are we ok returning the original, mutable list. If someone want to add/remove, they should go via add/remove method.
If we want to keep this for read sake:
```
protected List<AMRegistryClientListener> getListeners() {
return Collections.unmodifiableList(listeners);
}
```
##########
tez-api/src/main/java/org/apache/tez/client/registry/zookeeper/ZkAMRegistryClient.java:
##########
@@ -0,0 +1,205 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.client.registry.zookeeper;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
+
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.imps.CuratorFrameworkState;
+import org.apache.curator.framework.recipes.cache.ChildData;
+import org.apache.curator.framework.recipes.cache.TreeCache;
+import org.apache.curator.framework.recipes.cache.TreeCacheEvent;
+import org.apache.curator.framework.recipes.cache.TreeCacheListener;
+import org.apache.curator.shaded.com.google.common.base.Charsets;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.registry.client.binding.RegistryUtils;
+import org.apache.hadoop.registry.client.types.ServiceRecord;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.tez.client.registry.AMRecord;
+import org.apache.tez.client.registry.AMRegistryClient;
+import org.apache.tez.client.registry.AMRegistryClientListener;
+import org.apache.tez.dag.api.TezConfiguration;
+
+import com.fasterxml.jackson.core.JsonParseException;
+import com.google.common.base.Preconditions;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Curator/Zookeeper implementation of {@link AMRegistryClient}.
+ */
+@InterfaceAudience.Public
+public final class ZkAMRegistryClient extends AMRegistryClient {
+ private static final Logger LOG = LoggerFactory.getLogger(ZkAMRegistryClient.class);
+ private static final Map<String, ZkAMRegistryClient> INSTANCES = new HashMap<>();
+
+ private final Configuration conf;
+ //Cache of known AMs
+ private final ConcurrentHashMap<String, AMRecord> amRecordCache = new ConcurrentHashMap<>();
+ private CuratorFramework client;
+
+ private ZkAMRegistryClient(final Configuration conf) {
+ this.conf = conf;
+ }
+
+ public static synchronized ZkAMRegistryClient getClient(final Configuration conf) {
+ String namespace = conf.get(TezConfiguration.TEZ_AM_REGISTRY_NAMESPACE);
+ ZkAMRegistryClient registry = INSTANCES.get(namespace);
+ if (registry == null) {
+ registry = new ZkAMRegistryClient(conf);
+ INSTANCES.put(namespace, registry);
+ }
+ LOG.info("Returning tez AM registry ({}) for namespace '{}'", System.identityHashCode(registry), namespace);
+ return registry;
+ }
+
+ /**
+ * Deserializes a {@link ServiceRecord} from ZooKeeper data and converts it into an {@link AMRecord}
+ * for caching.
+ *
+ * @param childData the ZooKeeper node data containing a serialized {@link ServiceRecord}
+ * @return an {@link AMRecord} constructed from the deserialized {@link ServiceRecord}, or {@code null}
+ * if no data is present
+ * @throws IOException if the data cannot be deserialized into a {@link ServiceRecord}
+ */
+ public static AMRecord getAMRecord(final ChildData childData) throws IOException {
+ // not a leaf path. Only leaf path contains AMRecord
+ if (!childData.getPath().contains(ApplicationId.appIdStrPrefix)) {
+ return null;
+ }
+ byte[] data = childData.getData();
+ // only the path appeared, there is no data yet
+ if (data.length == 0) {
+ return null;
+ }
+ String value = new String(data, Charsets.UTF_8);
+ try {
+ RegistryUtils.ServiceRecordMarshal marshal = new RegistryUtils.ServiceRecordMarshal();
+ ServiceRecord serviceRecord = marshal.fromJson(value);
+ return new AMRecord(serviceRecord);
+ } catch (JsonParseException e) {
+ //Not a json AMRecord (SRV), could be some other data.
+ LOG.warn("Non-json data received while de-serializing AMRecord: {}. Ignoring...", value);
+ return null;
+ }
+ }
+
+ public void start() throws Exception {
+ ZkConfig zkConf = new ZkConfig(this.conf);
+ client = zkConf.createCuratorFramework();
+ final TreeCache cache = new TreeCache(client, zkConf.getZkNamespace());
+ client.start();
+ cache.start();
+ cache.getListenable().addListener(new ZkRegistryListener());
+ }
+
+ @Override
+ public AMRecord getRecord(String appId) {
+ if (amRecordCache.get(appId) == null) {
+ return null;
+ }
+ //Return a copy
+ return new AMRecord(amRecordCache.get(appId));
+ }
+
+ @Override
+ public List<AMRecord> getAllRecords() {
+ return amRecordCache.values().stream().map(AMRecord::new).collect(Collectors.toList());
+ }
+
+ @Override
+ public synchronized void addListener(AMRegistryClientListener listener) {
+ getListeners().add(listener);
+ }
+
+ @Override
+ public void close() {
+ client.close();
+ }
+
+ /**
+ * Callback listener for ZooKeeper events that updates the local cache
+ * when child nodes under the monitored path change.
+ */
+ private class ZkRegistryListener implements TreeCacheListener {
+
+ @Override
+ public void childEvent(final CuratorFramework clientParam, final TreeCacheEvent event) throws Exception {
+ Preconditions.checkArgument(clientParam != null && clientParam.getState() == CuratorFrameworkState.STARTED,
+ "Curator client is not started");
+
+ ChildData childData = event.getData();
+ switch (event.getType()) {
+ case NODE_ADDED:
+ if (isEmpty(childData)) {
+ LOG.info("AppId allocated: {}", childData.getPath());
+ } else {
+ AMRecord amRecord = getAMRecord(childData);
+ if (amRecord != null) {
+ LOG.info("AM registered with data: {}. Notifying {} listeners.", amRecord, getListeners().size());
Review Comment:
addListener is syncronized, but getListeners().size() is being just used like that, not sure if it can troubles
##########
tez-api/src/main/java/org/apache/tez/client/registry/zookeeper/ZkAMRegistryClient.java:
##########
@@ -0,0 +1,205 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.client.registry.zookeeper;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
+
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.imps.CuratorFrameworkState;
+import org.apache.curator.framework.recipes.cache.ChildData;
+import org.apache.curator.framework.recipes.cache.TreeCache;
+import org.apache.curator.framework.recipes.cache.TreeCacheEvent;
+import org.apache.curator.framework.recipes.cache.TreeCacheListener;
+import org.apache.curator.shaded.com.google.common.base.Charsets;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.registry.client.binding.RegistryUtils;
+import org.apache.hadoop.registry.client.types.ServiceRecord;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.tez.client.registry.AMRecord;
+import org.apache.tez.client.registry.AMRegistryClient;
+import org.apache.tez.client.registry.AMRegistryClientListener;
+import org.apache.tez.dag.api.TezConfiguration;
+
+import com.fasterxml.jackson.core.JsonParseException;
+import com.google.common.base.Preconditions;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Curator/Zookeeper implementation of {@link AMRegistryClient}.
+ */
+@InterfaceAudience.Public
+public final class ZkAMRegistryClient extends AMRegistryClient {
+ private static final Logger LOG = LoggerFactory.getLogger(ZkAMRegistryClient.class);
+ private static final Map<String, ZkAMRegistryClient> INSTANCES = new HashMap<>();
+
+ private final Configuration conf;
+ //Cache of known AMs
+ private final ConcurrentHashMap<String, AMRecord> amRecordCache = new ConcurrentHashMap<>();
+ private CuratorFramework client;
+
+ private ZkAMRegistryClient(final Configuration conf) {
+ this.conf = conf;
+ }
+
+ public static synchronized ZkAMRegistryClient getClient(final Configuration conf) {
+ String namespace = conf.get(TezConfiguration.TEZ_AM_REGISTRY_NAMESPACE);
+ ZkAMRegistryClient registry = INSTANCES.get(namespace);
+ if (registry == null) {
+ registry = new ZkAMRegistryClient(conf);
+ INSTANCES.put(namespace, registry);
+ }
+ LOG.info("Returning tez AM registry ({}) for namespace '{}'", System.identityHashCode(registry), namespace);
+ return registry;
+ }
+
+ /**
+ * Deserializes a {@link ServiceRecord} from ZooKeeper data and converts it into an {@link AMRecord}
+ * for caching.
+ *
+ * @param childData the ZooKeeper node data containing a serialized {@link ServiceRecord}
+ * @return an {@link AMRecord} constructed from the deserialized {@link ServiceRecord}, or {@code null}
+ * if no data is present
+ * @throws IOException if the data cannot be deserialized into a {@link ServiceRecord}
+ */
+ public static AMRecord getAMRecord(final ChildData childData) throws IOException {
+ // not a leaf path. Only leaf path contains AMRecord
+ if (!childData.getPath().contains(ApplicationId.appIdStrPrefix)) {
+ return null;
+ }
+ byte[] data = childData.getData();
+ // only the path appeared, there is no data yet
+ if (data.length == 0) {
+ return null;
+ }
+ String value = new String(data, Charsets.UTF_8);
+ try {
+ RegistryUtils.ServiceRecordMarshal marshal = new RegistryUtils.ServiceRecordMarshal();
+ ServiceRecord serviceRecord = marshal.fromJson(value);
+ return new AMRecord(serviceRecord);
+ } catch (JsonParseException e) {
+ //Not a json AMRecord (SRV), could be some other data.
+ LOG.warn("Non-json data received while de-serializing AMRecord: {}. Ignoring...", value);
+ return null;
+ }
+ }
+
+ public void start() throws Exception {
+ ZkConfig zkConf = new ZkConfig(this.conf);
+ client = zkConf.createCuratorFramework();
+ final TreeCache cache = new TreeCache(client, zkConf.getZkNamespace());
+ client.start();
+ cache.start();
Review Comment:
where is this closed
##########
tez-api/src/main/java/org/apache/tez/client/registry/zookeeper/ZkFrameworkClient.java:
##########
@@ -0,0 +1,164 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.client.registry.zookeeper;
+
+import java.io.IOException;
+
+import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.client.api.YarnClientApplication;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.util.Records;
+import org.apache.tez.client.FrameworkClient;
+import org.apache.tez.client.registry.AMRecord;
+import org.apache.tez.dag.api.TezConfiguration;
+
+public class ZkFrameworkClient extends FrameworkClient {
+
+ private AMRecord amRecord;
+ private ZkAMRegistryClient amRegistryClient = null;
+ private volatile boolean isRunning = false;
+ private String amHost;
+ private int amPort;
+
+ @Override
+ public synchronized void init(TezConfiguration tezConf) {
+ if (this.amRegistryClient == null) {
+ try {
+ this.amRegistryClient = ZkAMRegistryClient.getClient(tezConf);
+ this.isRunning = true;
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ @Override
+ public void start() {
+ try {
+ this.amRegistryClient.start();
+ isRunning = true;
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void stop() {
+ isRunning = false;
+ close();
+ }
+
+ @Override
+ public void close() {
+ if (amRegistryClient != null) {
+ amRegistryClient.close();
+ }
+ }
+
+ /**
+ * Creates a dummy {@link YarnClientApplication} using a pre-existing {@link ApplicationId}
+ * rather than requesting a new one from the ResourceManager.
+ *
+ * <p><strong>Note:</strong> This is a <em>dummy, backward-compatibility implementation</em>.
+ * Instead of allocating a fresh application ID from the ResourceManager, this method
+ * reuses the {@code applicationId} already obtained via {@code getApplicationReport()}.
+ * This allows legacy code paths to continue operating without requiring actual
+ * creation of a new application.</p>
+ *
+ * <p>Hidden assumption here: this method assumes that
+ * {@code getApplicationReport()} has already been called before
+ * {@code createApplication()}, ensuring that {@code amRecord.getApplicationId()}
+ * is always available. This assumption holds in all supported usage patterns:
+ * the only code path where {@code createApplication()} might be called first is
+ * {@code TezClient.submitDAGApplication()}, but that path is never exercised in
+ * Zookeeper standalone mode because that mode assumes applications are already
+ * running. Therefore, the ordering guarantee is valid in practice.</p>
+ *
+ * <p>
+ * The method constructs a minimal {@link ApplicationSubmissionContext} and a
+ * synthetic {@link GetNewApplicationResponse}, both populated with the already
+ * known application ID. These objects are then wrapped into a
+ * {@link YarnClientApplication} instance and returned.
+ * </p>
+ *
+ * @return a {@link YarnClientApplication} backed by a submission context and
+ * a mocked {@link GetNewApplicationResponse}, both tied to the pre-existing
+ * application ID.
+ */
+ @Override
+ public YarnClientApplication createApplication() {
+ ApplicationSubmissionContext context = Records.newRecord(ApplicationSubmissionContext.class);
+ ApplicationId appId = amRecord.getApplicationId();
Review Comment:
if AMRecord is `null`, it can lead to NPE, are we sure it can't be null
##########
tez-dag/src/test/java/org/apache/tez/client/registry/zookeeper/TestZkAMRegistryClient.java:
##########
@@ -0,0 +1,197 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.client.registry.zookeeper;
+
+import static org.junit.Assert.*;
+
+import java.io.File;
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.curator.test.TestingServer;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.util.SystemClock;
+import org.apache.tez.client.registry.AMRecord;
+import org.apache.tez.client.registry.AMRegistryClientListener;
+import org.apache.tez.dag.api.TezConfiguration;
+import org.apache.tez.dag.app.DAGAppMaster;
+import org.apache.tez.dag.app.MockDAGAppMaster;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Unit tests for {@link ZkAMRegistryClient}.
+ * <p>
+ * This test class validates the ZooKeeper-based AM (Application Master) registry and discovery
+ * mechanism. It tests that when a DAGAppMaster is started with STANDALONE_ZOOKEEPER framework mode,
+ * it properly registers itself to ZooKeeper and can be discovered by a {@link ZkAMRegistryClient}.
+ * </p>
+ * <p>
+ * The tests use an embedded ZooKeeper {@link TestingServer} to avoid external dependencies
+ * and ensure test isolation.
+ * </p>
+ */
+public class TestZkAMRegistryClient {
+ private static final Logger LOG = LoggerFactory.getLogger(TestZkAMRegistryClient.class);
+ private static final File TEST_DIR = new File(System.getProperty("test.build.data", "target"),
+ TestZkAMRegistryClient.class.getName()).getAbsoluteFile();
+
+ /**
+ * Embedded ZooKeeper server for testing. Uses Apache Curator's {@link TestingServer}
+ * to provide an in-memory ZooKeeper instance.
+ */
+ private TestingServer zkServer;
+
+ /**
+ * ZooKeeper-based AM registry client used to discover and retrieve AM records.
+ */
+ private ZkAMRegistryClient registryClient;
+
+ /**
+ * Mock DAGAppMaster instance that registers itself to the ZooKeeper registry.
+ */
+ private DAGAppMaster dagAppMaster;
+
+ @Before
+ public void setup() throws Exception {
+ zkServer = new TestingServer();
+ zkServer.start();
+ LOG.info("Started ZooKeeper test server on port: {}", zkServer.getPort());
+ }
+
+ @After
+ public void teardown() throws Exception {
+ if (dagAppMaster != null) {
+ dagAppMaster.stop();
+ }
+ IOUtils.closeQuietly(registryClient);
+ IOUtils.closeQuietly(zkServer);
+ }
+
+ /**
+ * Tests the complete ZooKeeper-based AM registry and discovery flow.
+ * <p>
+ * This test validates the following workflow:
+ * </p>
+ * <ol>
+ * <li>Configure Tez with STANDALONE_ZOOKEEPER framework mode</li>
+ * <li>Create and start a {@link ZkAMRegistryClient} with an event listener</li>
+ * <li>Start a {@link MockDAGAppMaster} which registers itself to ZooKeeper</li>
+ * <li>Verify that the registry client's listener is notified of the AM registration</li>
+ * <li>Verify the AM record can be retrieved via {@link ZkAMRegistryClient#getRecord(String)}</li>
+ * <li>Verify the AM appears in the list from {@link ZkAMRegistryClient#getAllRecords()}</li>
+ * <li>Validate all expected fields (host, port, applicationId) are correctly set</li>
+ * </ol>
+ * <p>
+ * The test uses a {@link CountDownLatch} to synchronize between the AM registration
+ * event and the test assertions, ensuring the AM has fully registered before validation.
+ * </p>
+ *
+ * @throws Exception if any part of the test fails
+ */
+ @Test(timeout = 10000)
+ public void testZkAmRegistryDiscovery() throws Exception {
+ TezConfiguration tezConf = getTezConfForZkDiscovery();
+
+ ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(), 1);
+ ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(appId, 1);
+ ContainerId containerId = ContainerId.newContainerId(attemptId, 1);
+
+ CountDownLatch amRegisteredLatch = new CountDownLatch(1);
+ AtomicBoolean amDiscovered = new AtomicBoolean(false);
+
+ // Create and start the ZkAMRegistryClient
+ registryClient = ZkAMRegistryClient.getClient(tezConf);
+ registryClient.addListener(new AMRegistryClientListener() {
+ @Override
+ public void onAdd(AMRecord amRecord) {
+ LOG.info("AM added to registry: {}", amRecord);
+ if (amRecord.getApplicationId().equals(appId)) {
+ amDiscovered.set(true);
+ amRegisteredLatch.countDown();
+ }
+ }
+
+ @Override
+ public void onRemove(AMRecord amRecord) {
+ LOG.info("AM removed from registry: {}", amRecord);
+ }
+ });
+ registryClient.start();
+
+ String workingDir = TEST_DIR.toString();
+ String[] localDirs = new String[]{TEST_DIR.toString()};
+ String[] logDirs = new String[]{TEST_DIR + "/logs"};
+ String jobUserName = UserGroupInformation.getCurrentUser().getShortUserName();
+
+ dagAppMaster = new MockDAGAppMaster(attemptId, containerId, "localhost", 0, 0, SystemClock.getInstance(),
+ System.currentTimeMillis(), true, workingDir, localDirs, logDirs, new AtomicBoolean(true), false, false,
+ new Credentials(), jobUserName, 1, 1);
+
+ dagAppMaster.init(tezConf);
+ dagAppMaster.start();
+
+ // Wait for AM to be registered in ZooKeeper
+ boolean registered = amRegisteredLatch.await(30, TimeUnit.SECONDS);
+ assertTrue("AM was not registered in ZooKeeper within timeout", registered);
+ assertTrue("AM was not discovered by registry client", amDiscovered.get());
+
+ // Verify the AM record is available through the registry client
+ AMRecord amRecord = registryClient.getRecord(appId.toString());
+ assertNotNull("AM record should be retrievable from registry", amRecord);
+ assertEquals("Application ID should match", appId, amRecord.getApplicationId());
+ assertNotNull("Host should be set", amRecord.getHost());
+ assertTrue("Port should be positive", amRecord.getPort() > 0);
+
+ // Verify getAllRecords also returns the AM
+ List<AMRecord> allRecords = registryClient.getAllRecords();
+ assertNotNull("getAllRecords should not return null", allRecords);
+ assertFalse("getAllRecords should contain at least one record", allRecords.isEmpty());
+
+ boolean found = false;
+ for (AMRecord record : allRecords) {
+ if (record.getApplicationId().equals(appId)) {
+ found = true;
+ break;
+ }
+ }
+ assertTrue("AM record should be in getAllRecords", found);
+ LOG.info("Test completed successfully. AM was discovered: {}", amRecord);
+ }
+
+ private TezConfiguration getTezConfForZkDiscovery() {
+ TezConfiguration tezConf = new TezConfiguration();
+ tezConf.set(TezConfiguration.TEZ_FRAMEWORK_MODE, "STANDALONE_ZOOKEEPER");
Review Comment:
STANDALONE_ZOOKEEPER.name()
##########
tez-dag/src/test/java/org/apache/tez/client/registry/zookeeper/TestZkAMRegistryClient.java:
##########
@@ -0,0 +1,197 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.client.registry.zookeeper;
+
+import static org.junit.Assert.*;
+
+import java.io.File;
+import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicBoolean;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.curator.test.TestingServer;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.util.SystemClock;
+import org.apache.tez.client.registry.AMRecord;
+import org.apache.tez.client.registry.AMRegistryClientListener;
+import org.apache.tez.dag.api.TezConfiguration;
+import org.apache.tez.dag.app.DAGAppMaster;
+import org.apache.tez.dag.app.MockDAGAppMaster;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Unit tests for {@link ZkAMRegistryClient}.
+ * <p>
+ * This test class validates the ZooKeeper-based AM (Application Master) registry and discovery
+ * mechanism. It tests that when a DAGAppMaster is started with STANDALONE_ZOOKEEPER framework mode,
+ * it properly registers itself to ZooKeeper and can be discovered by a {@link ZkAMRegistryClient}.
+ * </p>
+ * <p>
+ * The tests use an embedded ZooKeeper {@link TestingServer} to avoid external dependencies
+ * and ensure test isolation.
+ * </p>
+ */
+public class TestZkAMRegistryClient {
+ private static final Logger LOG = LoggerFactory.getLogger(TestZkAMRegistryClient.class);
+ private static final File TEST_DIR = new File(System.getProperty("test.build.data", "target"),
+ TestZkAMRegistryClient.class.getName()).getAbsoluteFile();
+
+ /**
+ * Embedded ZooKeeper server for testing. Uses Apache Curator's {@link TestingServer}
+ * to provide an in-memory ZooKeeper instance.
+ */
+ private TestingServer zkServer;
+
+ /**
+ * ZooKeeper-based AM registry client used to discover and retrieve AM records.
+ */
+ private ZkAMRegistryClient registryClient;
+
+ /**
+ * Mock DAGAppMaster instance that registers itself to the ZooKeeper registry.
+ */
+ private DAGAppMaster dagAppMaster;
+
+ @Before
+ public void setup() throws Exception {
+ zkServer = new TestingServer();
+ zkServer.start();
+ LOG.info("Started ZooKeeper test server on port: {}", zkServer.getPort());
+ }
+
+ @After
+ public void teardown() throws Exception {
+ if (dagAppMaster != null) {
+ dagAppMaster.stop();
+ }
+ IOUtils.closeQuietly(registryClient);
+ IOUtils.closeQuietly(zkServer);
+ }
+
+ /**
+ * Tests the complete ZooKeeper-based AM registry and discovery flow.
+ * <p>
+ * This test validates the following workflow:
+ * </p>
+ * <ol>
+ * <li>Configure Tez with STANDALONE_ZOOKEEPER framework mode</li>
+ * <li>Create and start a {@link ZkAMRegistryClient} with an event listener</li>
+ * <li>Start a {@link MockDAGAppMaster} which registers itself to ZooKeeper</li>
+ * <li>Verify that the registry client's listener is notified of the AM registration</li>
+ * <li>Verify the AM record can be retrieved via {@link ZkAMRegistryClient#getRecord(String)}</li>
+ * <li>Verify the AM appears in the list from {@link ZkAMRegistryClient#getAllRecords()}</li>
+ * <li>Validate all expected fields (host, port, applicationId) are correctly set</li>
+ * </ol>
+ * <p>
+ * The test uses a {@link CountDownLatch} to synchronize between the AM registration
+ * event and the test assertions, ensuring the AM has fully registered before validation.
+ * </p>
+ *
+ * @throws Exception if any part of the test fails
+ */
+ @Test(timeout = 10000)
+ public void testZkAmRegistryDiscovery() throws Exception {
+ TezConfiguration tezConf = getTezConfForZkDiscovery();
+
+ ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(), 1);
+ ApplicationAttemptId attemptId = ApplicationAttemptId.newInstance(appId, 1);
+ ContainerId containerId = ContainerId.newContainerId(attemptId, 1);
+
+ CountDownLatch amRegisteredLatch = new CountDownLatch(1);
+ AtomicBoolean amDiscovered = new AtomicBoolean(false);
+
+ // Create and start the ZkAMRegistryClient
+ registryClient = ZkAMRegistryClient.getClient(tezConf);
+ registryClient.addListener(new AMRegistryClientListener() {
+ @Override
+ public void onAdd(AMRecord amRecord) {
+ LOG.info("AM added to registry: {}", amRecord);
+ if (amRecord.getApplicationId().equals(appId)) {
+ amDiscovered.set(true);
+ amRegisteredLatch.countDown();
+ }
+ }
+
+ @Override
+ public void onRemove(AMRecord amRecord) {
+ LOG.info("AM removed from registry: {}", amRecord);
+ }
+ });
+ registryClient.start();
+
+ String workingDir = TEST_DIR.toString();
+ String[] localDirs = new String[]{TEST_DIR.toString()};
+ String[] logDirs = new String[]{TEST_DIR + "/logs"};
+ String jobUserName = UserGroupInformation.getCurrentUser().getShortUserName();
+
+ dagAppMaster = new MockDAGAppMaster(attemptId, containerId, "localhost", 0, 0, SystemClock.getInstance(),
+ System.currentTimeMillis(), true, workingDir, localDirs, logDirs, new AtomicBoolean(true), false, false,
+ new Credentials(), jobUserName, 1, 1);
+
+ dagAppMaster.init(tezConf);
+ dagAppMaster.start();
+
+ // Wait for AM to be registered in ZooKeeper
+ boolean registered = amRegisteredLatch.await(30, TimeUnit.SECONDS);
+ assertTrue("AM was not registered in ZooKeeper within timeout", registered);
+ assertTrue("AM was not discovered by registry client", amDiscovered.get());
+
+ // Verify the AM record is available through the registry client
+ AMRecord amRecord = registryClient.getRecord(appId.toString());
+ assertNotNull("AM record should be retrievable from registry", amRecord);
+ assertEquals("Application ID should match", appId, amRecord.getApplicationId());
+ assertNotNull("Host should be set", amRecord.getHost());
+ assertTrue("Port should be positive", amRecord.getPort() > 0);
+
+ // Verify getAllRecords also returns the AM
+ List<AMRecord> allRecords = registryClient.getAllRecords();
+ assertNotNull("getAllRecords should not return null", allRecords);
+ assertFalse("getAllRecords should contain at least one record", allRecords.isEmpty());
+
+ boolean found = false;
+ for (AMRecord record : allRecords) {
+ if (record.getApplicationId().equals(appId)) {
+ found = true;
+ break;
+ }
+ }
+ assertTrue("AM record should be in getAllRecords", found);
+ LOG.info("Test completed successfully. AM was discovered: {}", amRecord);
Review Comment:
I am not sure what value does this log line add :-)
##########
tez-dag/src/main/java/org/apache/tez/dag/api/client/registry/zookeeper/ZkAMRegistry.java:
##########
@@ -0,0 +1,175 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.api.client.registry.zookeeper;
+
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.curator.RetryLoop;
+import org.apache.curator.RetryPolicy;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.registry.client.binding.RegistryUtils;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.tez.client.registry.AMRecord;
+import org.apache.tez.client.registry.AMRegistry;
+import org.apache.tez.client.registry.zookeeper.ZkConfig;
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.data.Stat;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Curator/Zookeeper impl of AMRegistry (for internal use only)
+ * Clients should use org.apache.tez.dag.api.client.registry.zookeeper.ZkAMRegistryClient instead.
+ */
+@InterfaceAudience.Private
+public class ZkAMRegistry extends AMRegistry {
+
+ private static final Logger LOG = LoggerFactory.getLogger(ZkAMRegistry.class);
+
+ private final List<AMRecord> amRecords = new ArrayList<>();
+ private final String externalId;
+
+ private CuratorFramework client = null;
+ private String namespace = null;
+ private ZkConfig zkConfig = null;
+ private boolean started = false;
+
+ public ZkAMRegistry(String externalId) {
+ super("ZkAMRegistry");
+ this.externalId = externalId;
+ }
+
+ @Override
+ public void serviceInit(Configuration conf) {
+ if (zkConfig == null) {
+ zkConfig = new ZkConfig(conf);
+ this.client = zkConfig.createCuratorFramework();
+ this.namespace = zkConfig.getZkNamespace();
+ LOG.info("AMRegistryZkImpl initialized");
+ }
+ }
+
+ @Override
+ public void serviceStart() throws Exception {
+ if (!started) {
+ client.start();
+ started = true;
+ LOG.info("AMRegistryZkImpl started");
+ }
+ }
+
+ //Deletes from Zookeeper AMRecords that were added by this instance
+ @Override
+ public void serviceStop() throws Exception {
+ List<AMRecord> records = new ArrayList<>(amRecords);
+ for (AMRecord amRecord : records) {
+ remove(amRecord);
+ }
+ client.close();
+ LOG.info("AMRegistryZkImpl shutdown");
+ }
+
+ //Serialize AMRecord to ServiceRecord and deliver the JSON bytes to
+ //zkNode at the path: <TEZ_AM_REGISTRY_NAMESPACE>/<appId>
+ @Override
+ public void add(AMRecord server) throws Exception {
+ RegistryUtils.ServiceRecordMarshal marshal = new RegistryUtils.ServiceRecordMarshal();
+ String json = marshal.toJson(server.toServiceRecord());
+ try {
+ final String path = namespace + "/" + server.getApplicationId().toString();
+ client.setData().forPath(path, json.getBytes(StandardCharsets.UTF_8));
+ LOG.info("Added AMRecord to zkpath {}", path);
+ } catch (KeeperException.NoNodeException nne) {
+ client.create().creatingParentContainersIfNeeded().withMode(CreateMode.EPHEMERAL)
Review Comment:
are we sure about using `CreateMode.EPHEMERAL` and not `PERSISTENT`?
##########
tez-dag/src/main/java/org/apache/tez/dag/api/client/registry/zookeeper/ZkAMRegistry.java:
##########
@@ -0,0 +1,175 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.api.client.registry.zookeeper;
+
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.curator.RetryLoop;
+import org.apache.curator.RetryPolicy;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.registry.client.binding.RegistryUtils;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.tez.client.registry.AMRecord;
+import org.apache.tez.client.registry.AMRegistry;
+import org.apache.tez.client.registry.zookeeper.ZkConfig;
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.data.Stat;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Curator/Zookeeper impl of AMRegistry (for internal use only)
+ * Clients should use org.apache.tez.dag.api.client.registry.zookeeper.ZkAMRegistryClient instead.
+ */
+@InterfaceAudience.Private
+public class ZkAMRegistry extends AMRegistry {
+
+ private static final Logger LOG = LoggerFactory.getLogger(ZkAMRegistry.class);
+
+ private final List<AMRecord> amRecords = new ArrayList<>();
+ private final String externalId;
+
+ private CuratorFramework client = null;
+ private String namespace = null;
+ private ZkConfig zkConfig = null;
+ private boolean started = false;
+
+ public ZkAMRegistry(String externalId) {
+ super("ZkAMRegistry");
+ this.externalId = externalId;
+ }
+
+ @Override
+ public void serviceInit(Configuration conf) {
+ if (zkConfig == null) {
+ zkConfig = new ZkConfig(conf);
+ this.client = zkConfig.createCuratorFramework();
+ this.namespace = zkConfig.getZkNamespace();
+ LOG.info("AMRegistryZkImpl initialized");
+ }
+ }
+
+ @Override
+ public void serviceStart() throws Exception {
+ if (!started) {
+ client.start();
+ started = true;
+ LOG.info("AMRegistryZkImpl started");
+ }
+ }
+
+ //Deletes from Zookeeper AMRecords that were added by this instance
+ @Override
+ public void serviceStop() throws Exception {
+ List<AMRecord> records = new ArrayList<>(amRecords);
+ for (AMRecord amRecord : records) {
+ remove(amRecord);
+ }
+ client.close();
+ LOG.info("AMRegistryZkImpl shutdown");
+ }
+
+ //Serialize AMRecord to ServiceRecord and deliver the JSON bytes to
+ //zkNode at the path: <TEZ_AM_REGISTRY_NAMESPACE>/<appId>
+ @Override
+ public void add(AMRecord server) throws Exception {
+ RegistryUtils.ServiceRecordMarshal marshal = new RegistryUtils.ServiceRecordMarshal();
+ String json = marshal.toJson(server.toServiceRecord());
+ try {
+ final String path = namespace + "/" + server.getApplicationId().toString();
+ client.setData().forPath(path, json.getBytes(StandardCharsets.UTF_8));
+ LOG.info("Added AMRecord to zkpath {}", path);
+ } catch (KeeperException.NoNodeException nne) {
+ client.create().creatingParentContainersIfNeeded().withMode(CreateMode.EPHEMERAL)
+ .forPath(namespace + "/" + server.getApplicationId().toString(), json.getBytes(StandardCharsets.UTF_8));
+ }
+ amRecords.add(server);
+ }
+
+ @Override
+ public void remove(AMRecord server) throws Exception {
+ amRecords.remove(server);
+ final String path = namespace + "/" + server.getApplicationId().toString();
Review Comment:
this ``namespace + "/" + server.getApplicationId().toString()`` is 3 places, can we refactor and create
```
private String pathFor(ApplicationId id) {
return namespace + "/" + id;
}
```
##########
tez-api/src/test/java/org/apache/tez/client/TestTezClient.java:
##########
@@ -447,6 +457,94 @@ public TezClientForTest testTezClient(boolean isSession, boolean shouldStop, Str
return client;
}
+ public void testTezClientReconnect(boolean isSession) throws Exception {
+ //Setup 1
+ Map<String, LocalResource> lrs = Maps.newHashMap();
+ String lrName1 = "LR1";
+ lrs.put(lrName1, LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test"),
+ LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1));
+
+ //Client 1
+ TezClientForTest client = configureAndCreateTezClient(lrs, isSession, null);
+
+ //Submission Context 1
+ ArgumentCaptor<ApplicationSubmissionContext> captor = ArgumentCaptor.forClass(ApplicationSubmissionContext.class);
+ when(client.mockYarnClient.getApplicationReport(client.mockAppId).getYarnApplicationState())
+ .thenReturn(YarnApplicationState.RUNNING);
+
+ //Client 1 start
+ client.start();
Review Comment:
we are starting client and all, but no closing, is there a proper cleanup somewhere, so it doesn't leak resources or so, can you just double check once
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] TEZ-4007: Introduce AmExtensions and Zookeeper-based FrameworkServices [tez]
Posted by "abstractdog (via GitHub)" <gi...@apache.org>.
abstractdog commented on code in PR #427:
URL: https://github.com/apache/tez/pull/427#discussion_r2549593120
##########
tez-api/src/main/java/org/apache/tez/client/registry/AMRecord.java:
##########
@@ -97,37 +110,54 @@ public AMRecord(AMRecord other) {
*/
public AMRecord(ServiceRecord serviceRecord) {
this.appId = ApplicationId.fromString(serviceRecord.get(APP_ID_RECORD_KEY));
- this.host = serviceRecord.get(HOST_RECORD_KEY);
+ this.hostName = serviceRecord.get(HOST_NAME_RECORD_KEY);
+ this.hostIp = serviceRecord.get(HOST_IP_RECORD_KEY);
this.port = Integer.parseInt(serviceRecord.get(PORT_RECORD_KEY));
- this.id = serviceRecord.get(OPAQUE_ID_KEY);
+ this.externalId = serviceRecord.get(EXTERNAL_ID_KEY);
+ this.computeName = serviceRecord.get(COMPUTE_GROUP_NAME_KEY);
}
public ApplicationId getApplicationId() {
return appId;
}
public String getHost() {
- return host;
+ return hostName;
+ }
+
+ public String getHostName() {
+ return hostName;
+ }
Review Comment:
good catch! let's have it as `getHostName` to make a clear distinction between this and `getHostIp`
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] [DRAFT] TEZ-4007: Zookeeper based FrameworkServices and AmExtensions (3/3) + refactor [tez]
Posted by "tez-yetus (via GitHub)" <gi...@apache.org>.
tez-yetus commented on PR #427:
URL: https://github.com/apache/tez/pull/427#issuecomment-3422331280
:broken_heart: **-1 overall**
| Vote | Subsystem | Runtime | Logfile | Comment |
|:----:|----------:|--------:|:--------:|:-------:|
| +0 :ok: | reexec | 23m 43s | | Docker mode activated. |
|||| _ Prechecks _ |
| +1 :green_heart: | dupname | 0m 1s | | No case conflicting files found. |
| +0 :ok: | detsecrets | 0m 0s | | detect-secrets was not available. |
| +0 :ok: | xmllint | 0m 0s | | xmllint was not available. |
| +1 :green_heart: | @author | 0m 0s | | The patch does not contain any @author tags. |
| +1 :green_heart: | test4tests | 0m 0s | | The patch appears to include 10 new or modified test files. |
|||| _ master Compile Tests _ |
| +0 :ok: | mvndep | 2m 43s | | Maven dependency ordering for branch |
| +1 :green_heart: | mvninstall | 9m 57s | | master passed |
| +1 :green_heart: | compile | 5m 53s | | master passed |
| +1 :green_heart: | checkstyle | 3m 42s | | master passed |
| +1 :green_heart: | javadoc | 4m 35s | | master passed |
| +0 :ok: | spotbugs | 2m 14s | | tez-api in master has 610 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 0m 46s | | tez-common in master has 13 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 1m 12s | | tez-runtime-library in master has 241 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 0m 57s | | tez-mapreduce in master has 129 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 0m 44s | | tez-examples in master has 2 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 1m 45s | | tez-dag in master has 785 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 6m 57s | | root in master has 2066 extant spotbugs warnings. |
|||| _ Patch Compile Tests _ |
| +0 :ok: | mvndep | 0m 17s | | Maven dependency ordering for patch |
| +1 :green_heart: | mvninstall | 6m 45s | | the patch passed |
| +1 :green_heart: | codespell | 0m 51s | | No new issues. |
| +1 :green_heart: | compile | 6m 1s | | the patch passed |
| +1 :green_heart: | javac | 6m 1s | | the patch passed |
| +1 :green_heart: | blanks | 0m 0s | | The patch has no blanks issues. |
| -0 :warning: | checkstyle | 0m 26s | [/results-checkstyle-tez-api.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/14/artifact/out/results-checkstyle-tez-api.txt) | tez-api: The patch generated 3 new + 134 unchanged - 11 fixed = 137 total (was 145) |
| -0 :warning: | checkstyle | 0m 23s | [/results-checkstyle-tez-examples.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/14/artifact/out/results-checkstyle-tez-examples.txt) | tez-examples: The patch generated 6 new + 0 unchanged - 0 fixed = 6 total (was 0) |
| -0 :warning: | checkstyle | 0m 36s | [/results-checkstyle-tez-dag.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/14/artifact/out/results-checkstyle-tez-dag.txt) | tez-dag: The patch generated 2 new + 818 unchanged - 2 fixed = 820 total (was 820) |
| -0 :warning: | checkstyle | 0m 47s | [/results-checkstyle-root.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/14/artifact/out/results-checkstyle-root.txt) | root: The patch generated 11 new + 954 unchanged - 13 fixed = 965 total (was 967) |
| -1 :x: | javadoc | 0m 42s | [/results-javadoc-javadoc-tez-api.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/14/artifact/out/results-javadoc-javadoc-tez-api.txt) | tez-api generated 1 new + 10 unchanged - 0 fixed = 11 total (was 10) |
| -1 :x: | javadoc | 1m 23s | [/results-javadoc-javadoc-root.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/14/artifact/out/results-javadoc-javadoc-root.txt) | root generated 1 new + 19 unchanged - 0 fixed = 20 total (was 19) |
| -1 :x: | spotbugs | 1m 44s | [/new-spotbugs-tez-api.html](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/14/artifact/out/new-spotbugs-tez-api.html) | tez-api generated 1 new + 610 unchanged - 0 fixed = 611 total (was 610) |
| -1 :x: | spotbugs | 7m 15s | [/new-spotbugs-root.html](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/14/artifact/out/new-spotbugs-root.html) | root generated 1 new + 2066 unchanged - 0 fixed = 2067 total (was 2066) |
|||| _ Other Tests _ |
| +1 :green_heart: | unit | 0m 52s | | tez-api in the patch passed. |
| +1 :green_heart: | unit | 0m 46s | | tez-common in the patch passed. |
| +1 :green_heart: | unit | 6m 8s | | tez-runtime-library in the patch passed. |
| +1 :green_heart: | unit | 1m 37s | | tez-mapreduce in the patch passed. |
| +1 :green_heart: | unit | 0m 31s | | tez-examples in the patch passed. |
| +1 :green_heart: | unit | 0m 47s | | tez-dag in the patch passed. |
| +1 :green_heart: | unit | 65m 55s | | root in the patch passed. |
| -1 :x: | asflicense | 3m 9s | [/results-asflicense.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/14/artifact/out/results-asflicense.txt) | The patch generated 2 ASF License warnings. |
| | | 187m 20s | | |
| Reason | Tests |
|-------:|:------|
| SpotBugs | module:tez-api |
| | new org.apache.tez.client.TezYarnClient(YarnClient) may expose internal representation by storing an externally mutable object into TezYarnClient.yarnClient At TezYarnClient.java:representation by storing an externally mutable object into TezYarnClient.yarnClient At TezYarnClient.java:[line 47] |
| SpotBugs | module:root |
| | new org.apache.tez.client.TezYarnClient(YarnClient) may expose internal representation by storing an externally mutable object into TezYarnClient.yarnClient At TezYarnClient.java:representation by storing an externally mutable object into TezYarnClient.yarnClient At TezYarnClient.java:[line 47] |
| Subsystem | Report/Notes |
|----------:|:-------------|
| Docker | ClientAPI=1.51 ServerAPI=1.51 base: https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/14/artifact/out/Dockerfile |
| GITHUB PR | https://github.com/apache/tez/pull/427 |
| Optional Tests | dupname asflicense javac javadoc unit codespell detsecrets xmllint compile spotbugs checkstyle |
| uname | Linux 79e428107d37 5.15.0-152-generic #162-Ubuntu SMP Wed Jul 23 09:48:42 UTC 2025 x86_64 x86_64 x86_64 GNU/Linux |
| Build tool | maven |
| Personality | /home/jenkins/jenkins-home/workspace/tez-multibranch_PR-427/src/.yetus/personality.sh |
| git revision | master / 3564579cd87fae89918e78a700536543398ec131 |
| Default Java | Ubuntu-21.0.8+9-Ubuntu-0ubuntu124.04.1 |
| Test Results | https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/14/testReport/ |
| Max. process+thread count | 2102 (vs. ulimit of 5500) |
| modules | C: tez-api tez-common tez-runtime-library tez-mapreduce tez-examples tez-dag . U: . |
| Console output | https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/14/console |
| versions | git=2.43.0 maven=3.8.7 spotbugs=4.9.3 codespell=2.0.0 |
| Powered by | Apache Yetus 0.15.1 https://yetus.apache.org |
This message was automatically generated.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] TEZ-4007: Introduce AmExtensions and Zookeeper-based FrameworkServices [tez]
Posted by "tez-yetus (via GitHub)" <gi...@apache.org>.
tez-yetus commented on PR #427:
URL: https://github.com/apache/tez/pull/427#issuecomment-3738387978
:broken_heart: **-1 overall**
| Vote | Subsystem | Runtime | Logfile | Comment |
|:----:|----------:|--------:|:--------:|:-------:|
| +0 :ok: | reexec | 20m 46s | | Docker mode activated. |
|||| _ Prechecks _ |
| +1 :green_heart: | dupname | 0m 0s | | No case conflicting files found. |
| +0 :ok: | detsecrets | 0m 0s | | detect-secrets was not available. |
| +0 :ok: | shelldocs | 0m 1s | | Shelldocs was not available. |
| +0 :ok: | xmllint | 0m 0s | | xmllint was not available. |
| +1 :green_heart: | @author | 0m 0s | | The patch does not contain any @author tags. |
| +1 :green_heart: | test4tests | 0m 0s | | The patch appears to include 14 new or modified test files. |
|||| _ master Compile Tests _ |
| +0 :ok: | mvndep | 2m 15s | | Maven dependency ordering for branch |
| +1 :green_heart: | mvninstall | 9m 32s | | master passed |
| +1 :green_heart: | compile | 5m 24s | | master passed |
| +1 :green_heart: | checkstyle | 3m 17s | | master passed |
| +1 :green_heart: | javadoc | 4m 5s | | master passed |
| -1 :x: | spotbugs | 1m 15s | [/branch-spotbugs-tez-api.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/40/artifact/out/branch-spotbugs-tez-api.txt) | tez-api in master failed. |
| -1 :x: | spotbugs | 0m 29s | [/branch-spotbugs-tez-common.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/40/artifact/out/branch-spotbugs-tez-common.txt) | tez-common in master failed. |
| -1 :x: | spotbugs | 0m 38s | [/branch-spotbugs-tez-runtime-library.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/40/artifact/out/branch-spotbugs-tez-runtime-library.txt) | tez-runtime-library in master failed. |
| -1 :x: | spotbugs | 0m 31s | [/branch-spotbugs-tez-examples.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/40/artifact/out/branch-spotbugs-tez-examples.txt) | tez-examples in master failed. |
| -1 :x: | spotbugs | 0m 43s | [/branch-spotbugs-tez-dag.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/40/artifact/out/branch-spotbugs-tez-dag.txt) | tez-dag in master failed. |
| -1 :x: | spotbugs | 0m 24s | [/branch-spotbugs-root.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/40/artifact/out/branch-spotbugs-root.txt) | root in master failed. |
| -0 :warning: | patch | 8m 59s | | Used diff version of patch file. Binary files and potentially other changes not applied. Please rebase and squash commits if necessary. |
|||| _ Patch Compile Tests _ |
| +0 :ok: | mvndep | 0m 19s | | Maven dependency ordering for patch |
| +1 :green_heart: | mvninstall | 6m 20s | | the patch passed |
| +1 :green_heart: | codespell | 0m 51s | | No new issues. |
| +1 :green_heart: | compile | 5m 25s | | the patch passed |
| +1 :green_heart: | javac | 5m 25s | | the patch passed |
| +1 :green_heart: | blanks | 0m 0s | | The patch has no blanks issues. |
| +1 :green_heart: | checkstyle | 0m 27s | | tez-api: The patch generated 0 new + 118 unchanged - 3 fixed = 118 total (was 121) |
| +1 :green_heart: | checkstyle | 0m 23s | | The patch passed checkstyle in tez-common |
| +1 :green_heart: | checkstyle | 0m 27s | | The patch passed checkstyle in tez-runtime-library |
| +1 :green_heart: | checkstyle | 0m 24s | | tez-examples: The patch generated 0 new + 10 unchanged - 1 fixed = 10 total (was 11) |
| -0 :warning: | checkstyle | 0m 34s | [/results-checkstyle-tez-dag.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/40/artifact/out/results-checkstyle-tez-dag.txt) | tez-dag: The patch generated 2 new + 610 unchanged - 2 fixed = 612 total (was 612) |
| -0 :warning: | checkstyle | 0m 46s | [/results-checkstyle-root.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/40/artifact/out/results-checkstyle-root.txt) | root: The patch generated 2 new + 740 unchanged - 6 fixed = 742 total (was 746) |
| +1 :green_heart: | hadolint | 0m 1s | | No new issues. |
| +1 :green_heart: | shellcheck | 0m 0s | | No new issues. |
| +1 :green_heart: | javadoc | 3m 57s | | the patch passed |
| +1 :green_heart: | spotbugs | 13m 41s | | the patch passed |
|||| _ Other Tests _ |
| +1 :green_heart: | unit | 2m 35s | | tez-api in the patch passed. |
| +1 :green_heart: | unit | 0m 43s | | tez-common in the patch passed. |
| +1 :green_heart: | unit | 6m 12s | | tez-runtime-library in the patch passed. |
| +1 :green_heart: | unit | 0m 31s | | tez-examples in the patch passed. |
| +1 :green_heart: | unit | 5m 54s | | tez-dag in the patch passed. |
| +1 :green_heart: | unit | 72m 52s | | root in the patch passed. |
| +1 :green_heart: | asflicense | 2m 47s | | The patch does not generate ASF License warnings. |
| | | 178m 42s | | |
| Subsystem | Report/Notes |
|----------:|:-------------|
| Docker | ClientAPI=1.52 ServerAPI=1.52 base: https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/40/artifact/out/Dockerfile |
| GITHUB PR | https://github.com/apache/tez/pull/427 |
| Optional Tests | dupname asflicense codespell detsecrets hadolint shellcheck shelldocs javac javadoc unit xmllint compile spotbugs checkstyle |
| uname | Linux be3143d28af1 5.15.0-161-generic #171-Ubuntu SMP Sat Oct 11 08:17:01 UTC 2025 x86_64 x86_64 x86_64 GNU/Linux |
| Build tool | maven |
| Personality | /home/jenkins/jenkins-home/workspace/tez-multibranch_PR-427/src/.yetus/personality.sh |
| git revision | master / 0f9a6789796d2e66ed16363e9ed4c30f205a52b5 |
| Default Java | Ubuntu-21.0.9+10-Ubuntu-124.04 |
| Test Results | https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/40/testReport/ |
| Max. process+thread count | 1306 (vs. ulimit of 5500) |
| modules | C: tez-api tez-common tez-runtime-library tez-examples tez-dag . U: . |
| Console output | https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/40/console |
| versions | git=2.43.0 maven=3.8.7 hadolint=1.18.0-0-g76eee5c codespell=2.0.0 shellcheck=0.7.1 |
| Powered by | Apache Yetus 0.15.1 https://yetus.apache.org |
This message was automatically generated.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] TEZ-4007: Introduce AmExtensions and Zookeeper-based FrameworkServices [tez]
Posted by "abstractdog (via GitHub)" <gi...@apache.org>.
abstractdog commented on code in PR #427:
URL: https://github.com/apache/tez/pull/427#discussion_r2681528387
##########
tez-api/src/main/java/org/apache/tez/frameworkplugins/zookeeper/ZkStandaloneClientFrameworkService.java:
##########
@@ -0,0 +1,29 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tez.frameworkplugins.zookeeper;
+
+
+import org.apache.tez.client.FrameworkClient;
+import org.apache.tez.client.registry.zookeeper.ZkFrameworkClient;
+import org.apache.tez.frameworkplugins.ClientFrameworkService;
+
+public class ZkStandaloneClientFrameworkService implements ClientFrameworkService {
+ @Override public FrameworkClient newFrameworkClient() {
Review Comment:
good catch, thanks!
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] TEZ-4007: Introduce AmExtensions and Zookeeper-based FrameworkServices [tez]
Posted by "abstractdog (via GitHub)" <gi...@apache.org>.
abstractdog commented on PR #427:
URL: https://github.com/apache/tez/pull/427#issuecomment-3728629713
@ayushtkn : address your comments, good catches, thanks!
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] TEZ-4007: Introduce AmExtensions and Zookeeper-based FrameworkServices [tez]
Posted by "abstractdog (via GitHub)" <gi...@apache.org>.
abstractdog commented on PR #427:
URL: https://github.com/apache/tez/pull/427#issuecomment-3748793917
Thanks, @ayushtkn, for the approval!
I'm about to merge this soon, let me wait another few hours to maybe let @okumin, @zhangbutao chime in if I addressed all their comments regarding the code
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] TEZ-4007: Introduce AmExtensions and Zookeeper-based FrameworkServices [tez]
Posted by "abstractdog (via GitHub)" <gi...@apache.org>.
abstractdog commented on code in PR #427:
URL: https://github.com/apache/tez/pull/427#discussion_r2565261793
##########
tez-dag/src/main/java/org/apache/tez/dag/api/client/registry/zookeeper/ZkAMRegistry.java:
##########
@@ -0,0 +1,175 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.api.client.registry.zookeeper;
+
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.curator.RetryLoop;
+import org.apache.curator.RetryPolicy;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.registry.client.binding.RegistryUtils;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.tez.client.registry.AMRecord;
+import org.apache.tez.client.registry.AMRegistry;
+import org.apache.tez.client.registry.zookeeper.ZkConfig;
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.data.Stat;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Curator/Zookeeper impl of AMRegistry (for internal use only)
+ * Clients should use org.apache.tez.dag.api.client.registry.zookeeper.ZkAMRegistryClient instead.
+ */
+@InterfaceAudience.Private
+public class ZkAMRegistry extends AMRegistry {
+
+ private static final Logger LOG = LoggerFactory.getLogger(ZkAMRegistry.class);
+
+ private final List<AMRecord> amRecords = new ArrayList<>();
+ private final String externalId;
+
+ private CuratorFramework client = null;
+ private String namespace = null;
+ private ZkConfig zkConfig = null;
+ private boolean started = false;
+
+ public ZkAMRegistry(String externalId) {
+ super("ZkAMRegistry");
+ this.externalId = externalId;
+ }
+
+ @Override
+ public void serviceInit(Configuration conf) {
+ if (zkConfig == null) {
+ zkConfig = new ZkConfig(conf);
+ this.client = zkConfig.createCuratorFramework();
+ this.namespace = zkConfig.getZkNamespace();
+ LOG.info("AMRegistryZkImpl initialized");
+ }
+ }
+
+ @Override
+ public void serviceStart() throws Exception {
+ if (!started) {
+ client.start();
+ started = true;
+ LOG.info("AMRegistryZkImpl started");
+ }
+ }
+
+ //Deletes from Zookeeper AMRecords that were added by this instance
+ @Override
+ public void serviceStop() throws Exception {
+ List<AMRecord> records = new ArrayList<>(amRecords);
+ for (AMRecord amRecord : records) {
+ remove(amRecord);
+ }
+ client.close();
+ LOG.info("AMRegistryZkImpl shutdown");
+ }
+
+ //Serialize AMRecord to ServiceRecord and deliver the JSON bytes to
+ //zkNode at the path: <TEZ_AM_REGISTRY_NAMESPACE>/<appId>
+ @Override
+ public void add(AMRecord server) throws Exception {
+ RegistryUtils.ServiceRecordMarshal marshal = new RegistryUtils.ServiceRecordMarshal();
+ String json = marshal.toJson(server.toServiceRecord());
+ try {
+ final String path = namespace + "/" + server.getApplicationId().toString();
+ client.setData().forPath(path, json.getBytes(StandardCharsets.UTF_8));
+ LOG.info("Added AMRecord to zkpath {}", path);
+ } catch (KeeperException.NoNodeException nne) {
+ client.create().creatingParentContainersIfNeeded().withMode(CreateMode.EPHEMERAL)
+ .forPath(namespace + "/" + server.getApplicationId().toString(), json.getBytes(StandardCharsets.UTF_8));
+ }
+ amRecords.add(server);
+ }
+
+ @Override
+ public void remove(AMRecord server) throws Exception {
+ amRecords.remove(server);
+ final String path = namespace + "/" + server.getApplicationId().toString();
+ client.delete().forPath(path);
+ LOG.info("Deleted AMRecord from zkpath {}", path);
+ }
+
+ @Override
+ public ApplicationId generateNewId() throws Exception {
+ createNamespaceIfNotExists();
+ long namespaceCreationTime = getNamespaceCreationTime();
+
+ boolean success = false;
+ long startTime = System.currentTimeMillis();
+ RetryPolicy retryPolicy = zkConfig.getRetryPolicy();
+ int tryId = 0;
+ for (int i = 0; (i < zkConfig.getCuratorMaxRetries()) && !success; i++) {
+ List<String> children = client.getChildren().forPath(namespace);
+ if (children != null && !children.isEmpty()) {
+ children.sort(Collections.reverseOrder());
+ String last = children.getFirst();
+ ApplicationId lastAppId = ApplicationId.fromString(last);
+ tryId = lastAppId.getId() + 1;
Review Comment:
added a unit test for this
regarding lexiographical order, this is not the problem, as application IDs have leading zero padding, added log messages before and after
```
2025-11-27 08:22:52,222 INFO [pool-9-thread-13] zookeeper.ZkAMRegistry (ZkAMRegistry.java:generateNewId(141)) - Children before sort: [application_1764228156152_0010, application_1764228156152_0000, application_1764228156152_0011, application_1764228156152_0001, application_1764228156152_0012, application_1764228156152_0002, application_1764228156152_0013, application_1764228156152_0003, application_1764228156152_0014, application_1764228156152_0004, application_1764228156152_0015, application_1764228156152_0005, application_1764228156152_0016, application_1764228156152_0006, application_1764228156152_0017, application_1764228156152_0007, application_1764228156152_0018, application_1764228156152_0008, application_1764228156152_0009]
2025-11-27 08:22:52,222 INFO [pool-9-thread-13] zookeeper.ZkAMRegistry (ZkAMRegistry.java:generateNewId(145)) - Children after sort (last: application_1764228156152_0018): [application_1764228156152_0018, application_1764228156152_0017, application_1764228156152_0016, application_1764228156152_0015, application_1764228156152_0014, application_1764228156152_0013, application_1764228156152_0012, application_1764228156152_0011, application_1764228156152_0010, application_1764228156152_0009, application_1764228156152_0008, application_1764228156152_0007, application_1764228156152_0006, application_1764228156152_0005, application_1764228156152_0004, application_1764228156152_0003, application_1764228156152_0002, application_1764228156152_0001, application_1764228156152_0000]
```
I guess if this were a problem, it should swap "10" and "9" unexpectedly, doesn't even have to go for 99 vs 100
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] TEZ-4007: Introduce AmExtensions and Zookeeper-based FrameworkServices [tez]
Posted by "abstractdog (via GitHub)" <gi...@apache.org>.
abstractdog commented on code in PR #427:
URL: https://github.com/apache/tez/pull/427#discussion_r2567539463
##########
tez-dag/src/main/java/org/apache/tez/dag/api/client/registry/zookeeper/ZkAMRegistry.java:
##########
@@ -0,0 +1,175 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.api.client.registry.zookeeper;
+
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.curator.RetryLoop;
+import org.apache.curator.RetryPolicy;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.registry.client.binding.RegistryUtils;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.tez.client.registry.AMRecord;
+import org.apache.tez.client.registry.AMRegistry;
+import org.apache.tez.client.registry.zookeeper.ZkConfig;
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.data.Stat;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Curator/Zookeeper impl of AMRegistry (for internal use only)
+ * Clients should use org.apache.tez.dag.api.client.registry.zookeeper.ZkAMRegistryClient instead.
+ */
+@InterfaceAudience.Private
+public class ZkAMRegistry extends AMRegistry {
+
+ private static final Logger LOG = LoggerFactory.getLogger(ZkAMRegistry.class);
+
+ private final List<AMRecord> amRecords = new ArrayList<>();
+ private final String externalId;
+
+ private CuratorFramework client = null;
+ private String namespace = null;
+ private ZkConfig zkConfig = null;
+ private boolean started = false;
+
+ public ZkAMRegistry(String externalId) {
+ super("ZkAMRegistry");
+ this.externalId = externalId;
+ }
+
+ @Override
+ public void serviceInit(Configuration conf) {
+ if (zkConfig == null) {
+ zkConfig = new ZkConfig(conf);
+ this.client = zkConfig.createCuratorFramework();
+ this.namespace = zkConfig.getZkNamespace();
+ LOG.info("AMRegistryZkImpl initialized");
+ }
+ }
+
+ @Override
+ public void serviceStart() throws Exception {
+ if (!started) {
+ client.start();
+ started = true;
+ LOG.info("AMRegistryZkImpl started");
+ }
+ }
+
+ //Deletes from Zookeeper AMRecords that were added by this instance
+ @Override
+ public void serviceStop() throws Exception {
+ List<AMRecord> records = new ArrayList<>(amRecords);
+ for (AMRecord amRecord : records) {
+ remove(amRecord);
+ }
+ client.close();
+ LOG.info("AMRegistryZkImpl shutdown");
+ }
+
+ //Serialize AMRecord to ServiceRecord and deliver the JSON bytes to
+ //zkNode at the path: <TEZ_AM_REGISTRY_NAMESPACE>/<appId>
+ @Override
+ public void add(AMRecord server) throws Exception {
+ RegistryUtils.ServiceRecordMarshal marshal = new RegistryUtils.ServiceRecordMarshal();
+ String json = marshal.toJson(server.toServiceRecord());
+ try {
+ final String path = namespace + "/" + server.getApplicationId().toString();
+ client.setData().forPath(path, json.getBytes(StandardCharsets.UTF_8));
+ LOG.info("Added AMRecord to zkpath {}", path);
+ } catch (KeeperException.NoNodeException nne) {
Review Comment:
the current logic cannot account for and handle other zookeeper-side problems, because it would make this implementation complicated, instead, a fatal problem during the AM id generation process leads to AM start failure, which is eventually retried by the environment: we're assuming that in a kubernetes env, AM start failure is retried until the desired replicas
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] TEZ-4007: Introduce AmExtensions and Zookeeper-based FrameworkServices [tez]
Posted by "abstractdog (via GitHub)" <gi...@apache.org>.
abstractdog commented on code in PR #427:
URL: https://github.com/apache/tez/pull/427#discussion_r2567589882
##########
tez-dag/src/main/java/org/apache/tez/frameworkplugins/zookeeper/ZkStandaloneAMExtensions.java:
##########
@@ -0,0 +1,90 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tez.frameworkplugins.zookeeper;
+
+import java.io.IOException;
+import java.util.Map;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.tez.client.registry.AMRegistry;
+import org.apache.tez.common.TezUtilsInternal;
+import org.apache.tez.common.security.JobTokenIdentifier;
+import org.apache.tez.common.security.JobTokenSecretManager;
+import org.apache.tez.common.security.TokenCache;
+import org.apache.tez.dag.api.records.DAGProtos;
+import org.apache.tez.dag.app.ClusterInfo;
+import org.apache.tez.dag.app.dag.Vertex;
+import org.apache.tez.frameworkplugins.AMExtensions;
+import org.apache.tez.frameworkplugins.ServerFrameworkService;
+
+public class ZkStandaloneAMExtensions implements AMExtensions {
+
+ private final ServerFrameworkService frameworkService;
+
+ public ZkStandaloneAMExtensions(ServerFrameworkService frameworkService) {
+ this.frameworkService = frameworkService;
+ }
+
+ @Override
+ public ContainerId allocateContainerId(Configuration conf) {
+ try {
+ AMRegistry amRegistry = frameworkService.getAMRegistry(conf);
+ if (amRegistry != null) {
+ ApplicationId appId = amRegistry.generateNewId();
+ // attemptId is set to 1 only then APP_LAUNCHED event gets triggered
+ ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.newInstance(appId, 1);
+ return ContainerId.newContainerId(applicationAttemptId, 0);
+ } else {
+ throw new RuntimeException("AMRegistry is required for ZkStandaloneAmExtensions");
+ }
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void checkTaskResources(Map<String, Vertex> vertices, ClusterInfo clusterInfo) {
+ // no-op: Zookeeper-based framework current enforce task resources at the moment
+ }
+
+ @Override
+ public DAGProtos.ConfigurationProto loadConfigurationProto() throws IOException {
+ return TezUtilsInternal.loadConfProtoFromText();
+ }
+
+ @Override
+ public Token<JobTokenIdentifier> getSessionToken(ApplicationAttemptId appAttemptID,
+ JobTokenSecretManager jobTokenSecretManager, Credentials amCredentials) {
+ JobTokenIdentifier identifier = new JobTokenIdentifier(new Text(appAttemptID.getApplicationId().toString()));
Review Comment:
the standalone mode doesn't account for different application attempt ids
while generating the app id, we rely on the zookeeper logic implemented in ZkAmRegistry, then deriving an artificial container id in this class (ZkStandaloneAMExtensions.allocateContainerId), see constant app attempt id:
```
ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.newInstance(appId, 1);
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] [DRAFT] TEZ-4007: Zookeeper based FrameworkServices and AmExtensions (3/3) + refactor [tez]
Posted by "tez-yetus (via GitHub)" <gi...@apache.org>.
tez-yetus commented on PR #427:
URL: https://github.com/apache/tez/pull/427#issuecomment-3406897328
:broken_heart: **-1 overall**
| Vote | Subsystem | Runtime | Logfile | Comment |
|:----:|----------:|--------:|:--------:|:-------:|
| +0 :ok: | reexec | 0m 15s | | Docker mode activated. |
|||| _ Prechecks _ |
| +1 :green_heart: | dupname | 0m 1s | | No case conflicting files found. |
| +0 :ok: | detsecrets | 0m 0s | | detect-secrets was not available. |
| +0 :ok: | xmllint | 0m 0s | | xmllint was not available. |
| +1 :green_heart: | @author | 0m 0s | | The patch does not contain any @author tags. |
| +1 :green_heart: | test4tests | 0m 0s | | The patch appears to include 10 new or modified test files. |
|||| _ master Compile Tests _ |
| +0 :ok: | mvndep | 2m 40s | | Maven dependency ordering for branch |
| +1 :green_heart: | mvninstall | 9m 48s | | master passed |
| +1 :green_heart: | compile | 5m 55s | | master passed |
| +1 :green_heart: | checkstyle | 3m 44s | | master passed |
| +1 :green_heart: | javadoc | 4m 37s | | master passed |
| +0 :ok: | spotbugs | 2m 10s | | tez-api in master has 610 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 0m 46s | | tez-common in master has 13 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 1m 12s | | tez-runtime-library in master has 241 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 0m 57s | | tez-mapreduce in master has 129 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 0m 43s | | tez-examples in master has 2 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 1m 43s | | tez-dag in master has 785 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 7m 0s | | root in master has 2066 extant spotbugs warnings. |
|||| _ Patch Compile Tests _ |
| +0 :ok: | mvndep | 0m 16s | | Maven dependency ordering for patch |
| +1 :green_heart: | mvninstall | 6m 47s | | the patch passed |
| +1 :green_heart: | codespell | 0m 50s | | No new issues. |
| +1 :green_heart: | compile | 6m 2s | | the patch passed |
| +1 :green_heart: | javac | 6m 2s | | the patch passed |
| +1 :green_heart: | blanks | 0m 1s | | The patch has no blanks issues. |
| -0 :warning: | checkstyle | 0m 25s | [/results-checkstyle-tez-api.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/7/artifact/out/results-checkstyle-tez-api.txt) | tez-api: The patch generated 3 new + 132 unchanged - 11 fixed = 135 total (was 143) |
| -0 :warning: | checkstyle | 0m 23s | [/results-checkstyle-tez-examples.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/7/artifact/out/results-checkstyle-tez-examples.txt) | tez-examples: The patch generated 6 new + 0 unchanged - 0 fixed = 6 total (was 0) |
| -0 :warning: | checkstyle | 0m 36s | [/results-checkstyle-tez-dag.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/7/artifact/out/results-checkstyle-tez-dag.txt) | tez-dag: The patch generated 1 new + 818 unchanged - 2 fixed = 819 total (was 820) |
| -0 :warning: | checkstyle | 0m 48s | [/results-checkstyle-root.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/7/artifact/out/results-checkstyle-root.txt) | root: The patch generated 10 new + 952 unchanged - 13 fixed = 962 total (was 965) |
| +1 :green_heart: | javadoc | 4m 28s | | the patch passed |
| +1 :green_heart: | spotbugs | 15m 27s | | the patch passed |
|||| _ Other Tests _ |
| +1 :green_heart: | unit | 2m 28s | | tez-api in the patch passed. |
| +1 :green_heart: | unit | 0m 44s | | tez-common in the patch passed. |
| +1 :green_heart: | unit | 6m 7s | | tez-runtime-library in the patch passed. |
| +1 :green_heart: | unit | 1m 35s | | tez-mapreduce in the patch passed. |
| +1 :green_heart: | unit | 0m 31s | | tez-examples in the patch passed. |
| -1 :x: | unit | 5m 23s | [/patch-unit-tez-dag.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/7/artifact/out/patch-unit-tez-dag.txt) | tez-dag in the patch passed. |
| -1 :x: | unit | 72m 46s | [/patch-unit-root.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/7/artifact/out/patch-unit-root.txt) | root in the patch passed. |
| -1 :x: | asflicense | 3m 10s | [/results-asflicense.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/7/artifact/out/results-asflicense.txt) | The patch generated 1 ASF License warnings. |
| | | 176m 35s | | |
| Reason | Tests |
|-------:|:------|
| Failed junit tests | tez.dag.app.dag.impl.TestDAGRecovery |
| | tez.dag.app.dag.impl.TestDAGRecovery |
| Subsystem | Report/Notes |
|----------:|:-------------|
| Docker | ClientAPI=1.51 ServerAPI=1.51 base: https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/7/artifact/out/Dockerfile |
| GITHUB PR | https://github.com/apache/tez/pull/427 |
| Optional Tests | dupname asflicense javac javadoc unit codespell detsecrets xmllint compile spotbugs checkstyle |
| uname | Linux 79a4a0fc2c4a 5.15.0-152-generic #162-Ubuntu SMP Wed Jul 23 09:48:42 UTC 2025 x86_64 x86_64 x86_64 GNU/Linux |
| Build tool | maven |
| Personality | /home/jenkins/jenkins-home/workspace/tez-multibranch_PR-427/src/.yetus/personality.sh |
| git revision | master / 15bc7cf223bf6d0b0741712679e2b3c7fab26fa7 |
| Default Java | Ubuntu-21.0.8+9-Ubuntu-0ubuntu124.04.1 |
| Test Results | https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/7/testReport/ |
| Max. process+thread count | 2109 (vs. ulimit of 5500) |
| modules | C: tez-api tez-common tez-runtime-library tez-mapreduce tez-examples tez-dag . U: . |
| Console output | https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/7/console |
| versions | git=2.43.0 maven=3.8.7 spotbugs=4.9.3 codespell=2.0.0 |
| Powered by | Apache Yetus 0.15.1 https://yetus.apache.org |
This message was automatically generated.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] TEZ-4007: Zookeeper based FrameworkServices and AmExtensions (3/3) - initial patch [tez]
Posted by "tez-yetus (via GitHub)" <gi...@apache.org>.
tez-yetus commented on PR #427:
URL: https://github.com/apache/tez/pull/427#issuecomment-3262425102
:broken_heart: **-1 overall**
| Vote | Subsystem | Runtime | Logfile | Comment |
|:----:|----------:|--------:|:--------:|:-------:|
| +0 :ok: | reexec | 22m 54s | | Docker mode activated. |
|||| _ Prechecks _ |
| +1 :green_heart: | dupname | 0m 0s | | No case conflicting files found. |
| +0 :ok: | detsecrets | 0m 1s | | detect-secrets was not available. |
| +0 :ok: | xmllint | 0m 1s | | xmllint was not available. |
| +1 :green_heart: | @author | 0m 0s | | The patch does not contain any @author tags. |
| +1 :green_heart: | test4tests | 0m 0s | | The patch appears to include 7 new or modified test files. |
|||| _ master Compile Tests _ |
| +0 :ok: | mvndep | 2m 35s | | Maven dependency ordering for branch |
| +1 :green_heart: | mvninstall | 9m 43s | | master passed |
| +1 :green_heart: | compile | 5m 56s | | master passed |
| +1 :green_heart: | checkstyle | 3m 39s | | master passed |
| +1 :green_heart: | javadoc | 4m 29s | | master passed |
| +0 :ok: | spotbugs | 2m 11s | | tez-api in master has 610 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 0m 48s | | tez-common in master has 13 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 1m 10s | | tez-runtime-library in master has 241 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 0m 56s | | tez-mapreduce in master has 129 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 0m 43s | | tez-examples in master has 2 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 1m 44s | | tez-dag in master has 785 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 6m 57s | | root in master has 2066 extant spotbugs warnings. |
|||| _ Patch Compile Tests _ |
| +0 :ok: | mvndep | 0m 16s | | Maven dependency ordering for patch |
| +1 :green_heart: | mvninstall | 6m 39s | | the patch passed |
| -1 :x: | codespell | 0m 50s | [/results-codespell.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/1/artifact/out/results-codespell.txt) | The patch generated 1 new + 26 unchanged - 0 fixed = 27 total (was 26) |
| +1 :green_heart: | compile | 5m 51s | | the patch passed |
| +1 :green_heart: | javac | 5m 51s | | the patch passed |
| -1 :x: | blanks | 0m 0s | [/blanks-eol.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/1/artifact/out/blanks-eol.txt) | The patch has 4 line(s) that end in blanks. Use git apply --whitespace=fix <<patch_file>>. Refer https://git-scm.com/docs/git-apply |
| -0 :warning: | checkstyle | 0m 26s | [/results-checkstyle-tez-api.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/1/artifact/out/results-checkstyle-tez-api.txt) | tez-api: The patch generated 28 new + 112 unchanged - 11 fixed = 140 total (was 123) |
| -0 :warning: | checkstyle | 0m 28s | [/results-checkstyle-tez-runtime-library.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/1/artifact/out/results-checkstyle-tez-runtime-library.txt) | tez-runtime-library: The patch generated 5 new + 0 unchanged - 0 fixed = 5 total (was 0) |
| -0 :warning: | checkstyle | 0m 25s | [/results-checkstyle-tez-mapreduce.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/1/artifact/out/results-checkstyle-tez-mapreduce.txt) | tez-mapreduce: The patch generated 11 new + 0 unchanged - 0 fixed = 11 total (was 0) |
| -0 :warning: | checkstyle | 0m 22s | [/results-checkstyle-tez-examples.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/1/artifact/out/results-checkstyle-tez-examples.txt) | tez-examples: The patch generated 19 new + 0 unchanged - 0 fixed = 19 total (was 0) |
| -0 :warning: | checkstyle | 0m 36s | [/results-checkstyle-tez-dag.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/1/artifact/out/results-checkstyle-tez-dag.txt) | tez-dag: The patch generated 44 new + 697 unchanged - 1 fixed = 741 total (was 698) |
| -0 :warning: | checkstyle | 0m 46s | [/results-checkstyle-root.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/1/artifact/out/results-checkstyle-root.txt) | root: The patch generated 107 new + 862 unchanged - 12 fixed = 969 total (was 874) |
| +1 :green_heart: | javadoc | 4m 29s | | the patch passed |
| -1 :x: | spotbugs | 1m 44s | [/new-spotbugs-tez-api.html](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/1/artifact/out/new-spotbugs-tez-api.html) | tez-api generated 6 new + 610 unchanged - 0 fixed = 616 total (was 610) |
| -1 :x: | spotbugs | 2m 0s | [/new-spotbugs-tez-dag.html](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/1/artifact/out/new-spotbugs-tez-dag.html) | tez-dag generated 6 new + 785 unchanged - 0 fixed = 791 total (was 785) |
| -1 :x: | spotbugs | 7m 18s | [/new-spotbugs-root.html](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/1/artifact/out/new-spotbugs-root.html) | root generated 12 new + 2066 unchanged - 0 fixed = 2078 total (was 2066) |
|||| _ Other Tests _ |
| -1 :x: | unit | 2m 29s | [/patch-unit-tez-api.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/1/artifact/out/patch-unit-tez-api.txt) | tez-api in the patch passed. |
| +1 :green_heart: | unit | 0m 41s | | tez-common in the patch passed. |
| +1 :green_heart: | unit | 6m 4s | | tez-runtime-library in the patch passed. |
| +1 :green_heart: | unit | 1m 37s | | tez-mapreduce in the patch passed. |
| +1 :green_heart: | unit | 0m 31s | | tez-examples in the patch passed. |
| -1 :x: | unit | 5m 22s | [/patch-unit-tez-dag.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/1/artifact/out/patch-unit-tez-dag.txt) | tez-dag in the patch passed. |
| -1 :x: | unit | 72m 35s | [/patch-unit-root.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/1/artifact/out/patch-unit-root.txt) | root in the patch passed. |
| -1 :x: | asflicense | 3m 8s | [/results-asflicense.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/1/artifact/out/results-asflicense.txt) | The patch generated 9 ASF License warnings. |
| | | 198m 15s | | |
| Reason | Tests |
|-------:|:------|
| SpotBugs | module:tez-api |
| | Found reliance on default encoding in org.apache.tez.client.registry.zookeeper.ZkAMRegistryClient.getAMRecord(ChildData):in org.apache.tez.client.registry.zookeeper.ZkAMRegistryClient.getAMRecord(ChildData): new String(byte[]) At ZkAMRegistryClient.java:[line 101] |
| | Shared primitive variable "amPort" in one thread may not yield the value of the most recent write from another thread At ZkFrameworkClient.java:one thread may not yield the value of the most recent write from another thread At ZkFrameworkClient.java:[line 109] |
| | Shared primitive variable "isRunning" in one thread may not yield the value of the most recent write from another thread At ZkFrameworkClient.java:one thread may not yield the value of the most recent write from another thread At ZkFrameworkClient.java:[line 65] |
| | Shared primitive variable "isRunning" in one thread may not yield the value of the most recent write from another thread At ZkFrameworkClient.java:one thread may not yield the value of the most recent write from another thread At ZkFrameworkClient.java:[line 69] |
| | org.apache.tez.client.registry.zookeeper.ZkFrameworkClient.init(TezConfiguration) may expose internal representation by storing an externally mutable object into ZkFrameworkClient.tezConf At ZkFrameworkClient.java:by storing an externally mutable object into ZkFrameworkClient.tezConf At ZkFrameworkClient.java:[line 48] |
| | Unread field:ZkFrameworkClient.java:[line 48] |
| SpotBugs | module:tez-dag |
| | new org.apache.tez.dag.api.client.DAGClientServer(DAGClientHandler, ApplicationAttemptId, FileSystem) may expose internal representation by storing an externally mutable object into DAGClientServer.applicationAttemptId At DAGClientServer.java:expose internal representation by storing an externally mutable object into DAGClientServer.applicationAttemptId At DAGClientServer.java:[line 63] |
| | Found reliance on default encoding in org.apache.tez.dag.api.client.registry.zookeeper.ZkAMRegistry.add(AMRecord):in org.apache.tez.dag.api.client.registry.zookeeper.ZkAMRegistry.add(AMRecord): String.getBytes() At ZkAMRegistry.java:[line 99] |
| | Found reliance on default encoding in org.apache.tez.dag.app.DAGAppMaster.getConfigurationProtoFromText():in org.apache.tez.dag.app.DAGAppMaster.getConfigurationProtoFromText(): new java.io.InputStreamReader(InputStream) At DAGAppMaster.java:[line 2889] |
| | Found reliance on default encoding in org.apache.tez.dag.app.DAGAppMaster.getConfigurationProtoFromText():in org.apache.tez.dag.app.DAGAppMaster.getConfigurationProtoFromText(): String.getBytes() At DAGAppMaster.java:[line 2897] |
| | org.apache.tez.dag.app.DAGAppMaster.amExts isn't final but should be At DAGAppMaster.java:be At DAGAppMaster.java:[line 237] |
| | org.apache.tez.dag.app.DAGAppMaster.getConfigurationProtoFromText() may fail to close stream At DAGAppMaster.java:stream At DAGAppMaster.java:[line 2889] |
| SpotBugs | module:root |
| | Found reliance on default encoding in org.apache.tez.client.registry.zookeeper.ZkAMRegistryClient.getAMRecord(ChildData):in org.apache.tez.client.registry.zookeeper.ZkAMRegistryClient.getAMRecord(ChildData): new String(byte[]) At ZkAMRegistryClient.java:[line 101] |
| | Shared primitive variable "amPort" in one thread may not yield the value of the most recent write from another thread At ZkFrameworkClient.java:one thread may not yield the value of the most recent write from another thread At ZkFrameworkClient.java:[line 109] |
| | Shared primitive variable "isRunning" in one thread may not yield the value of the most recent write from another thread At ZkFrameworkClient.java:one thread may not yield the value of the most recent write from another thread At ZkFrameworkClient.java:[line 65] |
| | Shared primitive variable "isRunning" in one thread may not yield the value of the most recent write from another thread At ZkFrameworkClient.java:one thread may not yield the value of the most recent write from another thread At ZkFrameworkClient.java:[line 69] |
| | org.apache.tez.client.registry.zookeeper.ZkFrameworkClient.init(TezConfiguration) may expose internal representation by storing an externally mutable object into ZkFrameworkClient.tezConf At ZkFrameworkClient.java:by storing an externally mutable object into ZkFrameworkClient.tezConf At ZkFrameworkClient.java:[line 48] |
| | Unread field:ZkFrameworkClient.java:[line 48] |
| | new org.apache.tez.dag.api.client.DAGClientServer(DAGClientHandler, ApplicationAttemptId, FileSystem) may expose internal representation by storing an externally mutable object into DAGClientServer.applicationAttemptId At DAGClientServer.java:expose internal representation by storing an externally mutable object into DAGClientServer.applicationAttemptId At DAGClientServer.java:[line 63] |
| | Found reliance on default encoding in org.apache.tez.dag.api.client.registry.zookeeper.ZkAMRegistry.add(AMRecord):in org.apache.tez.dag.api.client.registry.zookeeper.ZkAMRegistry.add(AMRecord): String.getBytes() At ZkAMRegistry.java:[line 99] |
| | Found reliance on default encoding in org.apache.tez.dag.app.DAGAppMaster.getConfigurationProtoFromText():in org.apache.tez.dag.app.DAGAppMaster.getConfigurationProtoFromText(): new java.io.InputStreamReader(InputStream) At DAGAppMaster.java:[line 2889] |
| | Found reliance on default encoding in org.apache.tez.dag.app.DAGAppMaster.getConfigurationProtoFromText():in org.apache.tez.dag.app.DAGAppMaster.getConfigurationProtoFromText(): String.getBytes() At DAGAppMaster.java:[line 2897] |
| | org.apache.tez.dag.app.DAGAppMaster.amExts isn't final but should be At DAGAppMaster.java:be At DAGAppMaster.java:[line 237] |
| | org.apache.tez.dag.app.DAGAppMaster.getConfigurationProtoFromText() may fail to close stream At DAGAppMaster.java:stream At DAGAppMaster.java:[line 2889] |
| Failed junit tests | tez.client.TestTezClientUtils |
| | tez.dag.api.TestTezConfiguration |
| | tez.dag.app.rm.TestTaskScheduler |
| | tez.client.TestTezClientUtils |
| | tez.dag.api.TestTezConfiguration |
| | tez.dag.app.rm.TestTaskScheduler |
| Subsystem | Report/Notes |
|----------:|:-------------|
| Docker | ClientAPI=1.51 ServerAPI=1.51 base: https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/1/artifact/out/Dockerfile |
| GITHUB PR | https://github.com/apache/tez/pull/427 |
| Optional Tests | dupname asflicense javac javadoc unit codespell detsecrets xmllint compile spotbugs checkstyle |
| uname | Linux d0a6e9d870ce 5.15.0-152-generic #162-Ubuntu SMP Wed Jul 23 09:48:42 UTC 2025 x86_64 x86_64 x86_64 GNU/Linux |
| Build tool | maven |
| Personality | /home/jenkins/jenkins-home/workspace/tez-multibranch_PR-427/src/.yetus/personality.sh |
| git revision | master / 4702ed68f755a9a359471bbaf94a3b31ef61c202 |
| Default Java | Ubuntu-21.0.8+9-Ubuntu-0ubuntu124.04.1 |
| Test Results | https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/1/testReport/ |
| Max. process+thread count | 1412 (vs. ulimit of 5500) |
| modules | C: tez-api tez-common tez-runtime-library tez-mapreduce tez-examples tez-dag . U: . |
| Console output | https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/1/console |
| versions | git=2.43.0 maven=3.8.7 spotbugs=4.9.3 codespell=2.0.0 |
| Powered by | Apache Yetus 0.15.1 https://yetus.apache.org |
This message was automatically generated.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] TEZ-4007: Introduce AmExtensions and Zookeeper-based FrameworkServices [tez]
Posted by "abstractdog (via GitHub)" <gi...@apache.org>.
abstractdog commented on code in PR #427:
URL: https://github.com/apache/tez/pull/427#discussion_r2549577799
##########
tez-api/src/main/java/org/apache/tez/client/TezClient.java:
##########
@@ -484,7 +484,9 @@ public synchronized TezClient getClient(ApplicationId appId) throws TezException
}
private void startFrameworkClient() {
- frameworkClient = createFrameworkClient();
+ if (frameworkClient == null) {
+ frameworkClient = createFrameworkClient();
+ }
Review Comment:
right, if we want to make it idempotent, it should instead be:
```
if (frameworkClient != null) {
return;
}
frameworkClient = createFrameworkClient();
frameworkClient.init(amConfig.getTezConfiguration());
frameworkClient.start();
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] TEZ-4007: Introduce AmExtensions and Zookeeper-based FrameworkServices [tez]
Posted by "abstractdog (via GitHub)" <gi...@apache.org>.
abstractdog commented on code in PR #427:
URL: https://github.com/apache/tez/pull/427#discussion_r2549694323
##########
tez-api/src/main/java/org/apache/tez/client/registry/AMRegistry.java:
##########
@@ -0,0 +1,104 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.client.registry;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.service.ServiceStateException;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Base class for {@code AMRegistry} implementations.
+ *
+ * <p>The specific implementation is configured via the
+ * {@code tez.am.registry.class} property.</p>
+ *
+ * <p>Implementations are expected to provide appropriate service lifecycle
+ * behavior, including:
+ * <ul>
+ * <li>{@code init}</li>
+ * <li>{@code serviceStart}</li>
+ * <li>{@code serviceStop}</li>
+ * </ul>
+ * </p>
+ *
+ * <p>{@code init} and {@code serviceStart} are invoked during
+ * {@code DAGAppMaster.serviceInit()}, while {@code serviceStop} is called
+ * when {@code DAGAppMaster} shuts down.</p>
+ */
+public abstract class AMRegistry extends AbstractService {
+
+ private static final Logger LOG = LoggerFactory.getLogger(AMRegistry.class);
+ private List<AMRecord> amRecords = new ArrayList<>();
Review Comment:
also, there is a private amRecords in the ZkAMRegistry subclass, which makes it really confusing, let me cleanup this codepath
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] TEZ-4007: Introduce AmExtensions and Zookeeper-based FrameworkServices [tez]
Posted by "abstractdog (via GitHub)" <gi...@apache.org>.
abstractdog commented on code in PR #427:
URL: https://github.com/apache/tez/pull/427#discussion_r2549739970
##########
tez-api/src/main/java/org/apache/tez/client/registry/AMRegistryClient.java:
##########
@@ -0,0 +1,98 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.client.registry;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Client-side interface for discovering Application Master (AM) instances
+ * registered in the AM registry.
+ *
+ * <p>Implementations are responsible for locating AM endpoints and returning
+ * their metadata. This API is used by client components to discover running
+ * Tez AMs.</p>
+ *
+ * <p>Listeners may be registered to receive notifications when AM records
+ * appear or are removed.</p>
+ */
+public abstract class AMRegistryClient implements Closeable {
+
+ private final List<AMRegistryClientListener> listeners = new ArrayList<>();
+
+ /**
+ * Returns the current set of registered listeners.
+ *
+ * @return a mutable list of listeners
+ */
+ protected List<AMRegistryClientListener> getListeners() {
+ return listeners;
+ }
+
+ /**
+ * Lookup AM metadata for the given application ID.
+ *
+ * @param appId the application ID
+ * @return the AM record if found, otherwise {@code null}
+ * @throws IOException if the lookup fails
+ */
+ public abstract AMRecord getRecord(String appId) throws IOException;
+
+ /**
+ * Retrieve all AM records known in the registry.
+ *
+ * @return a list of AM records (possibly empty)
+ * @throws IOException if the fetch fails
+ */
+ public abstract List<AMRecord> getAllRecords() throws IOException;
+
+ /**
+ * Register a listener for AM registry events.
+ * The listener will be notified when AM records are added or removed.
+ *
+ * @param listener the listener to add
+ */
+ public synchronized void addListener(AMRegistryClientListener listener) {
+ listeners.add(listener);
+ }
+
+ /**
+ * Notify listeners of a newly added AM record.
+ *
+ * @param record the added AM record
+ */
+ protected synchronized void notifyOnAdded(AMRecord record) {
+ for (AMRegistryClientListener listener : listeners) {
+ listener.onAdd(record);
+ }
Review Comment:
yeah, same applies to removal
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] TEZ-4007: Introduce AmExtensions and Zookeeper-based FrameworkServices [tez]
Posted by "abstractdog (via GitHub)" <gi...@apache.org>.
abstractdog commented on code in PR #427:
URL: https://github.com/apache/tez/pull/427#discussion_r2551047023
##########
tez-api/src/main/java/org/apache/tez/client/registry/zookeeper/ZkAMRegistryClient.java:
##########
@@ -0,0 +1,205 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.client.registry.zookeeper;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
+
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.imps.CuratorFrameworkState;
+import org.apache.curator.framework.recipes.cache.ChildData;
+import org.apache.curator.framework.recipes.cache.TreeCache;
+import org.apache.curator.framework.recipes.cache.TreeCacheEvent;
+import org.apache.curator.framework.recipes.cache.TreeCacheListener;
+import org.apache.curator.shaded.com.google.common.base.Charsets;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.registry.client.binding.RegistryUtils;
+import org.apache.hadoop.registry.client.types.ServiceRecord;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.tez.client.registry.AMRecord;
+import org.apache.tez.client.registry.AMRegistryClient;
+import org.apache.tez.client.registry.AMRegistryClientListener;
+import org.apache.tez.dag.api.TezConfiguration;
+
+import com.fasterxml.jackson.core.JsonParseException;
+import com.google.common.base.Preconditions;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Curator/Zookeeper implementation of {@link AMRegistryClient}.
+ */
+@InterfaceAudience.Public
+public final class ZkAMRegistryClient extends AMRegistryClient {
+ private static final Logger LOG = LoggerFactory.getLogger(ZkAMRegistryClient.class);
+ private static final Map<String, ZkAMRegistryClient> INSTANCES = new HashMap<>();
+
+ private final Configuration conf;
+ //Cache of known AMs
+ private final ConcurrentHashMap<String, AMRecord> amRecordCache = new ConcurrentHashMap<>();
+ private CuratorFramework client;
+
+ private ZkAMRegistryClient(final Configuration conf) {
+ this.conf = conf;
+ }
+
+ public static synchronized ZkAMRegistryClient getClient(final Configuration conf) {
+ String namespace = conf.get(TezConfiguration.TEZ_AM_REGISTRY_NAMESPACE);
+ ZkAMRegistryClient registry = INSTANCES.get(namespace);
+ if (registry == null) {
+ registry = new ZkAMRegistryClient(conf);
+ INSTANCES.put(namespace, registry);
+ }
+ LOG.info("Returning tez AM registry ({}) for namespace '{}'", System.identityHashCode(registry), namespace);
+ return registry;
+ }
+
+ /**
+ * Deserializes a {@link ServiceRecord} from ZooKeeper data and converts it into an {@link AMRecord}
+ * for caching.
+ *
+ * @param childData the ZooKeeper node data containing a serialized {@link ServiceRecord}
+ * @return an {@link AMRecord} constructed from the deserialized {@link ServiceRecord}, or {@code null}
+ * if no data is present
+ * @throws IOException if the data cannot be deserialized into a {@link ServiceRecord}
+ */
+ public static AMRecord getAMRecord(final ChildData childData) throws IOException {
+ // not a leaf path. Only leaf path contains AMRecord
+ if (!childData.getPath().contains(ApplicationId.appIdStrPrefix)) {
+ return null;
+ }
+ byte[] data = childData.getData();
+ // only the path appeared, there is no data yet
+ if (data.length == 0) {
+ return null;
+ }
+ String value = new String(data, Charsets.UTF_8);
+ try {
+ RegistryUtils.ServiceRecordMarshal marshal = new RegistryUtils.ServiceRecordMarshal();
+ ServiceRecord serviceRecord = marshal.fromJson(value);
+ return new AMRecord(serviceRecord);
+ } catch (JsonParseException e) {
+ //Not a json AMRecord (SRV), could be some other data.
+ LOG.warn("Non-json data received while de-serializing AMRecord: {}. Ignoring...", value);
+ return null;
+ }
+ }
+
+ public void start() throws Exception {
+ ZkConfig zkConf = new ZkConfig(this.conf);
+ client = zkConf.createCuratorFramework();
+ final TreeCache cache = new TreeCache(client, zkConf.getZkNamespace());
+ client.start();
+ cache.start();
Review Comment:
currently nowhere :) good catch! taking care of it
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] TEZ-4007: Introduce AmExtensions and Zookeeper-based FrameworkServices [tez]
Posted by "abstractdog (via GitHub)" <gi...@apache.org>.
abstractdog commented on code in PR #427:
URL: https://github.com/apache/tez/pull/427#discussion_r2564918638
##########
tez-api/src/main/java/org/apache/tez/frameworkplugins/FrameworkUtils.java:
##########
@@ -0,0 +1,96 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tez.frameworkplugins;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.ServiceLoader;
+
+import javax.annotation.Nullable;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.tez.common.ReflectionUtils;
+import org.apache.tez.dag.api.TezConfiguration;
+import org.apache.tez.dag.api.TezConstants;
+import org.apache.tez.dag.api.TezReflectionException;
+
+public final class FrameworkUtils {
+
+ private FrameworkUtils() {}
+
+ /*
+ Searches for a FrameworkService provider which implements a target interface.
+ The interface should be either ClientFrameworkService or ServerFrameworkService.
+ Depending on which interface is used, either the client or server class of a
+ matching FrameworkMode will be used as the implementation.
+
+ NOTE: Layering of FrameworkServices in a decorator-style is currently not supported
+
+ An implementation is searched in the following order:
+ 1. If conf is not null and the parameter TEZ_FRAMEWORK_MODE is set:
+ the value of TEZ_FRAMEWORK_MODE from the conf will be used
+ 2. If conf is null or the parameter TEZ_FRAMEWORK_MODE is not set
+ and the environment var TEZ_FRAMEWORK_MODE is not empty:
+ the value of the environment var will be used
+ 3. Otherwise:
+ the default java.util.ServiceLoader behavior will be used,
+ i.e. the implementation classname should appear in a file on the classpath at the location
+ META-INF/services/org.apache.tez.frameworkplugins.ClientFrameworkService
+ or META-INF/services/org.apache.tez.frameworkplugins.ServerFrameworkService
+ */
+ public static <T extends FrameworkService> T get(Class<T> interfaze, @Nullable Configuration conf) {
+ try {
+ if ((conf != null) && (conf.get(TezConfiguration.TEZ_FRAMEWORK_MODE) != null)) {
+ return getByMode(interfaze, conf.get(TezConfiguration.TEZ_FRAMEWORK_MODE));
+ } else if (System.getenv(TezConstants.TEZ_FRAMEWORK_MODE) != null) {
+ return getByMode(interfaze, System.getenv(TezConstants.TEZ_FRAMEWORK_MODE));
+ } else {
+ return getByServiceLoader(interfaze);
+ }
+ } catch (TezReflectionException e) {
+ throw new RuntimeException("Failed to load framework service for interface: " + interfaze.getName(), e);
+ }
+ }
+
+ private static <T extends FrameworkService> T getByServiceLoader(Class<T> interfaze) {
+ List<T> services = new ArrayList<>();
+ ServiceLoader<T> frameworkService = ServiceLoader.load(interfaze);
+ for (T service : frameworkService) {
+ services.add(service);
+ }
+ if (services.isEmpty()) {
+ return null;
Review Comment:
finally, I decided to remove getByServiceLoader, and make the users use it by getMode properly
let's keep it simple
also FrameworkUtils.get can be provided with a default class parameter to handle this case easily
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] TEZ-4007: Introduce AmExtensions and Zookeeper-based FrameworkServices [tez]
Posted by "abstractdog (via GitHub)" <gi...@apache.org>.
abstractdog commented on code in PR #427:
URL: https://github.com/apache/tez/pull/427#discussion_r2565205502
##########
tez-dag/src/main/java/org/apache/tez/dag/api/client/registry/zookeeper/ZkAMRegistry.java:
##########
@@ -0,0 +1,175 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.api.client.registry.zookeeper;
+
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.curator.RetryLoop;
+import org.apache.curator.RetryPolicy;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.registry.client.binding.RegistryUtils;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.tez.client.registry.AMRecord;
+import org.apache.tez.client.registry.AMRegistry;
+import org.apache.tez.client.registry.zookeeper.ZkConfig;
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.data.Stat;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Curator/Zookeeper impl of AMRegistry (for internal use only)
+ * Clients should use org.apache.tez.dag.api.client.registry.zookeeper.ZkAMRegistryClient instead.
+ */
+@InterfaceAudience.Private
+public class ZkAMRegistry extends AMRegistry {
+
+ private static final Logger LOG = LoggerFactory.getLogger(ZkAMRegistry.class);
+
+ private final List<AMRecord> amRecords = new ArrayList<>();
+ private final String externalId;
+
+ private CuratorFramework client = null;
+ private String namespace = null;
+ private ZkConfig zkConfig = null;
+ private boolean started = false;
+
+ public ZkAMRegistry(String externalId) {
+ super("ZkAMRegistry");
+ this.externalId = externalId;
+ }
+
+ @Override
+ public void serviceInit(Configuration conf) {
+ if (zkConfig == null) {
+ zkConfig = new ZkConfig(conf);
+ this.client = zkConfig.createCuratorFramework();
+ this.namespace = zkConfig.getZkNamespace();
+ LOG.info("AMRegistryZkImpl initialized");
+ }
+ }
+
+ @Override
+ public void serviceStart() throws Exception {
+ if (!started) {
+ client.start();
+ started = true;
+ LOG.info("AMRegistryZkImpl started");
+ }
+ }
+
+ //Deletes from Zookeeper AMRecords that were added by this instance
+ @Override
+ public void serviceStop() throws Exception {
+ List<AMRecord> records = new ArrayList<>(amRecords);
+ for (AMRecord amRecord : records) {
+ remove(amRecord);
+ }
+ client.close();
+ LOG.info("AMRegistryZkImpl shutdown");
+ }
+
+ //Serialize AMRecord to ServiceRecord and deliver the JSON bytes to
+ //zkNode at the path: <TEZ_AM_REGISTRY_NAMESPACE>/<appId>
+ @Override
+ public void add(AMRecord server) throws Exception {
+ RegistryUtils.ServiceRecordMarshal marshal = new RegistryUtils.ServiceRecordMarshal();
+ String json = marshal.toJson(server.toServiceRecord());
+ try {
+ final String path = namespace + "/" + server.getApplicationId().toString();
+ client.setData().forPath(path, json.getBytes(StandardCharsets.UTF_8));
+ LOG.info("Added AMRecord to zkpath {}", path);
+ } catch (KeeperException.NoNodeException nne) {
+ client.create().creatingParentContainersIfNeeded().withMode(CreateMode.EPHEMERAL)
Review Comment:
EPHEMERAL looks good
important difference between EPHEMERAL and PERSISTENT is that:
```
/**
* The znode will be deleted upon the client's disconnect.
*/
EPHEMERAL(1, true, false, false, false),
```
vs
```
/**
* The znode will not be automatically deleted upon client's disconnect.
*/
PERSISTENT(0, false, false, false, false),
```
in our usecase, an AM registers itself to ZK, so when AM is about to shutdown and client disconnects, it sounds quite convenient to get the corresponding nodes automatically deleted (node deletion -> update on client side about AM has been removed)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] [DRAFT] TEZ-4007: Zookeeper based FrameworkServices and AmExtensions (3/3) + refactor [tez]
Posted by "tez-yetus (via GitHub)" <gi...@apache.org>.
tez-yetus commented on PR #427:
URL: https://github.com/apache/tez/pull/427#issuecomment-3433178282
:broken_heart: **-1 overall**
| Vote | Subsystem | Runtime | Logfile | Comment |
|:----:|----------:|--------:|:--------:|:-------:|
| +0 :ok: | reexec | 14m 3s | | Docker mode activated. |
|||| _ Prechecks _ |
| +1 :green_heart: | dupname | 0m 0s | | No case conflicting files found. |
| +0 :ok: | detsecrets | 0m 0s | | detect-secrets was not available. |
| +0 :ok: | xmllint | 0m 0s | | xmllint was not available. |
| +1 :green_heart: | @author | 0m 0s | | The patch does not contain any @author tags. |
| +1 :green_heart: | test4tests | 0m 0s | | The patch appears to include 12 new or modified test files. |
|||| _ master Compile Tests _ |
| +0 :ok: | mvndep | 3m 1s | | Maven dependency ordering for branch |
| +1 :green_heart: | mvninstall | 8m 29s | | master passed |
| +1 :green_heart: | compile | 3m 14s | | master passed |
| +1 :green_heart: | checkstyle | 2m 0s | | master passed |
| +1 :green_heart: | javadoc | 2m 33s | | master passed |
| +0 :ok: | spotbugs | 1m 41s | | tez-api in master has 610 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 0m 28s | | tez-common in master has 13 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 0m 36s | | tez-runtime-library in master has 241 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 0m 31s | | tez-mapreduce in master has 129 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 0m 22s | | tez-examples in master has 2 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 0m 53s | | tez-dag in master has 785 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 3m 41s | | root in master has 2066 extant spotbugs warnings. |
|||| _ Patch Compile Tests _ |
| +0 :ok: | mvndep | 0m 11s | | Maven dependency ordering for patch |
| +1 :green_heart: | mvninstall | 3m 50s | | the patch passed |
| -1 :x: | codespell | 0m 27s | [/results-codespell.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/16/artifact/out/results-codespell.txt) | The patch generated 1 new + 30 unchanged - 0 fixed = 31 total (was 30) |
| +1 :green_heart: | compile | 3m 23s | | the patch passed |
| +1 :green_heart: | javac | 3m 23s | | the patch passed |
| -1 :x: | blanks | 0m 0s | [/blanks-eol.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/16/artifact/out/blanks-eol.txt) | The patch has 10 line(s) that end in blanks. Use git apply --whitespace=fix <<patch_file>>. Refer https://git-scm.com/docs/git-apply |
| -0 :warning: | checkstyle | 0m 15s | [/results-checkstyle-tez-api.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/16/artifact/out/results-checkstyle-tez-api.txt) | tez-api: The patch generated 3 new + 134 unchanged - 11 fixed = 137 total (was 145) |
| -0 :warning: | checkstyle | 0m 13s | [/results-checkstyle-tez-runtime-library.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/16/artifact/out/results-checkstyle-tez-runtime-library.txt) | tez-runtime-library: The patch generated 4 new + 0 unchanged - 0 fixed = 4 total (was 0) |
| -0 :warning: | checkstyle | 0m 11s | [/results-checkstyle-tez-examples.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/16/artifact/out/results-checkstyle-tez-examples.txt) | tez-examples: The patch generated 21 new + 10 unchanged - 2 fixed = 31 total (was 12) |
| -0 :warning: | checkstyle | 0m 16s | [/results-checkstyle-tez-dag.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/16/artifact/out/results-checkstyle-tez-dag.txt) | tez-dag: The patch generated 2 new + 818 unchanged - 2 fixed = 820 total (was 820) |
| -0 :warning: | checkstyle | 0m 23s | [/results-checkstyle-root.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/16/artifact/out/results-checkstyle-root.txt) | root: The patch generated 30 new + 964 unchanged - 15 fixed = 994 total (was 979) |
| -1 :x: | javadoc | 0m 25s | [/results-javadoc-javadoc-tez-api.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/16/artifact/out/results-javadoc-javadoc-tez-api.txt) | tez-api generated 1 new + 10 unchanged - 0 fixed = 11 total (was 10) |
| -1 :x: | javadoc | 0m 46s | [/results-javadoc-javadoc-root.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/16/artifact/out/results-javadoc-javadoc-root.txt) | root generated 1 new + 19 unchanged - 0 fixed = 20 total (was 19) |
| -1 :x: | spotbugs | 0m 56s | [/new-spotbugs-tez-api.html](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/16/artifact/out/new-spotbugs-tez-api.html) | tez-api generated 1 new + 610 unchanged - 0 fixed = 611 total (was 610) |
| -1 :x: | spotbugs | 3m 58s | [/new-spotbugs-root.html](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/16/artifact/out/new-spotbugs-root.html) | root generated 1 new + 2066 unchanged - 0 fixed = 2067 total (was 2066) |
|||| _ Other Tests _ |
| +1 :green_heart: | unit | 2m 3s | | tez-api in the patch passed. |
| +1 :green_heart: | unit | 0m 26s | | tez-common in the patch passed. |
| +1 :green_heart: | unit | 4m 11s | | tez-runtime-library in the patch passed. |
| +1 :green_heart: | unit | 1m 2s | | tez-mapreduce in the patch passed. |
| +1 :green_heart: | unit | 0m 20s | | tez-examples in the patch passed. |
| -1 :x: | unit | 4m 8s | [/patch-unit-tez-dag.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/16/artifact/out/patch-unit-tez-dag.txt) | tez-dag in the patch passed. |
| -1 :x: | unit | 59m 0s | [/patch-unit-root.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/16/artifact/out/patch-unit-root.txt) | root in the patch passed. |
| -1 :x: | asflicense | 1m 50s | [/results-asflicense.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/16/artifact/out/results-asflicense.txt) | The patch generated 6 ASF License warnings. |
| | | 138m 28s | | |
| Reason | Tests |
|-------:|:------|
| SpotBugs | module:tez-api |
| | new org.apache.tez.client.TezYarnClient(YarnClient) may expose internal representation by storing an externally mutable object into TezYarnClient.yarnClient At TezYarnClient.java:representation by storing an externally mutable object into TezYarnClient.yarnClient At TezYarnClient.java:[line 47] |
| SpotBugs | module:root |
| | new org.apache.tez.client.TezYarnClient(YarnClient) may expose internal representation by storing an externally mutable object into TezYarnClient.yarnClient At TezYarnClient.java:representation by storing an externally mutable object into TezYarnClient.yarnClient At TezYarnClient.java:[line 47] |
| Failed junit tests | tez.dag.app.dag.impl.TestDAGRecovery |
| | tez.dag.app.dag.impl.TestDAGRecovery |
| Subsystem | Report/Notes |
|----------:|:-------------|
| Docker | ClientAPI=1.51 ServerAPI=1.51 base: https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/16/artifact/out/Dockerfile |
| GITHUB PR | https://github.com/apache/tez/pull/427 |
| Optional Tests | dupname asflicense javac javadoc unit codespell detsecrets xmllint compile spotbugs checkstyle |
| uname | Linux 3fc7460ddabe 5.15.0-156-generic #166-Ubuntu SMP Sat Aug 9 00:02:46 UTC 2025 x86_64 x86_64 x86_64 GNU/Linux |
| Build tool | maven |
| Personality | /home/jenkins/jenkins-home/workspace/tez-multibranch_PR-427/src/.yetus/personality.sh |
| git revision | master / eec88ab640c3f5718633bf8df12e95d93e89da1e |
| Default Java | Ubuntu-21.0.8+9-Ubuntu-0ubuntu124.04.1 |
| Test Results | https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/16/testReport/ |
| Max. process+thread count | 1411 (vs. ulimit of 5500) |
| modules | C: tez-api tez-common tez-runtime-library tez-mapreduce tez-examples tez-dag . U: . |
| Console output | https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/16/console |
| versions | git=2.43.0 maven=3.8.7 spotbugs=4.9.3 codespell=2.0.0 |
| Powered by | Apache Yetus 0.15.1 https://yetus.apache.org |
This message was automatically generated.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] TEZ-4007: Introduce AmExtensions and Zookeeper-based FrameworkServices [tez]
Posted by "abstractdog (via GitHub)" <gi...@apache.org>.
abstractdog commented on code in PR #427:
URL: https://github.com/apache/tez/pull/427#discussion_r2672066145
##########
tez-api/src/main/java/org/apache/tez/client/registry/zookeeper/ZkAMRegistryClient.java:
##########
@@ -0,0 +1,209 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.client.registry.zookeeper;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.imps.CuratorFrameworkState;
+import org.apache.curator.framework.recipes.cache.ChildData;
+import org.apache.curator.framework.recipes.cache.TreeCache;
+import org.apache.curator.framework.recipes.cache.TreeCacheEvent;
+import org.apache.curator.framework.recipes.cache.TreeCacheListener;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.registry.client.types.ServiceRecord;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.tez.client.registry.AMRecord;
+import org.apache.tez.client.registry.AMRegistryClient;
+import org.apache.tez.client.registry.AMRegistryUtils;
+import org.apache.tez.dag.api.TezConfiguration;
+
+import com.fasterxml.jackson.core.JsonParseException;
+import com.google.common.base.Preconditions;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Curator/Zookeeper implementation of {@link AMRegistryClient}.
+ */
+@InterfaceAudience.Public
+public final class ZkAMRegistryClient extends AMRegistryClient {
+ private static final Logger LOG = LoggerFactory.getLogger(ZkAMRegistryClient.class);
+ private static final Map<String, ZkAMRegistryClient> INSTANCES = new HashMap<>();
+
+ private final Configuration conf;
+ // Cache of known AMs
+ private final ConcurrentHashMap<ApplicationId, AMRecord> amRecordCache = new ConcurrentHashMap<>();
+
+ private CuratorFramework client;
+ private TreeCache cache;
+ private ZkRegistryListener listener;
+
+ private ZkAMRegistryClient(final Configuration conf) {
+ this.conf = conf;
+ }
+
+ public static synchronized ZkAMRegistryClient getClient(final Configuration conf) {
+ String namespace = conf.get(TezConfiguration.TEZ_AM_REGISTRY_NAMESPACE);
+ ZkAMRegistryClient registry = INSTANCES.get(namespace);
+ if (registry == null) {
+ registry = new ZkAMRegistryClient(conf);
+ INSTANCES.put(namespace, registry);
+ }
+ LOG.info("Returning tez AM registry ({}) for namespace '{}'", System.identityHashCode(registry), namespace);
+ return registry;
+ }
+
+ /**
+ * Deserializes a {@link ServiceRecord} from ZooKeeper data and converts it into an {@link AMRecord}
+ * for caching.
+ *
+ * @param childData the ZooKeeper node data containing a serialized {@link ServiceRecord}
+ * @return an {@link AMRecord} constructed from the deserialized {@link ServiceRecord}, or {@code null}
+ * if no data is present
+ * @throws IOException if the data cannot be deserialized into a {@link ServiceRecord}
+ */
+ public static AMRecord getAMRecord(final ChildData childData) throws IOException {
+ // Not a leaf path. Only leaf path contains AMRecord.
+ if (!childData.getPath().contains(ApplicationId.appIdStrPrefix)) {
+ return null;
+ }
+ byte[] data = childData.getData();
+ // only the path appeared, there is no data yet
+ if (data.length == 0) {
+ return null;
+ }
+ String value = new String(data, StandardCharsets.UTF_8);
+ try {
+ return AMRegistryUtils.jsonStringToRecord(value);
+ } catch (JsonParseException e) {
+ //Not a json AMRecord (SRV), could be some other data.
+ LOG.warn("Non-json data received while de-serializing AMRecord: {}. Ignoring...", value);
+ return null;
+ }
+ }
+
+ public void start() throws Exception {
+ ZkConfig zkConf = new ZkConfig(this.conf);
+ client = zkConf.createCuratorFramework();
+ cache = new TreeCache(client, zkConf.getZkNamespace());
+ client.start();
+ cache.start();
+ listener = new ZkRegistryListener();
+ cache.getListenable().addListener(listener);
+ }
+
+ @Override
+ public AMRecord getRecord(ApplicationId appId) {
+ AMRecord rec = amRecordCache.get(appId);
+ // Return a copy.
+ return rec == null ? null : new AMRecord(rec);
+ }
+
+ @Override
+ public List<AMRecord> getAllRecords() {
+ return amRecordCache.values().stream().map(AMRecord::new).collect(Collectors.toList());
+ }
+
+ @Override
+ public void close() {
+ IOUtils.closeQuietly(cache);
+ IOUtils.closeQuietly(client);
Review Comment:
oh, this is a good catch that reveals a lot about how vague the current implementation is to me
INSTANCE is a static collection, so its lifecycle is not really clear, other than a zookeeper registry client for a specific namespace is shared across the whole JVM, but when how its closure is supposed to happen, is not clear, need to investigate this (I mean, it works in the downstream distro, but cannot tell how an why)
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] TEZ-4007: Introduce AmExtensions and Zookeeper-based FrameworkServices [tez]
Posted by "tez-yetus (via GitHub)" <gi...@apache.org>.
tez-yetus commented on PR #427:
URL: https://github.com/apache/tez/pull/427#issuecomment-3594118860
:confetti_ball: **+1 overall**
| Vote | Subsystem | Runtime | Logfile | Comment |
|:----:|----------:|--------:|:--------:|:-------:|
| +0 :ok: | reexec | 0m 30s | | Docker mode activated. |
|||| _ Prechecks _ |
| +1 :green_heart: | dupname | 0m 0s | | No case conflicting files found. |
| +0 :ok: | detsecrets | 0m 0s | | detect-secrets was not available. |
| +0 :ok: | xmllint | 0m 0s | | xmllint was not available. |
| +1 :green_heart: | @author | 0m 0s | | The patch does not contain any @author tags. |
| +1 :green_heart: | test4tests | 0m 0s | | The patch appears to include 14 new or modified test files. |
|||| _ master Compile Tests _ |
| +0 :ok: | mvndep | 2m 38s | | Maven dependency ordering for branch |
| +1 :green_heart: | mvninstall | 10m 0s | | master passed |
| +1 :green_heart: | compile | 5m 22s | | master passed |
| +1 :green_heart: | checkstyle | 3m 18s | | master passed |
| +1 :green_heart: | javadoc | 4m 3s | | master passed |
| +0 :ok: | spotbugs | 2m 8s | | tez-api in master has 610 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 0m 46s | | tez-common in master has 13 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 1m 14s | | tez-runtime-library in master has 241 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 0m 44s | | tez-examples in master has 2 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 1m 46s | | tez-dag in master has 785 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 6m 58s | | root in master has 2066 extant spotbugs warnings. |
| -0 :warning: | patch | 7m 43s | | Used diff version of patch file. Binary files and potentially other changes not applied. Please rebase and squash commits if necessary. |
|||| _ Patch Compile Tests _ |
| +0 :ok: | mvndep | 0m 17s | | Maven dependency ordering for patch |
| +1 :green_heart: | mvninstall | 6m 32s | | the patch passed |
| +1 :green_heart: | codespell | 0m 50s | | No new issues. |
| +1 :green_heart: | compile | 5m 25s | | the patch passed |
| +1 :green_heart: | javac | 5m 25s | | the patch passed |
| +1 :green_heart: | blanks | 0m 0s | | The patch has no blanks issues. |
| +1 :green_heart: | checkstyle | 0m 26s | | tez-api: The patch generated 0 new + 118 unchanged - 3 fixed = 118 total (was 121) |
| +1 :green_heart: | checkstyle | 0m 23s | | The patch passed checkstyle in tez-common |
| +1 :green_heart: | checkstyle | 0m 28s | | The patch passed checkstyle in tez-runtime-library |
| +1 :green_heart: | checkstyle | 0m 22s | | tez-examples: The patch generated 0 new + 10 unchanged - 1 fixed = 10 total (was 11) |
| -0 :warning: | checkstyle | 0m 34s | [/results-checkstyle-tez-dag.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/31/artifact/out/results-checkstyle-tez-dag.txt) | tez-dag: The patch generated 1 new + 610 unchanged - 2 fixed = 611 total (was 612) |
| -0 :warning: | checkstyle | 0m 46s | [/results-checkstyle-root.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/31/artifact/out/results-checkstyle-root.txt) | root: The patch generated 1 new + 740 unchanged - 6 fixed = 741 total (was 746) |
| +1 :green_heart: | javadoc | 3m 57s | | the patch passed |
| +1 :green_heart: | spotbugs | 14m 14s | | the patch passed |
|||| _ Other Tests _ |
| +1 :green_heart: | unit | 2m 36s | | tez-api in the patch passed. |
| +1 :green_heart: | unit | 0m 44s | | tez-common in the patch passed. |
| +1 :green_heart: | unit | 6m 9s | | tez-runtime-library in the patch passed. |
| +1 :green_heart: | unit | 0m 31s | | tez-examples in the patch passed. |
| +1 :green_heart: | unit | 6m 2s | | tez-dag in the patch passed. |
| +1 :green_heart: | unit | 72m 14s | | root in the patch passed. |
| +1 :green_heart: | asflicense | 2m 42s | | The patch does not generate ASF License warnings. |
| | | 169m 38s | | |
| Subsystem | Report/Notes |
|----------:|:-------------|
| Docker | ClientAPI=1.52 ServerAPI=1.52 base: https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/31/artifact/out/Dockerfile |
| GITHUB PR | https://github.com/apache/tez/pull/427 |
| Optional Tests | dupname asflicense javac javadoc unit codespell detsecrets xmllint compile spotbugs checkstyle |
| uname | Linux 7c7c4bb63872 5.15.0-161-generic #171-Ubuntu SMP Sat Oct 11 08:17:01 UTC 2025 x86_64 x86_64 x86_64 GNU/Linux |
| Build tool | maven |
| Personality | /home/jenkins/jenkins-home/workspace/tez-multibranch_PR-427/src/.yetus/personality.sh |
| git revision | master / 045ff683afdd693ba2dc940581a297060c80ea01 |
| Default Java | Ubuntu-21.0.8+9-Ubuntu-0ubuntu124.04.1 |
| Test Results | https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/31/testReport/ |
| Max. process+thread count | 2109 (vs. ulimit of 5500) |
| modules | C: tez-api tez-common tez-runtime-library tez-examples tez-dag . U: . |
| Console output | https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/31/console |
| versions | git=2.43.0 maven=3.8.7 spotbugs=4.9.3 codespell=2.0.0 |
| Powered by | Apache Yetus 0.15.1 https://yetus.apache.org |
This message was automatically generated.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] [DRAFT] TEZ-4007: Zookeeper based FrameworkServices and AmExtensions (3/3) + refactor [tez]
Posted by "tez-yetus (via GitHub)" <gi...@apache.org>.
tez-yetus commented on PR #427:
URL: https://github.com/apache/tez/pull/427#issuecomment-3473071477
:broken_heart: **-1 overall**
| Vote | Subsystem | Runtime | Logfile | Comment |
|:----:|----------:|--------:|:--------:|:-------:|
| +0 :ok: | reexec | 23m 29s | | Docker mode activated. |
|||| _ Prechecks _ |
| +1 :green_heart: | dupname | 0m 1s | | No case conflicting files found. |
| +0 :ok: | detsecrets | 0m 0s | | detect-secrets was not available. |
| +0 :ok: | xmllint | 0m 0s | | xmllint was not available. |
| +1 :green_heart: | @author | 0m 0s | | The patch does not contain any @author tags. |
| +1 :green_heart: | test4tests | 0m 0s | | The patch appears to include 13 new or modified test files. |
|||| _ master Compile Tests _ |
| +0 :ok: | mvndep | 2m 38s | | Maven dependency ordering for branch |
| +1 :green_heart: | mvninstall | 9m 46s | | master passed |
| +1 :green_heart: | compile | 5m 56s | | master passed |
| +1 :green_heart: | checkstyle | 3m 45s | | master passed |
| +1 :green_heart: | javadoc | 4m 38s | | master passed |
| +0 :ok: | spotbugs | 2m 13s | | tez-api in master has 610 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 0m 46s | | tez-common in master has 13 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 1m 13s | | tez-runtime-library in master has 241 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 0m 56s | | tez-mapreduce in master has 129 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 0m 43s | | tez-examples in master has 2 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 1m 42s | | tez-dag in master has 785 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 6m 54s | | root in master has 2066 extant spotbugs warnings. |
|||| _ Patch Compile Tests _ |
| +0 :ok: | mvndep | 0m 17s | | Maven dependency ordering for patch |
| +1 :green_heart: | mvninstall | 6m 40s | | the patch passed |
| -1 :x: | codespell | 0m 52s | [/results-codespell.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/17/artifact/out/results-codespell.txt) | The patch generated 1 new + 30 unchanged - 0 fixed = 31 total (was 30) |
| +1 :green_heart: | compile | 5m 54s | | the patch passed |
| +1 :green_heart: | javac | 5m 54s | | the patch passed |
| -1 :x: | blanks | 0m 0s | [/blanks-eol.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/17/artifact/out/blanks-eol.txt) | The patch has 4 line(s) that end in blanks. Use git apply --whitespace=fix <<patch_file>>. Refer https://git-scm.com/docs/git-apply |
| +1 :green_heart: | checkstyle | 0m 26s | | tez-api: The patch generated 0 new + 134 unchanged - 11 fixed = 134 total (was 145) |
| +1 :green_heart: | checkstyle | 0m 23s | | The patch passed checkstyle in tez-common |
| -0 :warning: | checkstyle | 0m 27s | [/results-checkstyle-tez-runtime-library.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/17/artifact/out/results-checkstyle-tez-runtime-library.txt) | tez-runtime-library: The patch generated 4 new + 0 unchanged - 0 fixed = 4 total (was 0) |
| +1 :green_heart: | checkstyle | 0m 25s | | The patch passed checkstyle in tez-mapreduce |
| -0 :warning: | checkstyle | 0m 23s | [/results-checkstyle-tez-examples.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/17/artifact/out/results-checkstyle-tez-examples.txt) | tez-examples: The patch generated 19 new + 10 unchanged - 1 fixed = 29 total (was 11) |
| -0 :warning: | checkstyle | 0m 34s | [/results-checkstyle-tez-dag.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/17/artifact/out/results-checkstyle-tez-dag.txt) | tez-dag: The patch generated 1 new + 842 unchanged - 2 fixed = 843 total (was 844) |
| -0 :warning: | checkstyle | 0m 47s | [/results-checkstyle-root.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/17/artifact/out/results-checkstyle-root.txt) | root: The patch generated 24 new + 988 unchanged - 14 fixed = 1012 total (was 1002) |
| -1 :x: | javadoc | 0m 41s | [/results-javadoc-javadoc-tez-api.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/17/artifact/out/results-javadoc-javadoc-tez-api.txt) | tez-api generated 1 new + 10 unchanged - 0 fixed = 11 total (was 10) |
| -1 :x: | javadoc | 1m 20s | [/results-javadoc-javadoc-root.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/17/artifact/out/results-javadoc-javadoc-root.txt) | root generated 1 new + 19 unchanged - 0 fixed = 20 total (was 19) |
| +1 :green_heart: | spotbugs | 15m 21s | | the patch passed |
|||| _ Other Tests _ |
| +1 :green_heart: | unit | 2m 36s | | tez-api in the patch passed. |
| +1 :green_heart: | unit | 0m 43s | | tez-common in the patch passed. |
| +1 :green_heart: | unit | 6m 8s | | tez-runtime-library in the patch passed. |
| +1 :green_heart: | unit | 1m 35s | | tez-mapreduce in the patch passed. |
| +1 :green_heart: | unit | 0m 30s | | tez-examples in the patch passed. |
| -1 :x: | unit | 5m 24s | [/patch-unit-tez-dag.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/17/artifact/out/patch-unit-tez-dag.txt) | tez-dag in the patch passed. |
| -1 :x: | unit | 71m 41s | [/patch-unit-root.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/17/artifact/out/patch-unit-root.txt) | root in the patch passed. |
| -1 :x: | asflicense | 3m 11s | [/results-asflicense.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/17/artifact/out/results-asflicense.txt) | The patch generated 5 ASF License warnings. |
| | | 198m 30s | | |
| Reason | Tests |
|-------:|:------|
| Failed junit tests | tez.dag.history.recovery.TestRecoveryService |
| | tez.dag.history.recovery.TestRecoveryService |
| Subsystem | Report/Notes |
|----------:|:-------------|
| Docker | ClientAPI=1.51 ServerAPI=1.51 base: https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/17/artifact/out/Dockerfile |
| GITHUB PR | https://github.com/apache/tez/pull/427 |
| Optional Tests | dupname asflicense javac javadoc unit codespell detsecrets xmllint compile spotbugs checkstyle |
| uname | Linux 603558fed814 5.15.0-152-generic #162-Ubuntu SMP Wed Jul 23 09:48:42 UTC 2025 x86_64 x86_64 x86_64 GNU/Linux |
| Build tool | maven |
| Personality | /home/jenkins/jenkins-home/workspace/tez-multibranch_PR-427/src/.yetus/personality.sh |
| git revision | master / 6fb70019f2132fac27011ec41e6160fc0c94f3d1 |
| Default Java | Ubuntu-21.0.8+9-Ubuntu-0ubuntu124.04.1 |
| Test Results | https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/17/testReport/ |
| Max. process+thread count | 1366 (vs. ulimit of 5500) |
| modules | C: tez-api tez-common tez-runtime-library tez-mapreduce tez-examples tez-dag . U: . |
| Console output | https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/17/console |
| versions | git=2.43.0 maven=3.8.7 spotbugs=4.9.3 codespell=2.0.0 |
| Powered by | Apache Yetus 0.15.1 https://yetus.apache.org |
This message was automatically generated.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] TEZ-4007: Introduce AmExtensions and Zookeeper-based FrameworkServices [tez]
Posted by "tez-yetus (via GitHub)" <gi...@apache.org>.
tez-yetus commented on PR #427:
URL: https://github.com/apache/tez/pull/427#issuecomment-3726034661
:broken_heart: **-1 overall**
| Vote | Subsystem | Runtime | Logfile | Comment |
|:----:|----------:|--------:|:--------:|:-------:|
| +0 :ok: | reexec | 23m 41s | | Docker mode activated. |
|||| _ Prechecks _ |
| +1 :green_heart: | dupname | 0m 1s | | No case conflicting files found. |
| +0 :ok: | detsecrets | 0m 0s | | detect-secrets was not available. |
| +0 :ok: | xmllint | 0m 0s | | xmllint was not available. |
| +1 :green_heart: | @author | 0m 0s | | The patch does not contain any @author tags. |
| +1 :green_heart: | test4tests | 0m 0s | | The patch appears to include 14 new or modified test files. |
|||| _ master Compile Tests _ |
| +0 :ok: | mvndep | 2m 45s | | Maven dependency ordering for branch |
| +1 :green_heart: | mvninstall | 10m 1s | | master passed |
| +1 :green_heart: | compile | 5m 28s | | master passed |
| +1 :green_heart: | checkstyle | 3m 18s | | master passed |
| +1 :green_heart: | javadoc | 3m 59s | | master passed |
| -1 :x: | spotbugs | 1m 18s | [/branch-spotbugs-tez-api.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/37/artifact/out/branch-spotbugs-tez-api.txt) | tez-api in master failed. |
| -1 :x: | spotbugs | 0m 30s | [/branch-spotbugs-tez-common.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/37/artifact/out/branch-spotbugs-tez-common.txt) | tez-common in master failed. |
| -1 :x: | spotbugs | 0m 38s | [/branch-spotbugs-tez-runtime-library.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/37/artifact/out/branch-spotbugs-tez-runtime-library.txt) | tez-runtime-library in master failed. |
| -1 :x: | spotbugs | 0m 30s | [/branch-spotbugs-tez-examples.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/37/artifact/out/branch-spotbugs-tez-examples.txt) | tez-examples in master failed. |
| -1 :x: | spotbugs | 0m 43s | [/branch-spotbugs-tez-dag.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/37/artifact/out/branch-spotbugs-tez-dag.txt) | tez-dag in master failed. |
| -1 :x: | spotbugs | 0m 24s | [/branch-spotbugs-root.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/37/artifact/out/branch-spotbugs-root.txt) | root in master failed. |
| -0 :warning: | patch | 12m 42s | | Used diff version of patch file. Binary files and potentially other changes not applied. Please rebase and squash commits if necessary. |
|||| _ Patch Compile Tests _ |
| +0 :ok: | mvndep | 0m 17s | | Maven dependency ordering for patch |
| +1 :green_heart: | mvninstall | 6m 22s | | the patch passed |
| +1 :green_heart: | codespell | 0m 50s | | No new issues. |
| +1 :green_heart: | compile | 5m 26s | | the patch passed |
| +1 :green_heart: | javac | 5m 26s | | the patch passed |
| +1 :green_heart: | blanks | 0m 0s | | The patch has no blanks issues. |
| +1 :green_heart: | checkstyle | 0m 26s | | tez-api: The patch generated 0 new + 118 unchanged - 3 fixed = 118 total (was 121) |
| +1 :green_heart: | checkstyle | 0m 23s | | The patch passed checkstyle in tez-common |
| +1 :green_heart: | checkstyle | 0m 28s | | The patch passed checkstyle in tez-runtime-library |
| +1 :green_heart: | checkstyle | 0m 23s | | tez-examples: The patch generated 0 new + 10 unchanged - 1 fixed = 10 total (was 11) |
| -0 :warning: | checkstyle | 0m 35s | [/results-checkstyle-tez-dag.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/37/artifact/out/results-checkstyle-tez-dag.txt) | tez-dag: The patch generated 2 new + 610 unchanged - 2 fixed = 612 total (was 612) |
| -0 :warning: | checkstyle | 0m 45s | [/results-checkstyle-root.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/37/artifact/out/results-checkstyle-root.txt) | root: The patch generated 2 new + 740 unchanged - 6 fixed = 742 total (was 746) |
| +1 :green_heart: | javadoc | 3m 58s | | the patch passed |
| -1 :x: | spotbugs | 0m 38s | [/patch-spotbugs-tez-api.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/37/artifact/out/patch-spotbugs-tez-api.txt) | tez-api in the patch failed. |
| -1 :x: | spotbugs | 0m 30s | [/patch-spotbugs-tez-common.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/37/artifact/out/patch-spotbugs-tez-common.txt) | tez-common in the patch failed. |
| -1 :x: | spotbugs | 0m 34s | [/patch-spotbugs-tez-runtime-library.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/37/artifact/out/patch-spotbugs-tez-runtime-library.txt) | tez-runtime-library in the patch failed. |
| -1 :x: | spotbugs | 0m 27s | [/patch-spotbugs-tez-examples.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/37/artifact/out/patch-spotbugs-tez-examples.txt) | tez-examples in the patch failed. |
| -1 :x: | spotbugs | 0m 42s | [/patch-spotbugs-tez-dag.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/37/artifact/out/patch-spotbugs-tez-dag.txt) | tez-dag in the patch failed. |
| -1 :x: | spotbugs | 0m 23s | [/patch-spotbugs-root.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/37/artifact/out/patch-spotbugs-root.txt) | root in the patch failed. |
|||| _ Other Tests _ |
| +1 :green_heart: | unit | 2m 37s | | tez-api in the patch passed. |
| +1 :green_heart: | unit | 0m 42s | | tez-common in the patch passed. |
| +1 :green_heart: | unit | 6m 10s | | tez-runtime-library in the patch passed. |
| +1 :green_heart: | unit | 0m 31s | | tez-examples in the patch passed. |
| +1 :green_heart: | unit | 6m 34s | | tez-dag in the patch passed. |
| -1 :x: | unit | 71m 32s | [/patch-unit-root.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/37/artifact/out/patch-unit-root.txt) | root in the patch passed. |
| +1 :green_heart: | asflicense | 2m 37s | | The patch does not generate ASF License warnings. |
| | | 170m 33s | | |
| Reason | Tests |
|-------:|:------|
| Failed junit tests | tez.test.TestRecovery |
| Subsystem | Report/Notes |
|----------:|:-------------|
| Docker | ClientAPI=1.52 ServerAPI=1.52 base: https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/37/artifact/out/Dockerfile |
| GITHUB PR | https://github.com/apache/tez/pull/427 |
| Optional Tests | dupname asflicense javac javadoc unit codespell detsecrets xmllint compile spotbugs checkstyle |
| uname | Linux 670710632bf8 5.15.0-161-generic #171-Ubuntu SMP Sat Oct 11 08:17:01 UTC 2025 x86_64 x86_64 x86_64 GNU/Linux |
| Build tool | maven |
| Personality | /home/jenkins/jenkins-home/workspace/tez-multibranch_PR-427/src/.yetus/personality.sh |
| git revision | master / bd1161c1b6c0208de49596f39ced6cfdbc86856b |
| Default Java | Ubuntu-21.0.9+10-Ubuntu-124.04 |
| Test Results | https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/37/testReport/ |
| Max. process+thread count | 1380 (vs. ulimit of 5500) |
| modules | C: tez-api tez-common tez-runtime-library tez-examples tez-dag . U: . |
| Console output | https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/37/console |
| versions | git=2.43.0 maven=3.8.7 codespell=2.0.0 |
| Powered by | Apache Yetus 0.15.1 https://yetus.apache.org |
This message was automatically generated.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] TEZ-4007: Introduce AmExtensions and Zookeeper-based FrameworkServices [tez]
Posted by "abstractdog (via GitHub)" <gi...@apache.org>.
abstractdog commented on code in PR #427:
URL: https://github.com/apache/tez/pull/427#discussion_r2549661395
##########
tez-api/src/main/java/org/apache/tez/client/registry/AMRecord.java:
##########
@@ -150,14 +180,21 @@ public boolean equals(Object other) {
public ServiceRecord toServiceRecord() {
ServiceRecord serviceRecord = new ServiceRecord();
serviceRecord.set(APP_ID_RECORD_KEY, appId);
- serviceRecord.set(HOST_RECORD_KEY, host);
+ serviceRecord.set(HOST_NAME_RECORD_KEY, hostName);
+ serviceRecord.set(HOST_IP_RECORD_KEY, hostIp);
serviceRecord.set(PORT_RECORD_KEY, port);
- serviceRecord.set(OPAQUE_ID_KEY, id);
+ serviceRecord.set(EXTERNAL_ID_KEY, externalId);
+ serviceRecord.set(COMPUTE_GROUP_NAME_KEY, computeName);
return serviceRecord;
}
+ @Override
+ public String toString() {
+ return toServiceRecord().attributes().toString();
+ }
Review Comment:
used in INFO level messages in downstream-only tez session client implementation, and toServiceRecord itself is used in ZkAMRegistry to store AM information in zookeeper as JSON
would you be fine to cache ServiceRecord in a field and return it like:
```
public ServiceRecord toServiceRecord() {
if (serviceRecord != null){
return serviceRecord;
}
serviceRecord = new ServiceRecord();
serviceRecord.set(APP_ID_RECORD_KEY, appId);
serviceRecord.set(HOST_NAME_RECORD_KEY, hostName);
serviceRecord.set(HOST_IP_RECORD_KEY, hostIp);
serviceRecord.set(PORT_RECORD_KEY, port);
serviceRecord.set(EXTERNAL_ID_KEY, externalId);
serviceRecord.set(COMPUTE_GROUP_NAME_KEY, computeName);
return serviceRecord;
}
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] TEZ-4007: Introduce AmExtensions and Zookeeper-based FrameworkServices [tez]
Posted by "okumin (via GitHub)" <gi...@apache.org>.
okumin commented on code in PR #427:
URL: https://github.com/apache/tez/pull/427#discussion_r2552095265
##########
tez-api/src/main/java/org/apache/tez/client/registry/zookeeper/ZkFrameworkClient.java:
##########
@@ -0,0 +1,132 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.client.registry.zookeeper;
+
+import java.io.IOException;
+
+import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.client.api.YarnClientApplication;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.util.Records;
+import org.apache.tez.client.FrameworkClient;
+import org.apache.tez.client.registry.AMRecord;
+import org.apache.tez.dag.api.TezConfiguration;
+
+public class ZkFrameworkClient extends FrameworkClient {
+
+ private AMRecord amRecord;
+ private ZkAMRegistryClient amRegistryClient = null;
+ private volatile boolean isRunning = false;
+ private String amHost;
+ private int amPort;
+
+ @Override
+ public synchronized void init(TezConfiguration tezConf) {
+ if (this.amRegistryClient == null) {
+ try {
+ this.amRegistryClient = ZkAMRegistryClient.getClient(tezConf);
+ this.isRunning = true;
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ @Override
+ public void start() {
+ try {
+ this.amRegistryClient.start();
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ isRunning = true;
+ }
+
+ @Override
+ public void stop() {
+ isRunning = false;
+ amRegistryClient.close();
+ }
+
+ @Override
+ public void close() throws IOException {
+ amRegistryClient.close();
+ }
+
+ @Override
+ public YarnClientApplication createApplication() throws YarnException, IOException {
+ ApplicationSubmissionContext context = Records.newRecord(ApplicationSubmissionContext.class);
+ ApplicationId appId = amRecord.getApplicationId();
Review Comment:
Thanks. I presume that, on YARN with the session mode, Tez creates an AM when a `TezClient` starts. In the case of ZK Standalone, I'm guessing an AM is made outside the lifecycle of a `TezClient`, and the TezClient pulls an app with an explicitly passed app ID. I'm still not 100% confident, and I'd appreciate it if you could give me tips. I put my understanding below.
https://docs.google.com/document/d/1SGIwfXgfwMfrtrHvNwdLsdbbHkb--5BsGJ9yt-9-bP0/edit?usp=sharing
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] TEZ-4007: Introduce AmExtensions and Zookeeper-based FrameworkServices [tez]
Posted by "abstractdog (via GitHub)" <gi...@apache.org>.
abstractdog commented on code in PR #427:
URL: https://github.com/apache/tez/pull/427#discussion_r2549747379
##########
tez-api/src/main/java/org/apache/tez/client/registry/AMRegistryClient.java:
##########
@@ -0,0 +1,98 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.client.registry;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Client-side interface for discovering Application Master (AM) instances
+ * registered in the AM registry.
+ *
+ * <p>Implementations are responsible for locating AM endpoints and returning
+ * their metadata. This API is used by client components to discover running
+ * Tez AMs.</p>
+ *
+ * <p>Listeners may be registered to receive notifications when AM records
+ * appear or are removed.</p>
+ */
+public abstract class AMRegistryClient implements Closeable {
+
+ private final List<AMRegistryClientListener> listeners = new ArrayList<>();
+
+ /**
+ * Returns the current set of registered listeners.
+ *
+ * @return a mutable list of listeners
+ */
+ protected List<AMRegistryClientListener> getListeners() {
+ return listeners;
+ }
Review Comment:
ack, removing getListeners altogether
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] TEZ-4007: Introduce AmExtensions and Zookeeper-based FrameworkServices [tez]
Posted by "abstractdog (via GitHub)" <gi...@apache.org>.
abstractdog commented on code in PR #427:
URL: https://github.com/apache/tez/pull/427#discussion_r2543212110
##########
tez-dag/src/main/java/org/apache/tez/frameworkplugins/yarn/YarnServerFrameworkService.java:
##########
@@ -0,0 +1,121 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tez.frameworkplugins.yarn;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.util.Map;
+import java.util.Optional;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.util.ConverterUtils;
+import org.apache.tez.client.registry.AMRegistry;
+import org.apache.tez.common.TezUtilsInternal;
+import org.apache.tez.common.security.JobTokenIdentifier;
+import org.apache.tez.common.security.JobTokenSecretManager;
+import org.apache.tez.common.security.TokenCache;
+import org.apache.tez.dag.api.TezConstants;
+import org.apache.tez.dag.api.records.DAGProtos;
+import org.apache.tez.dag.app.ClusterInfo;
+import org.apache.tez.dag.app.dag.Vertex;
+import org.apache.tez.frameworkplugins.AmExtensions;
+import org.apache.tez.frameworkplugins.ServerFrameworkService;
+import org.apache.tez.frameworkplugins.TaskResourceException;
+
+/**
+ * YARN-based server framework service implementation.
+ * Provides default YARN framework server functionality with default implementations
+ * for all AmExtensions methods.
+ */
+public class YarnServerFrameworkService implements ServerFrameworkService {
+
+ private final YarnAmExtensions amExtensions = new YarnAmExtensions();
+
+ @Override
+ public Optional<AMRegistry> createOrGetAMRegistry(Configuration conf) {
+ // Return empty for now - YARN mode doesn't require a custom AM registry
+ return Optional.empty();
+ }
+
+ @Override
+ public AmExtensions createAmExtensions() {
+ return new YarnAmExtensions();
Review Comment:
good catch, let me fix
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] TEZ-4007: Introduce AmExtensions and Zookeeper-based FrameworkServices [tez]
Posted by "abstractdog (via GitHub)" <gi...@apache.org>.
abstractdog commented on code in PR #427:
URL: https://github.com/apache/tez/pull/427#discussion_r2671733389
##########
tez-api/src/main/java/org/apache/tez/client/registry/AMRegistry.java:
##########
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.client.registry;
+
+
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+
+
+/**
+ * Base class for {@code AMRegistry} implementations.
+ *
+ * <p>The specific implementation is configured via the
+ * {@code tez.am.registry.class} property.</p>
+ *
+ * <p>Implementations are expected to provide appropriate service lifecycle
+ * behavior, including:
+ * <ul>
+ * <li>{@code init}</li>
+ * <li>{@code serviceStart}</li>
+ * <li>{@code serviceStop}</li>
+ * </ul>
+ * </p>
+ */
+public interface AMRegistry extends AutoCloseable {
+
+ void add(AMRecord server) throws Exception;
+
+ void remove(AMRecord server) throws Exception;
Review Comment:
record is better, fixing this
##########
tez-api/src/main/java/org/apache/tez/client/registry/AMRegistry.java:
##########
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.client.registry;
+
+
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+
+
+/**
+ * Base class for {@code AMRegistry} implementations.
+ *
+ * <p>The specific implementation is configured via the
+ * {@code tez.am.registry.class} property.</p>
+ *
+ * <p>Implementations are expected to provide appropriate service lifecycle
+ * behavior, including:
+ * <ul>
+ * <li>{@code init}</li>
+ * <li>{@code serviceStart}</li>
+ * <li>{@code serviceStop}</li>
+ * </ul>
+ * </p>
+ */
+public interface AMRegistry extends AutoCloseable {
+
+ void add(AMRecord server) throws Exception;
+
+ void remove(AMRecord server) throws Exception;
+
+ ApplicationId generateNewId() throws Exception;
+
+ AMRecord createAmRecord(ApplicationId appId, String hostName, String hostIp, int port,
+ String computeName);
+
+ void close();
Review Comment:
ack
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] TEZ-4007: Introduce AmExtensions and Zookeeper-based FrameworkServices [tez]
Posted by "abstractdog (via GitHub)" <gi...@apache.org>.
abstractdog commented on code in PR #427:
URL: https://github.com/apache/tez/pull/427#discussion_r2669024497
##########
tez-api/src/main/java/org/apache/tez/client/registry/AMRecord.java:
##########
@@ -148,16 +179,27 @@ public boolean equals(Object other) {
* @return a {@link ServiceRecord} populated with the values of this {@code AMRecord}
*/
public ServiceRecord toServiceRecord() {
- ServiceRecord serviceRecord = new ServiceRecord();
+ if (serviceRecord != null) {
+ return serviceRecord;
+ }
+ serviceRecord = new ServiceRecord();
serviceRecord.set(APP_ID_RECORD_KEY, appId);
- serviceRecord.set(HOST_RECORD_KEY, host);
+ serviceRecord.set(HOST_NAME_RECORD_KEY, hostName);
+ serviceRecord.set(HOST_IP_RECORD_KEY, hostIp);
serviceRecord.set(PORT_RECORD_KEY, port);
- serviceRecord.set(OPAQUE_ID_KEY, id);
+ serviceRecord.set(EXTERNAL_ID_KEY, externalId);
+ serviceRecord.set(COMPUTE_GROUP_NAME_KEY, computeName);
+
return serviceRecord;
}
+ @Override
+ public String toString() {
+ return toServiceRecord().attributes().toString();
Review Comment:
I think there is no "before setting the values" scenario given only final fields, I'm assuming the the logging must happen with the AMRecord instance when the constructor is definitely finished
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] TEZ-4007: Introduce AmExtensions and Zookeeper-based FrameworkServices [tez]
Posted by "tez-yetus (via GitHub)" <gi...@apache.org>.
tez-yetus commented on PR #427:
URL: https://github.com/apache/tez/pull/427#issuecomment-3749375975
:broken_heart: **-1 overall**
| Vote | Subsystem | Runtime | Logfile | Comment |
|:----:|----------:|--------:|:--------:|:-------:|
| +0 :ok: | reexec | 0m 45s | | Docker mode activated. |
|||| _ Prechecks _ |
| +1 :green_heart: | dupname | 0m 0s | | No case conflicting files found. |
| +0 :ok: | detsecrets | 0m 1s | | detect-secrets was not available. |
| +0 :ok: | xmllint | 0m 1s | | xmllint was not available. |
| +1 :green_heart: | @author | 0m 0s | | The patch does not contain any @author tags. |
| +1 :green_heart: | test4tests | 0m 0s | | The patch appears to include 14 new or modified test files. |
|||| _ master Compile Tests _ |
| +0 :ok: | mvndep | 2m 26s | | Maven dependency ordering for branch |
| +1 :green_heart: | mvninstall | 9m 37s | | master passed |
| +1 :green_heart: | compile | 5m 24s | | master passed |
| +1 :green_heart: | checkstyle | 3m 18s | | master passed |
| +1 :green_heart: | javadoc | 3m 58s | | master passed |
| -1 :x: | spotbugs | 1m 19s | [/branch-spotbugs-tez-api.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/41/artifact/out/branch-spotbugs-tez-api.txt) | tez-api in master failed. |
| -1 :x: | spotbugs | 0m 30s | [/branch-spotbugs-tez-common.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/41/artifact/out/branch-spotbugs-tez-common.txt) | tez-common in master failed. |
| -1 :x: | spotbugs | 0m 37s | [/branch-spotbugs-tez-runtime-library.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/41/artifact/out/branch-spotbugs-tez-runtime-library.txt) | tez-runtime-library in master failed. |
| -1 :x: | spotbugs | 0m 30s | [/branch-spotbugs-tez-examples.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/41/artifact/out/branch-spotbugs-tez-examples.txt) | tez-examples in master failed. |
| -1 :x: | spotbugs | 0m 44s | [/branch-spotbugs-tez-dag.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/41/artifact/out/branch-spotbugs-tez-dag.txt) | tez-dag in master failed. |
| -1 :x: | spotbugs | 0m 24s | [/branch-spotbugs-root.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/41/artifact/out/branch-spotbugs-root.txt) | root in master failed. |
| -0 :warning: | patch | 12m 41s | | Used diff version of patch file. Binary files and potentially other changes not applied. Please rebase and squash commits if necessary. |
|||| _ Patch Compile Tests _ |
| +0 :ok: | mvndep | 0m 16s | | Maven dependency ordering for patch |
| +1 :green_heart: | mvninstall | 6m 22s | | the patch passed |
| +1 :green_heart: | codespell | 0m 51s | | No new issues. |
| +1 :green_heart: | compile | 5m 28s | | the patch passed |
| +1 :green_heart: | javac | 5m 28s | | the patch passed |
| +1 :green_heart: | blanks | 0m 0s | | The patch has no blanks issues. |
| +1 :green_heart: | checkstyle | 0m 26s | | tez-api: The patch generated 0 new + 118 unchanged - 3 fixed = 118 total (was 121) |
| +1 :green_heart: | checkstyle | 0m 23s | | The patch passed checkstyle in tez-common |
| +1 :green_heart: | checkstyle | 0m 28s | | The patch passed checkstyle in tez-runtime-library |
| +1 :green_heart: | checkstyle | 0m 23s | | tez-examples: The patch generated 0 new + 10 unchanged - 1 fixed = 10 total (was 11) |
| -0 :warning: | checkstyle | 0m 33s | [/results-checkstyle-tez-dag.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/41/artifact/out/results-checkstyle-tez-dag.txt) | tez-dag: The patch generated 2 new + 610 unchanged - 2 fixed = 612 total (was 612) |
| -0 :warning: | checkstyle | 0m 46s | [/results-checkstyle-root.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/41/artifact/out/results-checkstyle-root.txt) | root: The patch generated 2 new + 740 unchanged - 6 fixed = 742 total (was 746) |
| +1 :green_heart: | javadoc | 3m 58s | | the patch passed |
| -1 :x: | spotbugs | 0m 38s | [/patch-spotbugs-tez-api.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/41/artifact/out/patch-spotbugs-tez-api.txt) | tez-api in the patch failed. |
| -1 :x: | spotbugs | 0m 31s | [/patch-spotbugs-tez-common.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/41/artifact/out/patch-spotbugs-tez-common.txt) | tez-common in the patch failed. |
| -1 :x: | spotbugs | 0m 38s | [/patch-spotbugs-tez-runtime-library.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/41/artifact/out/patch-spotbugs-tez-runtime-library.txt) | tez-runtime-library in the patch failed. |
| -1 :x: | spotbugs | 0m 28s | [/patch-spotbugs-tez-examples.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/41/artifact/out/patch-spotbugs-tez-examples.txt) | tez-examples in the patch failed. |
| -1 :x: | spotbugs | 0m 42s | [/patch-spotbugs-tez-dag.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/41/artifact/out/patch-spotbugs-tez-dag.txt) | tez-dag in the patch failed. |
| -1 :x: | spotbugs | 0m 24s | [/patch-spotbugs-root.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/41/artifact/out/patch-spotbugs-root.txt) | root in the patch failed. |
|||| _ Other Tests _ |
| +1 :green_heart: | unit | 2m 35s | | tez-api in the patch passed. |
| +1 :green_heart: | unit | 0m 42s | | tez-common in the patch passed. |
| +1 :green_heart: | unit | 6m 9s | | tez-runtime-library in the patch passed. |
| +1 :green_heart: | unit | 0m 30s | | tez-examples in the patch passed. |
| +1 :green_heart: | unit | 6m 28s | | tez-dag in the patch passed. |
| +1 :green_heart: | unit | 72m 8s | | root in the patch passed. |
| +1 :green_heart: | asflicense | 2m 36s | | The patch does not generate ASF License warnings. |
| | | 147m 21s | | |
| Subsystem | Report/Notes |
|----------:|:-------------|
| Docker | ClientAPI=1.52 ServerAPI=1.52 base: https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/41/artifact/out/Dockerfile |
| GITHUB PR | https://github.com/apache/tez/pull/427 |
| Optional Tests | dupname asflicense javac javadoc unit codespell detsecrets xmllint compile spotbugs checkstyle |
| uname | Linux 08e0e7068641 5.15.0-161-generic #171-Ubuntu SMP Sat Oct 11 08:17:01 UTC 2025 x86_64 x86_64 x86_64 GNU/Linux |
| Build tool | maven |
| Personality | /home/jenkins/jenkins-home/workspace/tez-multibranch_PR-427/src/.yetus/personality.sh |
| git revision | master / ff289be4f463be2f6d9da2d318089f92a8390ae5 |
| Default Java | Ubuntu-21.0.9+10-Ubuntu-124.04 |
| Test Results | https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/41/testReport/ |
| Max. process+thread count | 1389 (vs. ulimit of 5500) |
| modules | C: tez-api tez-common tez-runtime-library tez-examples tez-dag . U: . |
| Console output | https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/41/console |
| versions | git=2.43.0 maven=3.8.7 codespell=2.0.0 |
| Powered by | Apache Yetus 0.15.1 https://yetus.apache.org |
This message was automatically generated.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] TEZ-4007: Introduce AmExtensions and Zookeeper-based FrameworkServices [tez]
Posted by "abstractdog (via GitHub)" <gi...@apache.org>.
abstractdog commented on code in PR #427:
URL: https://github.com/apache/tez/pull/427#discussion_r2549661395
##########
tez-api/src/main/java/org/apache/tez/client/registry/AMRecord.java:
##########
@@ -150,14 +180,21 @@ public boolean equals(Object other) {
public ServiceRecord toServiceRecord() {
ServiceRecord serviceRecord = new ServiceRecord();
serviceRecord.set(APP_ID_RECORD_KEY, appId);
- serviceRecord.set(HOST_RECORD_KEY, host);
+ serviceRecord.set(HOST_NAME_RECORD_KEY, hostName);
+ serviceRecord.set(HOST_IP_RECORD_KEY, hostIp);
serviceRecord.set(PORT_RECORD_KEY, port);
- serviceRecord.set(OPAQUE_ID_KEY, id);
+ serviceRecord.set(EXTERNAL_ID_KEY, externalId);
+ serviceRecord.set(COMPUTE_GROUP_NAME_KEY, computeName);
return serviceRecord;
}
+ @Override
+ public String toString() {
+ return toServiceRecord().attributes().toString();
+ }
Review Comment:
used in INFO level messages in downstream-only tez session client implementation, and toServiceRecord itself is used in ZkAMRegistry to store AM information in zookeeper as JSON
would you be fine to cache ServiceRecord in a field and return it like:
```
public ServiceRecord toServiceRecord() {
if (serviceRecord != null) {
return serviceRecord;
}
serviceRecord = new ServiceRecord();
serviceRecord.set(APP_ID_RECORD_KEY, appId);
serviceRecord.set(HOST_NAME_RECORD_KEY, hostName);
serviceRecord.set(HOST_IP_RECORD_KEY, hostIp);
serviceRecord.set(PORT_RECORD_KEY, port);
serviceRecord.set(EXTERNAL_ID_KEY, externalId);
serviceRecord.set(COMPUTE_GROUP_NAME_KEY, computeName);
return serviceRecord;
}
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] TEZ-4007: Introduce AmExtensions and Zookeeper-based FrameworkServices [tez]
Posted by "abstractdog (via GitHub)" <gi...@apache.org>.
abstractdog commented on code in PR #427:
URL: https://github.com/apache/tez/pull/427#discussion_r2567530586
##########
tez-dag/src/main/java/org/apache/tez/dag/api/client/registry/zookeeper/ZkAMRegistry.java:
##########
@@ -0,0 +1,175 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.api.client.registry.zookeeper;
+
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.curator.RetryLoop;
+import org.apache.curator.RetryPolicy;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.registry.client.binding.RegistryUtils;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.tez.client.registry.AMRecord;
+import org.apache.tez.client.registry.AMRegistry;
+import org.apache.tez.client.registry.zookeeper.ZkConfig;
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.data.Stat;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Curator/Zookeeper impl of AMRegistry (for internal use only)
+ * Clients should use org.apache.tez.dag.api.client.registry.zookeeper.ZkAMRegistryClient instead.
+ */
+@InterfaceAudience.Private
+public class ZkAMRegistry extends AMRegistry {
+
+ private static final Logger LOG = LoggerFactory.getLogger(ZkAMRegistry.class);
+
+ private final List<AMRecord> amRecords = new ArrayList<>();
+ private final String externalId;
+
+ private CuratorFramework client = null;
+ private String namespace = null;
+ private ZkConfig zkConfig = null;
+ private boolean started = false;
+
+ public ZkAMRegistry(String externalId) {
+ super("ZkAMRegistry");
+ this.externalId = externalId;
+ }
+
+ @Override
+ public void serviceInit(Configuration conf) {
+ if (zkConfig == null) {
+ zkConfig = new ZkConfig(conf);
+ this.client = zkConfig.createCuratorFramework();
+ this.namespace = zkConfig.getZkNamespace();
+ LOG.info("AMRegistryZkImpl initialized");
+ }
+ }
+
+ @Override
+ public void serviceStart() throws Exception {
+ if (!started) {
+ client.start();
+ started = true;
+ LOG.info("AMRegistryZkImpl started");
+ }
+ }
+
+ //Deletes from Zookeeper AMRecords that were added by this instance
+ @Override
+ public void serviceStop() throws Exception {
+ List<AMRecord> records = new ArrayList<>(amRecords);
+ for (AMRecord amRecord : records) {
+ remove(amRecord);
+ }
+ client.close();
+ LOG.info("AMRegistryZkImpl shutdown");
+ }
+
+ //Serialize AMRecord to ServiceRecord and deliver the JSON bytes to
+ //zkNode at the path: <TEZ_AM_REGISTRY_NAMESPACE>/<appId>
+ @Override
+ public void add(AMRecord server) throws Exception {
+ RegistryUtils.ServiceRecordMarshal marshal = new RegistryUtils.ServiceRecordMarshal();
+ String json = marshal.toJson(server.toServiceRecord());
+ try {
+ final String path = namespace + "/" + server.getApplicationId().toString();
+ client.setData().forPath(path, json.getBytes(StandardCharsets.UTF_8));
+ LOG.info("Added AMRecord to zkpath {}", path);
+ } catch (KeeperException.NoNodeException nne) {
+ client.create().creatingParentContainersIfNeeded().withMode(CreateMode.EPHEMERAL)
+ .forPath(namespace + "/" + server.getApplicationId().toString(), json.getBytes(StandardCharsets.UTF_8));
+ }
+ amRecords.add(server);
+ }
+
+ @Override
+ public void remove(AMRecord server) throws Exception {
+ amRecords.remove(server);
+ final String path = namespace + "/" + server.getApplicationId().toString();
+ client.delete().forPath(path);
+ LOG.info("Deleted AMRecord from zkpath {}", path);
+ }
+
+ @Override
+ public ApplicationId generateNewId() throws Exception {
+ createNamespaceIfNotExists();
+ long namespaceCreationTime = getNamespaceCreationTime();
+
+ boolean success = false;
+ long startTime = System.currentTimeMillis();
+ RetryPolicy retryPolicy = zkConfig.getRetryPolicy();
+ int tryId = 0;
+ for (int i = 0; (i < zkConfig.getCuratorMaxRetries()) && !success; i++) {
+ List<String> children = client.getChildren().forPath(namespace);
+ if (children != null && !children.isEmpty()) {
+ children.sort(Collections.reverseOrder());
+ String last = children.getFirst();
+ ApplicationId lastAppId = ApplicationId.fromString(last);
+ tryId = lastAppId.getId() + 1;
+ }
+ ApplicationId tryAppId = ApplicationId.newInstance(namespaceCreationTime, tryId);
+ try {
+ client
+ .create()
+ .withMode(CreateMode.EPHEMERAL)
Review Comment:
EPHEMERAL works here with the current logic (described in a previous comment), auto-deletion is a value
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] TEZ-4007: Introduce AmExtensions and Zookeeper-based FrameworkServices [tez]
Posted by "abstractdog (via GitHub)" <gi...@apache.org>.
abstractdog commented on code in PR #427:
URL: https://github.com/apache/tez/pull/427#discussion_r2567618554
##########
tez-dag/src/main/java/org/apache/tez/frameworkplugins/zookeeper/ZkStandaloneServerFrameworkService.java:
##########
@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tez.frameworkplugins.zookeeper;
+
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.tez.client.registry.AMRegistry;
+import org.apache.tez.dag.api.TezConstants;
+import org.apache.tez.dag.api.client.registry.zookeeper.ZkAMRegistry;
+import org.apache.tez.frameworkplugins.AMExtensions;
+import org.apache.tez.frameworkplugins.ServerFrameworkService;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class ZkStandaloneServerFrameworkService implements ServerFrameworkService {
+ private static final Logger LOG = LoggerFactory.getLogger(ZkStandaloneServerFrameworkService.class);
+ private ZkAMRegistry amRegistry;
+
+ @Override
+ public synchronized AMRegistry getAMRegistry(Configuration conf) {
+ if (amRegistry == null) {
+ try {
+ final String externalID = System.getenv(TezConstants.TEZ_AM_EXTERNAL_ID);
+ amRegistry = new ZkAMRegistry(externalID);
+ amRegistry.init(conf);
+ amRegistry.start();
Review Comment:
added javadoc comment to clarify this:
```
/**
* Returns a singleton {@link AMRegistry} instance backed by ZooKeeper.
*
* <p>If the registry has not yet been created, this method initializes and starts
* a new {@link ZkAMRegistry} using the external AM identifier obtained from the
* {@code TEZ_AM_EXTERNAL_ID} environment variable.</p>
*
* <p>When the registry is used as a service within the DAGAppMaster, the
* DAGAppMaster is responsible for managing its lifecycle, including closure.</p>
*
* @param conf the configuration used to initialize the registry; must not be null
* @return the initialized and started {@link AMRegistry} instance
* @throws IllegalStateException if the {@code TEZ_AM_EXTERNAL_ID} environment variable is not set
* @throws RuntimeException if an error occurs while creating, initializing, or starting the registry
*/
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] TEZ-4007: Introduce AmExtensions and Zookeeper-based FrameworkServices [tez]
Posted by "abstractdog (via GitHub)" <gi...@apache.org>.
abstractdog commented on code in PR #427:
URL: https://github.com/apache/tez/pull/427#discussion_r2567667239
##########
tez-dag/src/main/java/org/apache/tez/dag/api/client/registry/zookeeper/ZkAMRegistry.java:
##########
@@ -0,0 +1,175 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.api.client.registry.zookeeper;
+
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.curator.RetryLoop;
+import org.apache.curator.RetryPolicy;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.registry.client.binding.RegistryUtils;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.tez.client.registry.AMRecord;
+import org.apache.tez.client.registry.AMRegistry;
+import org.apache.tez.client.registry.zookeeper.ZkConfig;
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.data.Stat;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Curator/Zookeeper impl of AMRegistry (for internal use only)
+ * Clients should use org.apache.tez.dag.api.client.registry.zookeeper.ZkAMRegistryClient instead.
+ */
+@InterfaceAudience.Private
+public class ZkAMRegistry extends AMRegistry {
+
+ private static final Logger LOG = LoggerFactory.getLogger(ZkAMRegistry.class);
+
+ private final List<AMRecord> amRecords = new ArrayList<>();
Review Comment:
yeah, fixed as:
```
private final List<AMRecord> amRecords = Collections.synchronizedList(new ArrayList<>());
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] TEZ-4007: Introduce AmExtensions and Zookeeper-based FrameworkServices [tez]
Posted by "abstractdog (via GitHub)" <gi...@apache.org>.
abstractdog commented on PR #427:
URL: https://github.com/apache/tez/pull/427#issuecomment-3595442285
@ayushtkn : all your comments have been addressed
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] TEZ-4007: Introduce AmExtensions and Zookeeper-based FrameworkServices [tez]
Posted by "abstractdog (via GitHub)" <gi...@apache.org>.
abstractdog commented on code in PR #427:
URL: https://github.com/apache/tez/pull/427#discussion_r2671851521
##########
tez-api/src/main/java/org/apache/tez/client/registry/AMRegistryClient.java:
##########
@@ -0,0 +1,118 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.client.registry;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Client-side interface for discovering Application Master (AM) instances
+ * registered in the AM registry.
+ *
+ * <p>Implementations are responsible for locating AM endpoints and returning
+ * their metadata. This API is used by client components to discover running
+ * Tez AMs.</p>
+ *
+ * <p>Listeners may be registered to receive notifications when AM records
+ * appear or are removed.</p>
+ */
+public abstract class AMRegistryClient implements Closeable {
+ private static final Logger LOG = LoggerFactory.getLogger(AMRegistryClient.class);
+
+ private final List<AMRegistryClientListener> listeners = new ArrayList<>();
+
+ /**
+ * Lookup AM metadata for the given application ID.
+ *
+ * @param appId the application ID
+ * @return the AM record if found, otherwise {@code null}
+ * @throws IOException if the lookup fails
+ */
+ public abstract AMRecord getRecord(ApplicationId appId) throws IOException;
+
+ /**
+ * Retrieve all AM records known in the registry.
+ *
+ * @return a list of AM records (possibly empty)
+ * @throws IOException if the fetch fails
+ */
+ public abstract List<AMRecord> getAllRecords() throws IOException;
+
+ /**
+ * Register a listener for AM registry events.
+ * The listener will be notified when AM records are added or removed.
+ *
+ * @param listener the listener to add
+ */
+ public synchronized void addListener(AMRegistryClientListener listener) {
+ listeners.add(listener);
+ }
+
Review Comment:
simply hasn't needed such a thing so far, currently, we just subscribe and close the client in the end
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] TEZ-4007: Introduce AmExtensions and Zookeeper-based FrameworkServices [tez]
Posted by "abstractdog (via GitHub)" <gi...@apache.org>.
abstractdog commented on code in PR #427:
URL: https://github.com/apache/tez/pull/427#discussion_r2672393297
##########
tez-api/src/main/java/org/apache/tez/client/registry/zookeeper/ZkAMRegistryClient.java:
##########
@@ -0,0 +1,209 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.client.registry.zookeeper;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.imps.CuratorFrameworkState;
+import org.apache.curator.framework.recipes.cache.ChildData;
+import org.apache.curator.framework.recipes.cache.TreeCache;
+import org.apache.curator.framework.recipes.cache.TreeCacheEvent;
+import org.apache.curator.framework.recipes.cache.TreeCacheListener;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.registry.client.types.ServiceRecord;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.tez.client.registry.AMRecord;
+import org.apache.tez.client.registry.AMRegistryClient;
+import org.apache.tez.client.registry.AMRegistryUtils;
+import org.apache.tez.dag.api.TezConfiguration;
+
+import com.fasterxml.jackson.core.JsonParseException;
+import com.google.common.base.Preconditions;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Curator/Zookeeper implementation of {@link AMRegistryClient}.
+ */
+@InterfaceAudience.Public
+public final class ZkAMRegistryClient extends AMRegistryClient {
+ private static final Logger LOG = LoggerFactory.getLogger(ZkAMRegistryClient.class);
+ private static final Map<String, ZkAMRegistryClient> INSTANCES = new HashMap<>();
+
+ private final Configuration conf;
+ // Cache of known AMs
+ private final ConcurrentHashMap<ApplicationId, AMRecord> amRecordCache = new ConcurrentHashMap<>();
+
+ private CuratorFramework client;
+ private TreeCache cache;
+ private ZkRegistryListener listener;
+
+ private ZkAMRegistryClient(final Configuration conf) {
+ this.conf = conf;
+ }
+
+ public static synchronized ZkAMRegistryClient getClient(final Configuration conf) {
+ String namespace = conf.get(TezConfiguration.TEZ_AM_REGISTRY_NAMESPACE);
+ ZkAMRegistryClient registry = INSTANCES.get(namespace);
+ if (registry == null) {
+ registry = new ZkAMRegistryClient(conf);
+ INSTANCES.put(namespace, registry);
+ }
+ LOG.info("Returning tez AM registry ({}) for namespace '{}'", System.identityHashCode(registry), namespace);
+ return registry;
+ }
+
+ /**
+ * Deserializes a {@link ServiceRecord} from ZooKeeper data and converts it into an {@link AMRecord}
+ * for caching.
+ *
+ * @param childData the ZooKeeper node data containing a serialized {@link ServiceRecord}
+ * @return an {@link AMRecord} constructed from the deserialized {@link ServiceRecord}, or {@code null}
+ * if no data is present
+ * @throws IOException if the data cannot be deserialized into a {@link ServiceRecord}
+ */
+ public static AMRecord getAMRecord(final ChildData childData) throws IOException {
+ // Not a leaf path. Only leaf path contains AMRecord.
+ if (!childData.getPath().contains(ApplicationId.appIdStrPrefix)) {
+ return null;
+ }
+ byte[] data = childData.getData();
+ // only the path appeared, there is no data yet
+ if (data.length == 0) {
+ return null;
+ }
+ String value = new String(data, StandardCharsets.UTF_8);
+ try {
+ return AMRegistryUtils.jsonStringToRecord(value);
+ } catch (JsonParseException e) {
+ //Not a json AMRecord (SRV), could be some other data.
+ LOG.warn("Non-json data received while de-serializing AMRecord: {}. Ignoring...", value);
+ return null;
+ }
+ }
+
+ public void start() throws Exception {
+ ZkConfig zkConf = new ZkConfig(this.conf);
+ client = zkConf.createCuratorFramework();
+ cache = new TreeCache(client, zkConf.getZkNamespace());
+ client.start();
+ cache.start();
Review Comment:
ack, while there is no clear example and guidance in official ZK docs other than API doc, this makes sense and it's inline with other sources on the web
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] TEZ-4007: Introduce AmExtensions and Zookeeper-based FrameworkServices [tez]
Posted by "abstractdog (via GitHub)" <gi...@apache.org>.
abstractdog commented on code in PR #427:
URL: https://github.com/apache/tez/pull/427#discussion_r2672423322
##########
tez-api/src/main/java/org/apache/tez/client/registry/zookeeper/ZkFrameworkClient.java:
##########
@@ -0,0 +1,170 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.client.registry.zookeeper;
+
+import java.io.IOException;
+
+import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.client.api.YarnClientApplication;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.util.Records;
+import org.apache.tez.client.FrameworkClient;
+import org.apache.tez.client.registry.AMRecord;
+import org.apache.tez.dag.api.TezConfiguration;
+
+import com.google.common.annotations.VisibleForTesting;
+
+public class ZkFrameworkClient extends FrameworkClient {
+
+ private AMRecord amRecord;
Review Comment:
it should, also, tracking `amHost` and `amPort` separately doesn't make sense, let's just rely on a single volatile record field
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] TEZ-4007: Introduce AmExtensions and Zookeeper-based FrameworkServices [tez]
Posted by "abstractdog (via GitHub)" <gi...@apache.org>.
abstractdog commented on code in PR #427:
URL: https://github.com/apache/tez/pull/427#discussion_r2549815047
##########
tez-api/src/main/java/org/apache/tez/client/registry/zookeeper/ZkAMRegistryClient.java:
##########
@@ -0,0 +1,205 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.client.registry.zookeeper;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
+
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.imps.CuratorFrameworkState;
+import org.apache.curator.framework.recipes.cache.ChildData;
+import org.apache.curator.framework.recipes.cache.TreeCache;
+import org.apache.curator.framework.recipes.cache.TreeCacheEvent;
+import org.apache.curator.framework.recipes.cache.TreeCacheListener;
+import org.apache.curator.shaded.com.google.common.base.Charsets;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.registry.client.binding.RegistryUtils;
+import org.apache.hadoop.registry.client.types.ServiceRecord;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.tez.client.registry.AMRecord;
+import org.apache.tez.client.registry.AMRegistryClient;
+import org.apache.tez.client.registry.AMRegistryClientListener;
+import org.apache.tez.dag.api.TezConfiguration;
+
+import com.fasterxml.jackson.core.JsonParseException;
+import com.google.common.base.Preconditions;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Curator/Zookeeper implementation of {@link AMRegistryClient}.
+ */
+@InterfaceAudience.Public
+public final class ZkAMRegistryClient extends AMRegistryClient {
+ private static final Logger LOG = LoggerFactory.getLogger(ZkAMRegistryClient.class);
+ private static final Map<String, ZkAMRegistryClient> INSTANCES = new HashMap<>();
+
+ private final Configuration conf;
+ //Cache of known AMs
+ private final ConcurrentHashMap<String, AMRecord> amRecordCache = new ConcurrentHashMap<>();
+ private CuratorFramework client;
+
+ private ZkAMRegistryClient(final Configuration conf) {
+ this.conf = conf;
+ }
+
+ public static synchronized ZkAMRegistryClient getClient(final Configuration conf) {
+ String namespace = conf.get(TezConfiguration.TEZ_AM_REGISTRY_NAMESPACE);
+ ZkAMRegistryClient registry = INSTANCES.get(namespace);
+ if (registry == null) {
+ registry = new ZkAMRegistryClient(conf);
+ INSTANCES.put(namespace, registry);
+ }
+ LOG.info("Returning tez AM registry ({}) for namespace '{}'", System.identityHashCode(registry), namespace);
+ return registry;
+ }
+
+ /**
+ * Deserializes a {@link ServiceRecord} from ZooKeeper data and converts it into an {@link AMRecord}
+ * for caching.
+ *
+ * @param childData the ZooKeeper node data containing a serialized {@link ServiceRecord}
+ * @return an {@link AMRecord} constructed from the deserialized {@link ServiceRecord}, or {@code null}
+ * if no data is present
+ * @throws IOException if the data cannot be deserialized into a {@link ServiceRecord}
+ */
+ public static AMRecord getAMRecord(final ChildData childData) throws IOException {
+ // not a leaf path. Only leaf path contains AMRecord
+ if (!childData.getPath().contains(ApplicationId.appIdStrPrefix)) {
+ return null;
+ }
+ byte[] data = childData.getData();
+ // only the path appeared, there is no data yet
+ if (data.length == 0) {
+ return null;
+ }
+ String value = new String(data, Charsets.UTF_8);
+ try {
+ RegistryUtils.ServiceRecordMarshal marshal = new RegistryUtils.ServiceRecordMarshal();
+ ServiceRecord serviceRecord = marshal.fromJson(value);
+ return new AMRecord(serviceRecord);
+ } catch (JsonParseException e) {
+ //Not a json AMRecord (SRV), could be some other data.
+ LOG.warn("Non-json data received while de-serializing AMRecord: {}. Ignoring...", value);
+ return null;
+ }
+ }
+
+ public void start() throws Exception {
+ ZkConfig zkConf = new ZkConfig(this.conf);
+ client = zkConf.createCuratorFramework();
+ final TreeCache cache = new TreeCache(client, zkConf.getZkNamespace());
+ client.start();
+ cache.start();
+ cache.getListenable().addListener(new ZkRegistryListener());
+ }
+
+ @Override
+ public AMRecord getRecord(String appId) {
+ if (amRecordCache.get(appId) == null) {
+ return null;
+ }
+ //Return a copy
+ return new AMRecord(amRecordCache.get(appId));
+ }
+
+ @Override
+ public List<AMRecord> getAllRecords() {
+ return amRecordCache.values().stream().map(AMRecord::new).collect(Collectors.toList());
+ }
+
+ @Override
+ public synchronized void addListener(AMRegistryClientListener listener) {
+ getListeners().add(listener);
+ }
+
+ @Override
+ public void close() {
+ client.close();
+ }
+
+ /**
+ * Callback listener for ZooKeeper events that updates the local cache
+ * when child nodes under the monitored path change.
+ */
+ private class ZkRegistryListener implements TreeCacheListener {
+
+ @Override
+ public void childEvent(final CuratorFramework clientParam, final TreeCacheEvent event) throws Exception {
+ Preconditions.checkArgument(clientParam != null && clientParam.getState() == CuratorFrameworkState.STARTED,
+ "Curator client is not started");
+
+ ChildData childData = event.getData();
+ switch (event.getType()) {
+ case NODE_ADDED:
+ if (isEmpty(childData)) {
+ LOG.info("AppId allocated: {}", childData.getPath());
+ } else {
+ AMRecord amRecord = getAMRecord(childData);
+ if (amRecord != null) {
+ LOG.info("AM registered with data: {}. Notifying {} listeners.", amRecord, getListeners().size());
+ amRecordCache.put(amRecord.getApplicationId().toString(), amRecord);
+ notifyOnAdded(amRecord);
+ }
+ }
+ break;
+ case NODE_UPDATED:
+ if (isEmpty(childData)) {
+ throw new RuntimeException("AM updated with empty data");
+ } else {
+ AMRecord amRecord = getAMRecord(childData);
+ if (amRecord != null) {
+ LOG.info("AM updated data: {}. Notifying {} listeners.", amRecord, getListeners().size());
+ amRecordCache.put(amRecord.getApplicationId().toString(), amRecord);
+ notifyOnAdded(amRecord);
+ }
+ }
+ break;
+ case NODE_REMOVED:
+ if (isEmpty(childData)) {
+ LOG.info("Unused AppId unregistered: {}", childData.getPath());
+ } else {
+ AMRecord amRecord = getAMRecord(childData);
+ if (amRecord != null) {
+ LOG.info("AM removed: {}. Notifying {} listeners.", amRecord, getListeners().size());
+ amRecordCache.remove(amRecord.getApplicationId().toString(), amRecord);
Review Comment:
refactored amRecordCache collection key to ApplicationId according to other comments, so this problem is resolved
no matter the AMRecord instance, the encapsulated ApplicationId.equals() takes care of the correct behavior:
```
public boolean equals(Object obj) {
if (this == obj)
return true;
if (obj == null)
return false;
if (getClass() != obj.getClass())
return false;
ApplicationId other = (ApplicationId) obj;
if (this.getClusterTimestamp() != other.getClusterTimestamp())
return false;
if (this.getId() != other.getId())
return false;
return true;
}
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] TEZ-4007: Introduce AmExtensions and Zookeeper-based FrameworkServices [tez]
Posted by "abstractdog (via GitHub)" <gi...@apache.org>.
abstractdog commented on code in PR #427:
URL: https://github.com/apache/tez/pull/427#discussion_r2565154649
##########
tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java:
##########
@@ -1612,19 +1613,14 @@ DAGState initializeDAG() {
addVertex(v);
}
- // check task resources, only check it in non-local mode
if (!appContext.isLocal()) {
- for (Vertex v : vertexMap.values()) {
- // TODO TEZ-2003 (post) TEZ-2624 Ideally, this should be per source.
- if (v.getTaskResource().compareTo(appContext.getClusterInfo().getMaxContainerCapability()) > 0) {
- String msg = "Vertex's TaskResource is beyond the cluster container capability," +
- "Vertex=" + v.getLogIdentifier() +", Requested TaskResource=" + v.getTaskResource()
- + ", Cluster MaxContainerCapability=" + appContext.getClusterInfo().getMaxContainerCapability();
- LOG.error(msg);
- addDiagnostic(msg);
- finished(DAGState.FAILED);
- return DAGState.FAILED;
- }
+ try {
+ appContext.getAmExtensions().checkTaskResources(vertexMap, appContext.getClusterInfo());
+ } catch (TaskResourceException e) {
Review Comment:
ack, also the specialized TaskResourceException exception can be removed, it has no added value at the moment
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] [DRAFT] TEZ-4007: Zookeeper based FrameworkServices and AmExtensions (3/3) - initial patch [tez]
Posted by "okumin (via GitHub)" <gi...@apache.org>.
okumin commented on code in PR #427:
URL: https://github.com/apache/tez/pull/427#discussion_r2365026800
##########
tez-api/src/main/java/org/apache/tez/client/registry/zookeeper/ZkAMRegistryClient.java:
##########
@@ -0,0 +1,188 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.tez.client.registry.zookeeper;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
+
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.imps.CuratorFrameworkState;
+import org.apache.curator.framework.recipes.cache.ChildData;
+import org.apache.curator.framework.recipes.cache.PathChildrenCache;
+import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent;
+import org.apache.curator.framework.recipes.cache.PathChildrenCacheListener;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.registry.client.binding.RegistryUtils;
+import org.apache.hadoop.registry.client.types.ServiceRecord;
+import org.apache.tez.client.registry.AMRecord;
+import org.apache.tez.client.registry.AMRegistryClient;
+import org.apache.tez.client.registry.AMRegistryClientListener;
+import org.apache.tez.dag.api.TezConfiguration;
+
+import com.google.common.base.Preconditions;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Curator/Zookeeper impl of AMRegistryClient
+*/
+@InterfaceAudience.Public
+public class ZkAMRegistryClient extends AMRegistryClient {
Review Comment:
Thanks. Background: I wonder if it can be independent of ZooKeeper technically. The reliability and fault-tolerance of ZK are beneficial in many environments, but in other cases, high availability is not necessary. Trino does not provide fault-tolerance with its coordinator, and some users appreciate the ease of use.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] [DRAFT] TEZ-4007: Zookeeper based FrameworkServices and AmExtensions (3/3) - initial patch [tez]
Posted by "abstractdog (via GitHub)" <gi...@apache.org>.
abstractdog commented on code in PR #427:
URL: https://github.com/apache/tez/pull/427#discussion_r2342367119
##########
tez-api/src/main/java/org/apache/tez/client/registry/zookeeper/ZkAMRegistryClient.java:
##########
@@ -0,0 +1,188 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.tez.client.registry.zookeeper;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
+
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.imps.CuratorFrameworkState;
+import org.apache.curator.framework.recipes.cache.ChildData;
+import org.apache.curator.framework.recipes.cache.PathChildrenCache;
+import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent;
+import org.apache.curator.framework.recipes.cache.PathChildrenCacheListener;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.registry.client.binding.RegistryUtils;
+import org.apache.hadoop.registry.client.types.ServiceRecord;
+import org.apache.tez.client.registry.AMRecord;
+import org.apache.tez.client.registry.AMRegistryClient;
+import org.apache.tez.client.registry.AMRegistryClientListener;
+import org.apache.tez.dag.api.TezConfiguration;
+
+import com.google.common.base.Preconditions;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Curator/Zookeeper impl of AMRegistryClient
+*/
+@InterfaceAudience.Public
+public class ZkAMRegistryClient extends AMRegistryClient {
Review Comment:
@okumin: both:
1. ZkAMRegistry.generateNewId takes care of generating applicationId (backed by zookeeper)
2. it also for service discovery (AM registers itself, client receives updates about that)
I'm happy to see that you're interested in the area!
please be aware that it's under refactor, I'll let you know, when it's ready
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] [DRAFT] TEZ-4007: Zookeeper based FrameworkServices and AmExtensions (3/3) - initial patch [tez]
Posted by "tez-yetus (via GitHub)" <gi...@apache.org>.
tez-yetus commented on PR #427:
URL: https://github.com/apache/tez/pull/427#issuecomment-3398513536
:broken_heart: **-1 overall**
| Vote | Subsystem | Runtime | Logfile | Comment |
|:----:|----------:|--------:|:--------:|:-------:|
| +0 :ok: | reexec | 0m 16s | | Docker mode activated. |
|||| _ Prechecks _ |
| +1 :green_heart: | dupname | 0m 1s | | No case conflicting files found. |
| +0 :ok: | detsecrets | 0m 0s | | detect-secrets was not available. |
| +0 :ok: | xmllint | 0m 0s | | xmllint was not available. |
| +1 :green_heart: | @author | 0m 0s | | The patch does not contain any @author tags. |
| +1 :green_heart: | test4tests | 0m 0s | | The patch appears to include 10 new or modified test files. |
|||| _ master Compile Tests _ |
| +0 :ok: | mvndep | 2m 49s | | Maven dependency ordering for branch |
| +1 :green_heart: | mvninstall | 10m 27s | | master passed |
| +1 :green_heart: | compile | 5m 55s | | master passed |
| +1 :green_heart: | checkstyle | 3m 45s | | master passed |
| +1 :green_heart: | javadoc | 4m 38s | | master passed |
| +0 :ok: | spotbugs | 2m 11s | | tez-api in master has 610 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 0m 46s | | tez-common in master has 13 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 1m 12s | | tez-runtime-library in master has 241 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 0m 58s | | tez-mapreduce in master has 129 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 0m 44s | | tez-examples in master has 2 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 1m 45s | | tez-dag in master has 785 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 6m 59s | | root in master has 2066 extant spotbugs warnings. |
|||| _ Patch Compile Tests _ |
| +0 :ok: | mvndep | 0m 16s | | Maven dependency ordering for patch |
| +1 :green_heart: | mvninstall | 6m 43s | | the patch passed |
| -1 :x: | codespell | 0m 51s | [/results-codespell.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/4/artifact/out/results-codespell.txt) | The patch generated 1 new + 28 unchanged - 0 fixed = 29 total (was 28) |
| +1 :green_heart: | compile | 6m 1s | | the patch passed |
| +1 :green_heart: | javac | 6m 1s | | the patch passed |
| -1 :x: | blanks | 0m 0s | [/blanks-eol.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/4/artifact/out/blanks-eol.txt) | The patch has 1 line(s) that end in blanks. Use git apply --whitespace=fix <<patch_file>>. Refer https://git-scm.com/docs/git-apply |
| -0 :warning: | checkstyle | 0m 27s | [/results-checkstyle-tez-api.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/4/artifact/out/results-checkstyle-tez-api.txt) | tez-api: The patch generated 48 new + 112 unchanged - 11 fixed = 160 total (was 123) |
| -0 :warning: | checkstyle | 0m 23s | [/results-checkstyle-tez-examples.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/4/artifact/out/results-checkstyle-tez-examples.txt) | tez-examples: The patch generated 6 new + 0 unchanged - 0 fixed = 6 total (was 0) |
| -0 :warning: | checkstyle | 0m 36s | [/results-checkstyle-tez-dag.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/4/artifact/out/results-checkstyle-tez-dag.txt) | tez-dag: The patch generated 5 new + 861 unchanged - 1 fixed = 866 total (was 862) |
| -0 :warning: | checkstyle | 0m 47s | [/results-checkstyle-root.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/4/artifact/out/results-checkstyle-root.txt) | root: The patch generated 59 new + 1026 unchanged - 12 fixed = 1085 total (was 1038) |
| +1 :green_heart: | javadoc | 4m 25s | | the patch passed |
| -1 :x: | spotbugs | 1m 41s | [/new-spotbugs-tez-api.html](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/4/artifact/out/new-spotbugs-tez-api.html) | tez-api generated 1 new + 610 unchanged - 0 fixed = 611 total (was 610) |
| -1 :x: | spotbugs | 2m 3s | [/new-spotbugs-tez-dag.html](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/4/artifact/out/new-spotbugs-tez-dag.html) | tez-dag generated 1 new + 785 unchanged - 0 fixed = 786 total (was 785) |
| -1 :x: | spotbugs | 7m 22s | [/new-spotbugs-root.html](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/4/artifact/out/new-spotbugs-root.html) | root generated 2 new + 2066 unchanged - 0 fixed = 2068 total (was 2066) |
|||| _ Other Tests _ |
| +1 :green_heart: | unit | 2m 32s | | tez-api in the patch passed. |
| +1 :green_heart: | unit | 0m 43s | | tez-common in the patch passed. |
| +1 :green_heart: | unit | 6m 8s | | tez-runtime-library in the patch passed. |
| +1 :green_heart: | unit | 1m 34s | | tez-mapreduce in the patch passed. |
| +1 :green_heart: | unit | 0m 31s | | tez-examples in the patch passed. |
| -1 :x: | unit | 5m 22s | [/patch-unit-tez-dag.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/4/artifact/out/patch-unit-tez-dag.txt) | tez-dag in the patch passed. |
| -1 :x: | unit | 71m 27s | [/patch-unit-root.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/4/artifact/out/patch-unit-root.txt) | root in the patch passed. |
| -1 :x: | asflicense | 3m 11s | [/results-asflicense.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/4/artifact/out/results-asflicense.txt) | The patch generated 9 ASF License warnings. |
| | | 176m 24s | | |
| Reason | Tests |
|-------:|:------|
| SpotBugs | module:tez-api |
| | Shared primitive variable "amPort" in one thread may not yield the value of the most recent write from another thread At ZkFrameworkClient.java:one thread may not yield the value of the most recent write from another thread At ZkFrameworkClient.java:[line 113] |
| SpotBugs | module:tez-dag |
| | Found reliance on default encoding in org.apache.tez.dag.api.client.registry.zookeeper.ZkAMRegistry.add(AMRecord):in org.apache.tez.dag.api.client.registry.zookeeper.ZkAMRegistry.add(AMRecord): String.getBytes() At ZkAMRegistry.java:[line 102] |
| SpotBugs | module:root |
| | Shared primitive variable "amPort" in one thread may not yield the value of the most recent write from another thread At ZkFrameworkClient.java:one thread may not yield the value of the most recent write from another thread At ZkFrameworkClient.java:[line 113] |
| | Found reliance on default encoding in org.apache.tez.dag.api.client.registry.zookeeper.ZkAMRegistry.add(AMRecord):in org.apache.tez.dag.api.client.registry.zookeeper.ZkAMRegistry.add(AMRecord): String.getBytes() At ZkAMRegistry.java:[line 102] |
| Failed junit tests | tez.dag.app.dag.impl.TestDAGRecovery |
| | tez.dag.app.rm.TestTaskScheduler |
| | tez.dag.app.dag.impl.TestDAGRecovery |
| | tez.dag.app.rm.TestTaskScheduler |
| Subsystem | Report/Notes |
|----------:|:-------------|
| Docker | ClientAPI=1.51 ServerAPI=1.51 base: https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/4/artifact/out/Dockerfile |
| GITHUB PR | https://github.com/apache/tez/pull/427 |
| Optional Tests | dupname asflicense javac javadoc unit codespell detsecrets xmllint compile spotbugs checkstyle |
| uname | Linux 1b3d570c1ad5 5.15.0-152-generic #162-Ubuntu SMP Wed Jul 23 09:48:42 UTC 2025 x86_64 x86_64 x86_64 GNU/Linux |
| Build tool | maven |
| Personality | /home/jenkins/jenkins-home/workspace/tez-multibranch_PR-427/src/.yetus/personality.sh |
| git revision | master / a32a6f5de1eabccb6935f94b12fb9dded224750c |
| Default Java | Ubuntu-21.0.8+9-Ubuntu-0ubuntu124.04.1 |
| Test Results | https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/4/testReport/ |
| Max. process+thread count | 1404 (vs. ulimit of 5500) |
| modules | C: tez-api tez-common tez-runtime-library tez-mapreduce tez-examples tez-dag . U: . |
| Console output | https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/4/console |
| versions | git=2.43.0 maven=3.8.7 spotbugs=4.9.3 codespell=2.0.0 |
| Powered by | Apache Yetus 0.15.1 https://yetus.apache.org |
This message was automatically generated.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] TEZ-4007: Introduce AmExtensions and Zookeeper-based FrameworkServices [tez]
Posted by "abstractdog (via GitHub)" <gi...@apache.org>.
abstractdog commented on PR #427:
URL: https://github.com/apache/tez/pull/427#issuecomment-3748284022
> hm, something happened to spotbugs, even the latest version didn't help: [0f9a678](https://github.com/apache/tez/commit/0f9a6789796d2e66ed16363e9ed4c30f205a52b5)
created [TEZ-4669](https://issues.apache.org/jira/browse/TEZ-4669) about this, assuming spotbugs to be clean with this patch according to earlier runs
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] TEZ-4007: Zookeeper based FrameworkServices and AmExtensions (3/3) including refactoring [tez]
Posted by "tez-yetus (via GitHub)" <gi...@apache.org>.
tez-yetus commented on PR #427:
URL: https://github.com/apache/tez/pull/427#issuecomment-3476924549
:broken_heart: **-1 overall**
| Vote | Subsystem | Runtime | Logfile | Comment |
|:----:|----------:|--------:|:--------:|:-------:|
| +0 :ok: | reexec | 0m 17s | | Docker mode activated. |
|||| _ Prechecks _ |
| +1 :green_heart: | dupname | 0m 1s | | No case conflicting files found. |
| +0 :ok: | detsecrets | 0m 0s | | detect-secrets was not available. |
| +0 :ok: | xmllint | 0m 0s | | xmllint was not available. |
| +1 :green_heart: | @author | 0m 0s | | The patch does not contain any @author tags. |
| +1 :green_heart: | test4tests | 0m 0s | | The patch appears to include 13 new or modified test files. |
|||| _ master Compile Tests _ |
| +0 :ok: | mvndep | 2m 56s | | Maven dependency ordering for branch |
| +1 :green_heart: | mvninstall | 10m 58s | | master passed |
| +1 :green_heart: | compile | 5m 57s | | master passed |
| +1 :green_heart: | checkstyle | 3m 46s | | master passed |
| +1 :green_heart: | javadoc | 4m 39s | | master passed |
| +0 :ok: | spotbugs | 2m 15s | | tez-api in master has 610 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 0m 44s | | tez-common in master has 13 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 1m 10s | | tez-runtime-library in master has 241 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 0m 57s | | tez-mapreduce in master has 129 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 0m 44s | | tez-examples in master has 2 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 1m 45s | | tez-dag in master has 785 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 6m 54s | | root in master has 2066 extant spotbugs warnings. |
|||| _ Patch Compile Tests _ |
| +0 :ok: | mvndep | 0m 17s | | Maven dependency ordering for patch |
| +1 :green_heart: | mvninstall | 6m 42s | | the patch passed |
| -1 :x: | codespell | 0m 50s | [/results-codespell.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/19/artifact/out/results-codespell.txt) | The patch generated 1 new + 30 unchanged - 0 fixed = 31 total (was 30) |
| +1 :green_heart: | compile | 5m 57s | | the patch passed |
| +1 :green_heart: | javac | 5m 57s | | the patch passed |
| -1 :x: | blanks | 0m 0s | [/blanks-eol.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/19/artifact/out/blanks-eol.txt) | The patch has 4 line(s) that end in blanks. Use git apply --whitespace=fix <<patch_file>>. Refer https://git-scm.com/docs/git-apply |
| +1 :green_heart: | checkstyle | 0m 27s | | tez-api: The patch generated 0 new + 142 unchanged - 3 fixed = 142 total (was 145) |
| +1 :green_heart: | checkstyle | 0m 24s | | The patch passed checkstyle in tez-common |
| -0 :warning: | checkstyle | 0m 27s | [/results-checkstyle-tez-runtime-library.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/19/artifact/out/results-checkstyle-tez-runtime-library.txt) | tez-runtime-library: The patch generated 4 new + 0 unchanged - 0 fixed = 4 total (was 0) |
| +1 :green_heart: | checkstyle | 0m 23s | | The patch passed checkstyle in tez-mapreduce |
| -0 :warning: | checkstyle | 0m 21s | [/results-checkstyle-tez-examples.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/19/artifact/out/results-checkstyle-tez-examples.txt) | tez-examples: The patch generated 19 new + 10 unchanged - 1 fixed = 29 total (was 11) |
| -0 :warning: | checkstyle | 0m 34s | [/results-checkstyle-tez-dag.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/19/artifact/out/results-checkstyle-tez-dag.txt) | tez-dag: The patch generated 1 new + 842 unchanged - 2 fixed = 843 total (was 844) |
| -0 :warning: | checkstyle | 0m 46s | [/results-checkstyle-root.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/19/artifact/out/results-checkstyle-root.txt) | root: The patch generated 24 new + 996 unchanged - 6 fixed = 1020 total (was 1002) |
| +1 :green_heart: | javadoc | 4m 32s | | the patch passed |
| +1 :green_heart: | spotbugs | 15m 26s | | the patch passed |
|||| _ Other Tests _ |
| +1 :green_heart: | unit | 2m 35s | | tez-api in the patch passed. |
| +1 :green_heart: | unit | 0m 43s | | tez-common in the patch passed. |
| +1 :green_heart: | unit | 6m 4s | | tez-runtime-library in the patch passed. |
| +1 :green_heart: | unit | 1m 36s | | tez-mapreduce in the patch passed. |
| +1 :green_heart: | unit | 0m 31s | | tez-examples in the patch passed. |
| +1 :green_heart: | unit | 5m 20s | | tez-dag in the patch passed. |
| +1 :green_heart: | unit | 72m 20s | | root in the patch passed. |
| -1 :x: | asflicense | 3m 11s | [/results-asflicense.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/19/artifact/out/results-asflicense.txt) | The patch generated 5 ASF License warnings. |
| | | 177m 39s | | |
| Subsystem | Report/Notes |
|----------:|:-------------|
| Docker | ClientAPI=1.51 ServerAPI=1.51 base: https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/19/artifact/out/Dockerfile |
| GITHUB PR | https://github.com/apache/tez/pull/427 |
| Optional Tests | dupname asflicense javac javadoc unit codespell detsecrets xmllint compile spotbugs checkstyle |
| uname | Linux 0628bc559c0a 5.15.0-152-generic #162-Ubuntu SMP Wed Jul 23 09:48:42 UTC 2025 x86_64 x86_64 x86_64 GNU/Linux |
| Build tool | maven |
| Personality | /home/jenkins/jenkins-home/workspace/tez-multibranch_PR-427/src/.yetus/personality.sh |
| git revision | master / c92ee64993d50d1139537ea1d67eacfc627021dc |
| Default Java | Ubuntu-21.0.8+9-Ubuntu-0ubuntu124.04.1 |
| Test Results | https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/19/testReport/ |
| Max. process+thread count | 1618 (vs. ulimit of 5500) |
| modules | C: tez-api tez-common tez-runtime-library tez-mapreduce tez-examples tez-dag . U: . |
| Console output | https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/19/console |
| versions | git=2.43.0 maven=3.8.7 spotbugs=4.9.3 codespell=2.0.0 |
| Powered by | Apache Yetus 0.15.1 https://yetus.apache.org |
This message was automatically generated.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] TEZ-4007: Introduce AmExtensions and Zookeeper-based FrameworkServices [tez]
Posted by "abstractdog (via GitHub)" <gi...@apache.org>.
abstractdog commented on code in PR #427:
URL: https://github.com/apache/tez/pull/427#discussion_r2551051434
##########
tez-api/src/main/java/org/apache/tez/client/registry/zookeeper/ZkAMRegistryClient.java:
##########
@@ -0,0 +1,205 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.client.registry.zookeeper;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
+
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.imps.CuratorFrameworkState;
+import org.apache.curator.framework.recipes.cache.ChildData;
+import org.apache.curator.framework.recipes.cache.TreeCache;
+import org.apache.curator.framework.recipes.cache.TreeCacheEvent;
+import org.apache.curator.framework.recipes.cache.TreeCacheListener;
+import org.apache.curator.shaded.com.google.common.base.Charsets;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.registry.client.binding.RegistryUtils;
+import org.apache.hadoop.registry.client.types.ServiceRecord;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.tez.client.registry.AMRecord;
+import org.apache.tez.client.registry.AMRegistryClient;
+import org.apache.tez.client.registry.AMRegistryClientListener;
+import org.apache.tez.dag.api.TezConfiguration;
+
+import com.fasterxml.jackson.core.JsonParseException;
+import com.google.common.base.Preconditions;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Curator/Zookeeper implementation of {@link AMRegistryClient}.
+ */
+@InterfaceAudience.Public
+public final class ZkAMRegistryClient extends AMRegistryClient {
+ private static final Logger LOG = LoggerFactory.getLogger(ZkAMRegistryClient.class);
+ private static final Map<String, ZkAMRegistryClient> INSTANCES = new HashMap<>();
+
+ private final Configuration conf;
+ //Cache of known AMs
+ private final ConcurrentHashMap<String, AMRecord> amRecordCache = new ConcurrentHashMap<>();
+ private CuratorFramework client;
+
+ private ZkAMRegistryClient(final Configuration conf) {
+ this.conf = conf;
+ }
+
+ public static synchronized ZkAMRegistryClient getClient(final Configuration conf) {
+ String namespace = conf.get(TezConfiguration.TEZ_AM_REGISTRY_NAMESPACE);
+ ZkAMRegistryClient registry = INSTANCES.get(namespace);
+ if (registry == null) {
+ registry = new ZkAMRegistryClient(conf);
+ INSTANCES.put(namespace, registry);
+ }
+ LOG.info("Returning tez AM registry ({}) for namespace '{}'", System.identityHashCode(registry), namespace);
+ return registry;
+ }
+
+ /**
+ * Deserializes a {@link ServiceRecord} from ZooKeeper data and converts it into an {@link AMRecord}
+ * for caching.
+ *
+ * @param childData the ZooKeeper node data containing a serialized {@link ServiceRecord}
+ * @return an {@link AMRecord} constructed from the deserialized {@link ServiceRecord}, or {@code null}
+ * if no data is present
+ * @throws IOException if the data cannot be deserialized into a {@link ServiceRecord}
+ */
+ public static AMRecord getAMRecord(final ChildData childData) throws IOException {
+ // not a leaf path. Only leaf path contains AMRecord
+ if (!childData.getPath().contains(ApplicationId.appIdStrPrefix)) {
+ return null;
+ }
+ byte[] data = childData.getData();
+ // only the path appeared, there is no data yet
+ if (data.length == 0) {
+ return null;
+ }
+ String value = new String(data, Charsets.UTF_8);
+ try {
+ RegistryUtils.ServiceRecordMarshal marshal = new RegistryUtils.ServiceRecordMarshal();
+ ServiceRecord serviceRecord = marshal.fromJson(value);
+ return new AMRecord(serviceRecord);
+ } catch (JsonParseException e) {
+ //Not a json AMRecord (SRV), could be some other data.
+ LOG.warn("Non-json data received while de-serializing AMRecord: {}. Ignoring...", value);
+ return null;
+ }
+ }
+
+ public void start() throws Exception {
+ ZkConfig zkConf = new ZkConfig(this.conf);
+ client = zkConf.createCuratorFramework();
+ final TreeCache cache = new TreeCache(client, zkConf.getZkNamespace());
+ client.start();
+ cache.start();
+ cache.getListenable().addListener(new ZkRegistryListener());
+ }
+
+ @Override
+ public AMRecord getRecord(String appId) {
+ if (amRecordCache.get(appId) == null) {
+ return null;
+ }
+ //Return a copy
+ return new AMRecord(amRecordCache.get(appId));
+ }
+
+ @Override
+ public List<AMRecord> getAllRecords() {
+ return amRecordCache.values().stream().map(AMRecord::new).collect(Collectors.toList());
+ }
+
+ @Override
+ public synchronized void addListener(AMRegistryClientListener listener) {
+ getListeners().add(listener);
+ }
+
+ @Override
+ public void close() {
+ client.close();
+ }
+
+ /**
+ * Callback listener for ZooKeeper events that updates the local cache
+ * when child nodes under the monitored path change.
+ */
+ private class ZkRegistryListener implements TreeCacheListener {
+
+ @Override
+ public void childEvent(final CuratorFramework clientParam, final TreeCacheEvent event) throws Exception {
+ Preconditions.checkArgument(clientParam != null && clientParam.getState() == CuratorFrameworkState.STARTED,
+ "Curator client is not started");
+
+ ChildData childData = event.getData();
+ switch (event.getType()) {
+ case NODE_ADDED:
+ if (isEmpty(childData)) {
+ LOG.info("AppId allocated: {}", childData.getPath());
+ } else {
+ AMRecord amRecord = getAMRecord(childData);
+ if (amRecord != null) {
+ LOG.info("AM registered with data: {}. Notifying {} listeners.", amRecord, getListeners().size());
+ amRecordCache.put(amRecord.getApplicationId().toString(), amRecord);
+ notifyOnAdded(amRecord);
+ }
+ }
+ break;
+ case NODE_UPDATED:
+ if (isEmpty(childData)) {
+ throw new RuntimeException("AM updated with empty data");
+ } else {
+ AMRecord amRecord = getAMRecord(childData);
+ if (amRecord != null) {
+ LOG.info("AM updated data: {}. Notifying {} listeners.", amRecord, getListeners().size());
Review Comment:
these messages are essential at the INFO level because they provide basic operational information about AMs coming and going
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] TEZ-4007: Introduce AmExtensions and Zookeeper-based FrameworkServices [tez]
Posted by "tez-yetus (via GitHub)" <gi...@apache.org>.
tez-yetus commented on PR #427:
URL: https://github.com/apache/tez/pull/427#issuecomment-3558453040
:broken_heart: **-1 overall**
| Vote | Subsystem | Runtime | Logfile | Comment |
|:----:|----------:|--------:|:--------:|:-------:|
| +0 :ok: | reexec | 0m 37s | | Docker mode activated. |
|||| _ Prechecks _ |
| +1 :green_heart: | dupname | 0m 0s | | No case conflicting files found. |
| +0 :ok: | detsecrets | 0m 0s | | detect-secrets was not available. |
| +0 :ok: | xmllint | 0m 0s | | xmllint was not available. |
| +1 :green_heart: | @author | 0m 0s | | The patch does not contain any @author tags. |
| +1 :green_heart: | test4tests | 0m 0s | | The patch appears to include 11 new or modified test files. |
|||| _ master Compile Tests _ |
| +0 :ok: | mvndep | 2m 32s | | Maven dependency ordering for branch |
| +1 :green_heart: | mvninstall | 10m 15s | | master passed |
| +1 :green_heart: | compile | 5m 1s | | master passed |
| +1 :green_heart: | checkstyle | 2m 51s | | master passed |
| +1 :green_heart: | javadoc | 3m 38s | | master passed |
| +0 :ok: | spotbugs | 2m 6s | | tez-api in master has 610 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 0m 41s | | tez-common in master has 13 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 1m 8s | | tez-runtime-library in master has 241 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 0m 40s | | tez-examples in master has 2 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 1m 41s | | tez-dag in master has 785 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 6m 54s | | root in master has 2066 extant spotbugs warnings. |
| -0 :warning: | patch | 7m 37s | | Used diff version of patch file. Binary files and potentially other changes not applied. Please rebase and squash commits if necessary. |
|||| _ Patch Compile Tests _ |
| +0 :ok: | mvndep | 0m 16s | | Maven dependency ordering for patch |
| +1 :green_heart: | mvninstall | 6m 38s | | the patch passed |
| +1 :green_heart: | codespell | 0m 51s | | No new issues. |
| +1 :green_heart: | compile | 5m 6s | | the patch passed |
| +1 :green_heart: | javac | 5m 6s | | the patch passed |
| +1 :green_heart: | blanks | 0m 0s | | The patch has no blanks issues. |
| -0 :warning: | checkstyle | 0m 22s | [/results-checkstyle-tez-api.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/25/artifact/out/results-checkstyle-tez-api.txt) | tez-api: The patch generated 5 new + 118 unchanged - 3 fixed = 123 total (was 121) |
| +1 :green_heart: | checkstyle | 0m 19s | | The patch passed checkstyle in tez-common |
| +1 :green_heart: | checkstyle | 0m 23s | | The patch passed checkstyle in tez-runtime-library |
| +1 :green_heart: | checkstyle | 0m 18s | | tez-examples: The patch generated 0 new + 10 unchanged - 1 fixed = 10 total (was 11) |
| -0 :warning: | checkstyle | 0m 31s | [/results-checkstyle-tez-dag.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/25/artifact/out/results-checkstyle-tez-dag.txt) | tez-dag: The patch generated 1 new + 610 unchanged - 2 fixed = 611 total (was 612) |
| -0 :warning: | checkstyle | 0m 40s | [/results-checkstyle-root.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/25/artifact/out/results-checkstyle-root.txt) | root: The patch generated 6 new + 740 unchanged - 6 fixed = 746 total (was 746) |
| +1 :green_heart: | javadoc | 3m 30s | | the patch passed |
| -1 :x: | spotbugs | 1m 41s | [/new-spotbugs-tez-api.html](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/25/artifact/out/new-spotbugs-tez-api.html) | tez-api generated 1 new + 610 unchanged - 0 fixed = 611 total (was 610) |
| -1 :x: | spotbugs | 7m 19s | [/new-spotbugs-root.html](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/25/artifact/out/new-spotbugs-root.html) | root generated 1 new + 2066 unchanged - 0 fixed = 2067 total (was 2066) |
|||| _ Other Tests _ |
| +1 :green_heart: | unit | 2m 30s | | tez-api in the patch passed. |
| +1 :green_heart: | unit | 0m 39s | | tez-common in the patch passed. |
| +1 :green_heart: | unit | 6m 1s | | tez-runtime-library in the patch passed. |
| +1 :green_heart: | unit | 0m 28s | | tez-examples in the patch passed. |
| +1 :green_heart: | unit | 5m 20s | | tez-dag in the patch passed. |
| +1 :green_heart: | unit | 72m 45s | | root in the patch passed. |
| +1 :green_heart: | asflicense | 2m 13s | | The patch does not generate ASF License warnings. |
| | | 166m 3s | | |
| Reason | Tests |
|-------:|:------|
| SpotBugs | module:tez-api |
| | new org.apache.tez.client.registry.AMRecord(ApplicationId, String, String, int, String, String) may expose internal representation by storing an externally mutable object into AMRecord.appId At AMRecord.java:String, String) may expose internal representation by storing an externally mutable object into AMRecord.appId At AMRecord.java:[line 70] |
| SpotBugs | module:root |
| | new org.apache.tez.client.registry.AMRecord(ApplicationId, String, String, int, String, String) may expose internal representation by storing an externally mutable object into AMRecord.appId At AMRecord.java:String, String) may expose internal representation by storing an externally mutable object into AMRecord.appId At AMRecord.java:[line 70] |
| Subsystem | Report/Notes |
|----------:|:-------------|
| Docker | ClientAPI=1.52 ServerAPI=1.52 base: https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/25/artifact/out/Dockerfile |
| GITHUB PR | https://github.com/apache/tez/pull/427 |
| Optional Tests | dupname asflicense javac javadoc unit codespell detsecrets xmllint compile spotbugs checkstyle |
| uname | Linux 837cfebd92eb 5.15.0-161-generic #171-Ubuntu SMP Sat Oct 11 08:17:01 UTC 2025 x86_64 x86_64 x86_64 GNU/Linux |
| Build tool | maven |
| Personality | /home/jenkins/jenkins-home/workspace/tez-multibranch_PR-427/src/.yetus/personality.sh |
| git revision | master / 021a52bdaad186f82c5675b3c724920dceaef0c8 |
| Default Java | Ubuntu-21.0.8+9-Ubuntu-0ubuntu124.04.1 |
| Test Results | https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/25/testReport/ |
| Max. process+thread count | 1350 (vs. ulimit of 5500) |
| modules | C: tez-api tez-common tez-runtime-library tez-examples tez-dag . U: . |
| Console output | https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/25/console |
| versions | git=2.43.0 maven=3.8.7 spotbugs=4.9.3 codespell=2.0.0 |
| Powered by | Apache Yetus 0.15.1 https://yetus.apache.org |
This message was automatically generated.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] TEZ-4007: Introduce AmExtensions and Zookeeper-based FrameworkServices [tez]
Posted by "abstractdog (via GitHub)" <gi...@apache.org>.
abstractdog commented on code in PR #427:
URL: https://github.com/apache/tez/pull/427#discussion_r2549694323
##########
tez-api/src/main/java/org/apache/tez/client/registry/AMRegistry.java:
##########
@@ -0,0 +1,104 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.client.registry;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.service.ServiceStateException;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Base class for {@code AMRegistry} implementations.
+ *
+ * <p>The specific implementation is configured via the
+ * {@code tez.am.registry.class} property.</p>
+ *
+ * <p>Implementations are expected to provide appropriate service lifecycle
+ * behavior, including:
+ * <ul>
+ * <li>{@code init}</li>
+ * <li>{@code serviceStart}</li>
+ * <li>{@code serviceStop}</li>
+ * </ul>
+ * </p>
+ *
+ * <p>{@code init} and {@code serviceStart} are invoked during
+ * {@code DAGAppMaster.serviceInit()}, while {@code serviceStop} is called
+ * when {@code DAGAppMaster} shuts down.</p>
+ */
+public abstract class AMRegistry extends AbstractService {
+
+ private static final Logger LOG = LoggerFactory.getLogger(AMRegistry.class);
+ private List<AMRecord> amRecords = new ArrayList<>();
Review Comment:
also, there is a private amRecords in the ZkAMRegistry subclass, which makes it really confusing, let me cleanup this codepath
EDIT: moving the collection to the subclass, making a final, synchronized collection
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] TEZ-4007: Introduce AmExtensions and Zookeeper-based FrameworkServices [tez]
Posted by "abstractdog (via GitHub)" <gi...@apache.org>.
abstractdog commented on code in PR #427:
URL: https://github.com/apache/tez/pull/427#discussion_r2549684127
##########
tez-api/src/main/java/org/apache/tez/client/registry/AMRegistry.java:
##########
@@ -0,0 +1,104 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.client.registry;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.service.AbstractService;
+import org.apache.hadoop.service.ServiceStateException;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Base class for {@code AMRegistry} implementations.
+ *
+ * <p>The specific implementation is configured via the
+ * {@code tez.am.registry.class} property.</p>
+ *
+ * <p>Implementations are expected to provide appropriate service lifecycle
+ * behavior, including:
+ * <ul>
+ * <li>{@code init}</li>
+ * <li>{@code serviceStart}</li>
+ * <li>{@code serviceStop}</li>
+ * </ul>
+ * </p>
+ *
+ * <p>{@code init} and {@code serviceStart} are invoked during
+ * {@code DAGAppMaster.serviceInit()}, while {@code serviceStop} is called
+ * when {@code DAGAppMaster} shuts down.</p>
+ */
+public abstract class AMRegistry extends AbstractService {
+
+ private static final Logger LOG = LoggerFactory.getLogger(AMRegistry.class);
+ private List<AMRecord> amRecords = new ArrayList<>();
+
+ @Override
+ public void init(Configuration conf) {
+ try {
+ this.serviceInit(conf);
+ } catch (Exception e) {
+ LOG.error("Failed to init AMRegistry: name={}, type={}", getName(), getClass().getName());
+ throw ServiceStateException.convert(e);
+ }
+ }
+
+ @Override
+ public void start() {
+ try {
+ this.serviceStart();
+ } catch(Exception e) {
+ LOG.error("Failed to start AMRegistry: name={}, type={}", getName(), getClass().getName());
+ throw ServiceStateException.convert(e);
+ }
+ }
Review Comment:
right, converting the exceptions to ServiceStateException doesn't add obvious value here anyway, I'm removing these overrides
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] TEZ-4007: Introduce AmExtensions and Zookeeper-based FrameworkServices [tez]
Posted by "tez-yetus (via GitHub)" <gi...@apache.org>.
tez-yetus commented on PR #427:
URL: https://github.com/apache/tez/pull/427#issuecomment-3596352596
:confetti_ball: **+1 overall**
| Vote | Subsystem | Runtime | Logfile | Comment |
|:----:|----------:|--------:|:--------:|:-------:|
| +0 :ok: | reexec | 13m 54s | | Docker mode activated. |
|||| _ Prechecks _ |
| +1 :green_heart: | dupname | 0m 1s | | No case conflicting files found. |
| +0 :ok: | detsecrets | 0m 0s | | detect-secrets was not available. |
| +0 :ok: | xmllint | 0m 0s | | xmllint was not available. |
| +1 :green_heart: | @author | 0m 0s | | The patch does not contain any @author tags. |
| +1 :green_heart: | test4tests | 0m 0s | | The patch appears to include 14 new or modified test files. |
|||| _ master Compile Tests _ |
| +0 :ok: | mvndep | 2m 27s | | Maven dependency ordering for branch |
| +1 :green_heart: | mvninstall | 7m 47s | | master passed |
| +1 :green_heart: | compile | 2m 54s | | master passed |
| +1 :green_heart: | checkstyle | 1m 45s | | master passed |
| +1 :green_heart: | javadoc | 2m 11s | | master passed |
| +0 :ok: | spotbugs | 1m 28s | | tez-api in master has 610 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 0m 27s | | tez-common in master has 13 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 0m 38s | | tez-runtime-library in master has 241 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 0m 26s | | tez-examples in master has 2 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 0m 53s | | tez-dag in master has 785 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 3m 40s | | root in master has 2066 extant spotbugs warnings. |
| -0 :warning: | patch | 4m 13s | | Used diff version of patch file. Binary files and potentially other changes not applied. Please rebase and squash commits if necessary. |
|||| _ Patch Compile Tests _ |
| +0 :ok: | mvndep | 0m 10s | | Maven dependency ordering for patch |
| +1 :green_heart: | mvninstall | 3m 41s | | the patch passed |
| +1 :green_heart: | codespell | 0m 29s | | No new issues. |
| +1 :green_heart: | compile | 2m 56s | | the patch passed |
| +1 :green_heart: | javac | 2m 56s | | the patch passed |
| +1 :green_heart: | blanks | 0m 0s | | The patch has no blanks issues. |
| +1 :green_heart: | checkstyle | 0m 13s | | tez-api: The patch generated 0 new + 117 unchanged - 3 fixed = 117 total (was 120) |
| +1 :green_heart: | checkstyle | 0m 12s | | The patch passed checkstyle in tez-common |
| +1 :green_heart: | checkstyle | 0m 14s | | The patch passed checkstyle in tez-runtime-library |
| +1 :green_heart: | checkstyle | 0m 12s | | tez-examples: The patch generated 0 new + 11 unchanged - 1 fixed = 11 total (was 12) |
| -0 :warning: | checkstyle | 0m 18s | [/results-checkstyle-tez-dag.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/33/artifact/out/results-checkstyle-tez-dag.txt) | tez-dag: The patch generated 1 new + 610 unchanged - 2 fixed = 611 total (was 612) |
| -0 :warning: | checkstyle | 0m 21s | [/results-checkstyle-root.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/33/artifact/out/results-checkstyle-root.txt) | root: The patch generated 1 new + 740 unchanged - 6 fixed = 741 total (was 746) |
| +1 :green_heart: | javadoc | 2m 15s | | the patch passed |
| +1 :green_heart: | spotbugs | 7m 37s | | the patch passed |
|||| _ Other Tests _ |
| +1 :green_heart: | unit | 2m 2s | | tez-api in the patch passed. |
| +1 :green_heart: | unit | 0m 27s | | tez-common in the patch passed. |
| +1 :green_heart: | unit | 4m 16s | | tez-runtime-library in the patch passed. |
| +1 :green_heart: | unit | 0m 18s | | tez-examples in the patch passed. |
| +1 :green_heart: | unit | 4m 52s | | tez-dag in the patch passed. |
| +1 :green_heart: | unit | 59m 20s | | root in the patch passed. |
| +1 :green_heart: | asflicense | 1m 43s | | The patch does not generate ASF License warnings. |
| | | 133m 34s | | |
| Subsystem | Report/Notes |
|----------:|:-------------|
| Docker | ClientAPI=1.52 ServerAPI=1.52 base: https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/33/artifact/out/Dockerfile |
| GITHUB PR | https://github.com/apache/tez/pull/427 |
| Optional Tests | dupname asflicense javac javadoc unit codespell detsecrets xmllint compile spotbugs checkstyle |
| uname | Linux d19566a5022f 5.15.0-156-generic #166-Ubuntu SMP Sat Aug 9 00:02:46 UTC 2025 x86_64 x86_64 x86_64 GNU/Linux |
| Build tool | maven |
| Personality | /home/jenkins/jenkins-home/workspace/tez-multibranch_PR-427/src/.yetus/personality.sh |
| git revision | master / 6bf4f1f242a6d46938828b76c798d6a38ca8b842 |
| Default Java | Ubuntu-21.0.9+10-Ubuntu-124.04 |
| Test Results | https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/33/testReport/ |
| Max. process+thread count | 2109 (vs. ulimit of 5500) |
| modules | C: tez-api tez-common tez-runtime-library tez-examples tez-dag . U: . |
| Console output | https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/33/console |
| versions | git=2.43.0 maven=3.8.7 spotbugs=4.9.3 codespell=2.0.0 |
| Powered by | Apache Yetus 0.15.1 https://yetus.apache.org |
This message was automatically generated.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] TEZ-4007: Introduce AmExtensions and Zookeeper-based FrameworkServices [tez]
Posted by "tez-yetus (via GitHub)" <gi...@apache.org>.
tez-yetus commented on PR #427:
URL: https://github.com/apache/tez/pull/427#issuecomment-3732388634
:broken_heart: **-1 overall**
| Vote | Subsystem | Runtime | Logfile | Comment |
|:----:|----------:|--------:|:--------:|:-------:|
| +0 :ok: | reexec | 0m 28s | | Docker mode activated. |
|||| _ Prechecks _ |
| +1 :green_heart: | dupname | 0m 1s | | No case conflicting files found. |
| +0 :ok: | detsecrets | 0m 0s | | detect-secrets was not available. |
| +0 :ok: | xmllint | 0m 0s | | xmllint was not available. |
| +1 :green_heart: | @author | 0m 0s | | The patch does not contain any @author tags. |
| +1 :green_heart: | test4tests | 0m 0s | | The patch appears to include 14 new or modified test files. |
|||| _ master Compile Tests _ |
| +0 :ok: | mvndep | 2m 27s | | Maven dependency ordering for branch |
| +1 :green_heart: | mvninstall | 9m 32s | | master passed |
| +1 :green_heart: | compile | 5m 25s | | master passed |
| +1 :green_heart: | checkstyle | 3m 16s | | master passed |
| +1 :green_heart: | javadoc | 4m 6s | | master passed |
| -1 :x: | spotbugs | 1m 17s | [/branch-spotbugs-tez-api.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/39/artifact/out/branch-spotbugs-tez-api.txt) | tez-api in master failed. |
| -1 :x: | spotbugs | 0m 31s | [/branch-spotbugs-tez-common.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/39/artifact/out/branch-spotbugs-tez-common.txt) | tez-common in master failed. |
| -1 :x: | spotbugs | 0m 37s | [/branch-spotbugs-tez-runtime-library.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/39/artifact/out/branch-spotbugs-tez-runtime-library.txt) | tez-runtime-library in master failed. |
| -1 :x: | spotbugs | 0m 31s | [/branch-spotbugs-tez-examples.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/39/artifact/out/branch-spotbugs-tez-examples.txt) | tez-examples in master failed. |
| -1 :x: | spotbugs | 0m 42s | [/branch-spotbugs-tez-dag.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/39/artifact/out/branch-spotbugs-tez-dag.txt) | tez-dag in master failed. |
| -1 :x: | spotbugs | 0m 24s | [/branch-spotbugs-root.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/39/artifact/out/branch-spotbugs-root.txt) | root in master failed. |
| -0 :warning: | patch | 12m 45s | | Used diff version of patch file. Binary files and potentially other changes not applied. Please rebase and squash commits if necessary. |
|||| _ Patch Compile Tests _ |
| +0 :ok: | mvndep | 0m 19s | | Maven dependency ordering for patch |
| +1 :green_heart: | mvninstall | 6m 24s | | the patch passed |
| +1 :green_heart: | codespell | 0m 51s | | No new issues. |
| +1 :green_heart: | compile | 5m 26s | | the patch passed |
| +1 :green_heart: | javac | 5m 26s | | the patch passed |
| +1 :green_heart: | blanks | 0m 0s | | The patch has no blanks issues. |
| +1 :green_heart: | checkstyle | 0m 26s | | tez-api: The patch generated 0 new + 118 unchanged - 3 fixed = 118 total (was 121) |
| +1 :green_heart: | checkstyle | 0m 24s | | The patch passed checkstyle in tez-common |
| +1 :green_heart: | checkstyle | 0m 29s | | The patch passed checkstyle in tez-runtime-library |
| +1 :green_heart: | checkstyle | 0m 22s | | tez-examples: The patch generated 0 new + 10 unchanged - 1 fixed = 10 total (was 11) |
| -0 :warning: | checkstyle | 0m 35s | [/results-checkstyle-tez-dag.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/39/artifact/out/results-checkstyle-tez-dag.txt) | tez-dag: The patch generated 2 new + 610 unchanged - 2 fixed = 612 total (was 612) |
| -0 :warning: | checkstyle | 0m 45s | [/results-checkstyle-root.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/39/artifact/out/results-checkstyle-root.txt) | root: The patch generated 2 new + 740 unchanged - 6 fixed = 742 total (was 746) |
| +1 :green_heart: | javadoc | 3m 59s | | the patch passed |
| +1 :green_heart: | spotbugs | 13m 39s | | the patch passed |
|||| _ Other Tests _ |
| +1 :green_heart: | unit | 2m 37s | | tez-api in the patch passed. |
| +1 :green_heart: | unit | 0m 43s | | tez-common in the patch passed. |
| +1 :green_heart: | unit | 6m 10s | | tez-runtime-library in the patch passed. |
| +1 :green_heart: | unit | 0m 32s | | tez-examples in the patch passed. |
| +1 :green_heart: | unit | 6m 21s | | tez-dag in the patch passed. |
| +1 :green_heart: | unit | 73m 1s | | root in the patch passed. |
| +1 :green_heart: | asflicense | 2m 47s | | The patch does not generate ASF License warnings. |
| | | 159m 21s | | |
| Subsystem | Report/Notes |
|----------:|:-------------|
| Docker | ClientAPI=1.52 ServerAPI=1.52 base: https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/39/artifact/out/Dockerfile |
| GITHUB PR | https://github.com/apache/tez/pull/427 |
| Optional Tests | dupname asflicense javac javadoc unit codespell detsecrets xmllint compile spotbugs checkstyle |
| uname | Linux dedb682cc43e 5.15.0-161-generic #171-Ubuntu SMP Sat Oct 11 08:17:01 UTC 2025 x86_64 x86_64 x86_64 GNU/Linux |
| Build tool | maven |
| Personality | /home/jenkins/jenkins-home/workspace/tez-multibranch_PR-427/src/.yetus/personality.sh |
| git revision | master / cee34c87986dc2c1a90049b65f14e1093959c8a4 |
| Default Java | Ubuntu-21.0.9+10-Ubuntu-124.04 |
| Test Results | https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/39/testReport/ |
| Max. process+thread count | 2109 (vs. ulimit of 5500) |
| modules | C: tez-api tez-common tez-runtime-library tez-examples tez-dag . U: . |
| Console output | https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/39/console |
| versions | git=2.43.0 maven=3.8.7 codespell=2.0.0 |
| Powered by | Apache Yetus 0.15.1 https://yetus.apache.org |
This message was automatically generated.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] TEZ-4007: Introduce AmExtensions and Zookeeper-based FrameworkServices [tez]
Posted by "tez-yetus (via GitHub)" <gi...@apache.org>.
tez-yetus commented on PR #427:
URL: https://github.com/apache/tez/pull/427#issuecomment-3737689145
(!) A patch to the testing environment has been detected.
Re-executing against the patched versions to perform further tests.
The console is at https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/40/console in case of problems.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] [DRAFT] TEZ-4007: Zookeeper based FrameworkServices and AmExtensions (3/3) - initial patch [tez]
Posted by "tez-yetus (via GitHub)" <gi...@apache.org>.
tez-yetus commented on PR #427:
URL: https://github.com/apache/tez/pull/427#issuecomment-3282932053
:broken_heart: **-1 overall**
| Vote | Subsystem | Runtime | Logfile | Comment |
|:----:|----------:|--------:|:--------:|:-------:|
| +0 :ok: | reexec | 23m 53s | | Docker mode activated. |
|||| _ Prechecks _ |
| +1 :green_heart: | dupname | 0m 1s | | No case conflicting files found. |
| +0 :ok: | detsecrets | 0m 0s | | detect-secrets was not available. |
| +0 :ok: | xmllint | 0m 0s | | xmllint was not available. |
| +1 :green_heart: | @author | 0m 0s | | The patch does not contain any @author tags. |
| +1 :green_heart: | test4tests | 0m 0s | | The patch appears to include 7 new or modified test files. |
|||| _ master Compile Tests _ |
| +0 :ok: | mvndep | 2m 36s | | Maven dependency ordering for branch |
| +1 :green_heart: | mvninstall | 9m 44s | | master passed |
| +1 :green_heart: | compile | 5m 59s | | master passed |
| +1 :green_heart: | checkstyle | 3m 43s | | master passed |
| +1 :green_heart: | javadoc | 4m 35s | | master passed |
| +0 :ok: | spotbugs | 2m 7s | | tez-api in master has 610 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 0m 46s | | tez-common in master has 13 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 1m 13s | | tez-runtime-library in master has 241 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 0m 57s | | tez-mapreduce in master has 129 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 0m 43s | | tez-examples in master has 2 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 1m 41s | | tez-dag in master has 785 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 6m 57s | | root in master has 2066 extant spotbugs warnings. |
|||| _ Patch Compile Tests _ |
| +0 :ok: | mvndep | 0m 16s | | Maven dependency ordering for patch |
| +1 :green_heart: | mvninstall | 6m 43s | | the patch passed |
| +1 :green_heart: | codespell | 0m 51s | | No new issues. |
| +1 :green_heart: | compile | 5m 57s | | the patch passed |
| +1 :green_heart: | javac | 5m 57s | | the patch passed |
| -1 :x: | blanks | 0m 0s | [/blanks-eol.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/2/artifact/out/blanks-eol.txt) | The patch has 1 line(s) that end in blanks. Use git apply --whitespace=fix <<patch_file>>. Refer https://git-scm.com/docs/git-apply |
| -0 :warning: | checkstyle | 0m 26s | [/results-checkstyle-tez-api.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/2/artifact/out/results-checkstyle-tez-api.txt) | tez-api: The patch generated 48 new + 112 unchanged - 11 fixed = 160 total (was 123) |
| -0 :warning: | checkstyle | 0m 27s | [/results-checkstyle-tez-runtime-library.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/2/artifact/out/results-checkstyle-tez-runtime-library.txt) | tez-runtime-library: The patch generated 1 new + 0 unchanged - 0 fixed = 1 total (was 0) |
| -0 :warning: | checkstyle | 0m 23s | [/results-checkstyle-tez-examples.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/2/artifact/out/results-checkstyle-tez-examples.txt) | tez-examples: The patch generated 6 new + 0 unchanged - 0 fixed = 6 total (was 0) |
| -0 :warning: | checkstyle | 0m 34s | [/results-checkstyle-tez-dag.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/2/artifact/out/results-checkstyle-tez-dag.txt) | tez-dag: The patch generated 29 new + 710 unchanged - 1 fixed = 739 total (was 711) |
| -0 :warning: | checkstyle | 0m 46s | [/results-checkstyle-root.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/2/artifact/out/results-checkstyle-root.txt) | root: The patch generated 84 new + 875 unchanged - 12 fixed = 959 total (was 887) |
| +1 :green_heart: | javadoc | 4m 28s | | the patch passed |
| -1 :x: | spotbugs | 1m 44s | [/new-spotbugs-tez-api.html](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/2/artifact/out/new-spotbugs-tez-api.html) | tez-api generated 4 new + 610 unchanged - 0 fixed = 614 total (was 610) |
| -1 :x: | spotbugs | 1m 59s | [/new-spotbugs-tez-dag.html](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/2/artifact/out/new-spotbugs-tez-dag.html) | tez-dag generated 4 new + 785 unchanged - 0 fixed = 789 total (was 785) |
| -1 :x: | spotbugs | 7m 15s | [/new-spotbugs-root.html](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/2/artifact/out/new-spotbugs-root.html) | root generated 8 new + 2066 unchanged - 0 fixed = 2074 total (was 2066) |
|||| _ Other Tests _ |
| -1 :x: | unit | 2m 30s | [/patch-unit-tez-api.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/2/artifact/out/patch-unit-tez-api.txt) | tez-api in the patch passed. |
| +1 :green_heart: | unit | 0m 44s | | tez-common in the patch passed. |
| +1 :green_heart: | unit | 6m 7s | | tez-runtime-library in the patch passed. |
| +1 :green_heart: | unit | 1m 36s | | tez-mapreduce in the patch passed. |
| +1 :green_heart: | unit | 0m 32s | | tez-examples in the patch passed. |
| -1 :x: | unit | 5m 6s | [/patch-unit-tez-dag.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/2/artifact/out/patch-unit-tez-dag.txt) | tez-dag in the patch passed. |
| -1 :x: | unit | 72m 3s | [/patch-unit-root.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/2/artifact/out/patch-unit-root.txt) | root in the patch passed. |
| -1 :x: | asflicense | 3m 9s | [/results-asflicense.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/2/artifact/out/results-asflicense.txt) | The patch generated 9 ASF License warnings. |
| | | 198m 48s | | |
| Reason | Tests |
|-------:|:------|
| SpotBugs | module:tez-api |
| | Found reliance on default encoding in org.apache.tez.client.registry.zookeeper.ZkAMRegistryClient.getAMRecord(ChildData):in org.apache.tez.client.registry.zookeeper.ZkAMRegistryClient.getAMRecord(ChildData): new String(byte[]) At ZkAMRegistryClient.java:[line 107] |
| | Shared primitive variable "amPort" in one thread may not yield the value of the most recent write from another thread At ZkFrameworkClient.java:one thread may not yield the value of the most recent write from another thread At ZkFrameworkClient.java:[line 107] |
| | Shared primitive variable "isRunning" in one thread may not yield the value of the most recent write from another thread At ZkFrameworkClient.java:one thread may not yield the value of the most recent write from another thread At ZkFrameworkClient.java:[line 63] |
| | Shared primitive variable "isRunning" in one thread may not yield the value of the most recent write from another thread At ZkFrameworkClient.java:one thread may not yield the value of the most recent write from another thread At ZkFrameworkClient.java:[line 67] |
| SpotBugs | module:tez-dag |
| | Found reliance on default encoding in org.apache.tez.dag.api.client.registry.zookeeper.ZkAMRegistry.add(AMRecord):in org.apache.tez.dag.api.client.registry.zookeeper.ZkAMRegistry.add(AMRecord): String.getBytes() At ZkAMRegistry.java:[line 102] |
| | Found reliance on default encoding in org.apache.tez.dag.app.DAGAppMaster.getConfigurationProtoFromText():in org.apache.tez.dag.app.DAGAppMaster.getConfigurationProtoFromText(): new java.io.InputStreamReader(InputStream) At DAGAppMaster.java:[line 2860] |
| | Found reliance on default encoding in org.apache.tez.dag.app.DAGAppMaster.getConfigurationProtoFromText():in org.apache.tez.dag.app.DAGAppMaster.getConfigurationProtoFromText(): String.getBytes() At DAGAppMaster.java:[line 2868] |
| | org.apache.tez.dag.app.DAGAppMaster.getConfigurationProtoFromText() may fail to close stream At DAGAppMaster.java:stream At DAGAppMaster.java:[line 2860] |
| SpotBugs | module:root |
| | Found reliance on default encoding in org.apache.tez.client.registry.zookeeper.ZkAMRegistryClient.getAMRecord(ChildData):in org.apache.tez.client.registry.zookeeper.ZkAMRegistryClient.getAMRecord(ChildData): new String(byte[]) At ZkAMRegistryClient.java:[line 107] |
| | Shared primitive variable "amPort" in one thread may not yield the value of the most recent write from another thread At ZkFrameworkClient.java:one thread may not yield the value of the most recent write from another thread At ZkFrameworkClient.java:[line 107] |
| | Shared primitive variable "isRunning" in one thread may not yield the value of the most recent write from another thread At ZkFrameworkClient.java:one thread may not yield the value of the most recent write from another thread At ZkFrameworkClient.java:[line 63] |
| | Shared primitive variable "isRunning" in one thread may not yield the value of the most recent write from another thread At ZkFrameworkClient.java:one thread may not yield the value of the most recent write from another thread At ZkFrameworkClient.java:[line 67] |
| | Found reliance on default encoding in org.apache.tez.dag.api.client.registry.zookeeper.ZkAMRegistry.add(AMRecord):in org.apache.tez.dag.api.client.registry.zookeeper.ZkAMRegistry.add(AMRecord): String.getBytes() At ZkAMRegistry.java:[line 102] |
| | Found reliance on default encoding in org.apache.tez.dag.app.DAGAppMaster.getConfigurationProtoFromText():in org.apache.tez.dag.app.DAGAppMaster.getConfigurationProtoFromText(): new java.io.InputStreamReader(InputStream) At DAGAppMaster.java:[line 2860] |
| | Found reliance on default encoding in org.apache.tez.dag.app.DAGAppMaster.getConfigurationProtoFromText():in org.apache.tez.dag.app.DAGAppMaster.getConfigurationProtoFromText(): String.getBytes() At DAGAppMaster.java:[line 2868] |
| | org.apache.tez.dag.app.DAGAppMaster.getConfigurationProtoFromText() may fail to close stream At DAGAppMaster.java:stream At DAGAppMaster.java:[line 2860] |
| Failed junit tests | tez.client.TestTezClientUtils |
| | tez.dag.api.TestTezConfiguration |
| | tez.dag.app.dag.impl.TestDAGRecovery |
| | tez.dag.app.dag.impl.TestDAGImpl |
| | tez.dag.app.rm.TestTaskScheduler |
| | tez.dag.app.dag.impl.TestCommit |
| | tez.client.TestTezClientUtils |
| | tez.dag.api.TestTezConfiguration |
| | tez.dag.app.dag.impl.TestDAGRecovery |
| | tez.dag.app.dag.impl.TestDAGImpl |
| | tez.dag.app.rm.TestTaskScheduler |
| | tez.dag.app.dag.impl.TestCommit |
| Subsystem | Report/Notes |
|----------:|:-------------|
| Docker | ClientAPI=1.51 ServerAPI=1.51 base: https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/2/artifact/out/Dockerfile |
| GITHUB PR | https://github.com/apache/tez/pull/427 |
| Optional Tests | dupname asflicense javac javadoc unit codespell detsecrets xmllint compile spotbugs checkstyle |
| uname | Linux 3a05bb1443ba 5.15.0-152-generic #162-Ubuntu SMP Wed Jul 23 09:48:42 UTC 2025 x86_64 x86_64 x86_64 GNU/Linux |
| Build tool | maven |
| Personality | /home/jenkins/jenkins-home/workspace/tez-multibranch_PR-427/src/.yetus/personality.sh |
| git revision | master / c24caebfd63989879fc9e4c63bc4db6572d1f2ad |
| Default Java | Ubuntu-21.0.8+9-Ubuntu-0ubuntu124.04.1 |
| Test Results | https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/2/testReport/ |
| Max. process+thread count | 1384 (vs. ulimit of 5500) |
| modules | C: tez-api tez-common tez-runtime-library tez-mapreduce tez-examples tez-dag . U: . |
| Console output | https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/2/console |
| versions | git=2.43.0 maven=3.8.7 spotbugs=4.9.3 codespell=2.0.0 |
| Powered by | Apache Yetus 0.15.1 https://yetus.apache.org |
This message was automatically generated.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] [DRAFT] TEZ-4007: Zookeeper based FrameworkServices and AmExtensions (3/3) - initial patch [tez]
Posted by "abstractdog (via GitHub)" <gi...@apache.org>.
abstractdog commented on code in PR #427:
URL: https://github.com/apache/tez/pull/427#discussion_r2365427986
##########
tez-api/src/main/java/org/apache/tez/client/registry/zookeeper/ZkAMRegistryClient.java:
##########
@@ -0,0 +1,188 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.tez.client.registry.zookeeper;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
+
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.imps.CuratorFrameworkState;
+import org.apache.curator.framework.recipes.cache.ChildData;
+import org.apache.curator.framework.recipes.cache.PathChildrenCache;
+import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent;
+import org.apache.curator.framework.recipes.cache.PathChildrenCacheListener;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.registry.client.binding.RegistryUtils;
+import org.apache.hadoop.registry.client.types.ServiceRecord;
+import org.apache.tez.client.registry.AMRecord;
+import org.apache.tez.client.registry.AMRegistryClient;
+import org.apache.tez.client.registry.AMRegistryClientListener;
+import org.apache.tez.dag.api.TezConfiguration;
+
+import com.google.common.base.Preconditions;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Curator/Zookeeper impl of AMRegistryClient
+*/
+@InterfaceAudience.Public
+public class ZkAMRegistryClient extends AMRegistryClient {
Review Comment:
absolutely! be aware that these patches have been working for years in downstream distributions. This PR is about contributing them back, so apparently, it's ZK the long-term plan is to improve abstractions and make it more pluggable, and replace ZK with Kubernetes etcd, for example, but the default (yarn-based) behavior doesn't utilize an AM registry at all
can you please elaborate on how Trino's lack of fault tolerance comes to this picture? I mean, this ZK-based registry is more about AM service discovery, not fault tolerance
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] [DRAFT] TEZ-4007: Zookeeper based FrameworkServices and AmExtensions (3/3) + refactor [tez]
Posted by "tez-yetus (via GitHub)" <gi...@apache.org>.
tez-yetus commented on PR #427:
URL: https://github.com/apache/tez/pull/427#issuecomment-3474798804
:broken_heart: **-1 overall**
| Vote | Subsystem | Runtime | Logfile | Comment |
|:----:|----------:|--------:|:--------:|:-------:|
| +0 :ok: | reexec | 0m 16s | | Docker mode activated. |
|||| _ Prechecks _ |
| +1 :green_heart: | dupname | 0m 1s | | No case conflicting files found. |
| +0 :ok: | detsecrets | 0m 0s | | detect-secrets was not available. |
| +0 :ok: | xmllint | 0m 0s | | xmllint was not available. |
| +1 :green_heart: | @author | 0m 0s | | The patch does not contain any @author tags. |
| +1 :green_heart: | test4tests | 0m 0s | | The patch appears to include 13 new or modified test files. |
|||| _ master Compile Tests _ |
| +0 :ok: | mvndep | 3m 9s | | Maven dependency ordering for branch |
| +1 :green_heart: | mvninstall | 10m 50s | | master passed |
| +1 :green_heart: | compile | 5m 56s | | master passed |
| +1 :green_heart: | checkstyle | 3m 46s | | master passed |
| +1 :green_heart: | javadoc | 4m 32s | | master passed |
| +0 :ok: | spotbugs | 2m 15s | | tez-api in master has 610 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 0m 48s | | tez-common in master has 13 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 1m 9s | | tez-runtime-library in master has 241 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 0m 57s | | tez-mapreduce in master has 129 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 0m 44s | | tez-examples in master has 2 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 1m 45s | | tez-dag in master has 785 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 7m 1s | | root in master has 2066 extant spotbugs warnings. |
|||| _ Patch Compile Tests _ |
| +0 :ok: | mvndep | 0m 17s | | Maven dependency ordering for patch |
| +1 :green_heart: | mvninstall | 6m 43s | | the patch passed |
| -1 :x: | codespell | 0m 50s | [/results-codespell.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/18/artifact/out/results-codespell.txt) | The patch generated 1 new + 30 unchanged - 0 fixed = 31 total (was 30) |
| +1 :green_heart: | compile | 6m 1s | | the patch passed |
| +1 :green_heart: | javac | 6m 1s | | the patch passed |
| -1 :x: | blanks | 0m 0s | [/blanks-eol.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/18/artifact/out/blanks-eol.txt) | The patch has 4 line(s) that end in blanks. Use git apply --whitespace=fix <<patch_file>>. Refer https://git-scm.com/docs/git-apply |
| +1 :green_heart: | checkstyle | 0m 26s | | tez-api: The patch generated 0 new + 134 unchanged - 11 fixed = 134 total (was 145) |
| +1 :green_heart: | checkstyle | 0m 23s | | The patch passed checkstyle in tez-common |
| -0 :warning: | checkstyle | 0m 27s | [/results-checkstyle-tez-runtime-library.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/18/artifact/out/results-checkstyle-tez-runtime-library.txt) | tez-runtime-library: The patch generated 4 new + 0 unchanged - 0 fixed = 4 total (was 0) |
| +1 :green_heart: | checkstyle | 0m 25s | | The patch passed checkstyle in tez-mapreduce |
| -0 :warning: | checkstyle | 0m 23s | [/results-checkstyle-tez-examples.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/18/artifact/out/results-checkstyle-tez-examples.txt) | tez-examples: The patch generated 19 new + 10 unchanged - 1 fixed = 29 total (was 11) |
| -0 :warning: | checkstyle | 0m 36s | [/results-checkstyle-tez-dag.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/18/artifact/out/results-checkstyle-tez-dag.txt) | tez-dag: The patch generated 1 new + 842 unchanged - 2 fixed = 843 total (was 844) |
| -0 :warning: | checkstyle | 0m 46s | [/results-checkstyle-root.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/18/artifact/out/results-checkstyle-root.txt) | root: The patch generated 24 new + 988 unchanged - 14 fixed = 1012 total (was 1002) |
| +1 :green_heart: | javadoc | 4m 30s | | the patch passed |
| +1 :green_heart: | spotbugs | 15m 27s | | the patch passed |
|||| _ Other Tests _ |
| +1 :green_heart: | unit | 2m 36s | | tez-api in the patch passed. |
| +1 :green_heart: | unit | 0m 40s | | tez-common in the patch passed. |
| +1 :green_heart: | unit | 6m 6s | | tez-runtime-library in the patch passed. |
| +1 :green_heart: | unit | 1m 36s | | tez-mapreduce in the patch passed. |
| +1 :green_heart: | unit | 0m 31s | | tez-examples in the patch passed. |
| +1 :green_heart: | unit | 5m 25s | | tez-dag in the patch passed. |
| +1 :green_heart: | unit | 72m 14s | | root in the patch passed. |
| -1 :x: | asflicense | 3m 11s | [/results-asflicense.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/18/artifact/out/results-asflicense.txt) | The patch generated 5 ASF License warnings. |
| | | 177m 53s | | |
| Subsystem | Report/Notes |
|----------:|:-------------|
| Docker | ClientAPI=1.51 ServerAPI=1.51 base: https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/18/artifact/out/Dockerfile |
| GITHUB PR | https://github.com/apache/tez/pull/427 |
| Optional Tests | dupname asflicense javac javadoc unit codespell detsecrets xmllint compile spotbugs checkstyle |
| uname | Linux 27410da06fa4 5.15.0-152-generic #162-Ubuntu SMP Wed Jul 23 09:48:42 UTC 2025 x86_64 x86_64 x86_64 GNU/Linux |
| Build tool | maven |
| Personality | /home/jenkins/jenkins-home/workspace/tez-multibranch_PR-427/src/.yetus/personality.sh |
| git revision | master / b70d0c58b1b79d0031f068bb7429d01c690e14d7 |
| Default Java | Ubuntu-21.0.8+9-Ubuntu-0ubuntu124.04.1 |
| Test Results | https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/18/testReport/ |
| Max. process+thread count | 1382 (vs. ulimit of 5500) |
| modules | C: tez-api tez-common tez-runtime-library tez-mapreduce tez-examples tez-dag . U: . |
| Console output | https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/18/console |
| versions | git=2.43.0 maven=3.8.7 spotbugs=4.9.3 codespell=2.0.0 |
| Powered by | Apache Yetus 0.15.1 https://yetus.apache.org |
This message was automatically generated.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] TEZ-4007: Introduce AmExtensions and Zookeeper-based FrameworkServices [tez]
Posted by "abstractdog (via GitHub)" <gi...@apache.org>.
abstractdog commented on PR #427:
URL: https://github.com/apache/tez/pull/427#issuecomment-3479559999
@ayushtkn, @okumin, @zhangbutao: this is ready for reviews now
please be aware that all the reported checkstyle/codespell/asflicense problems are not present any, becase they belong to classes that I removed, I created [TEZ-4660](https://issues.apache.org/jira/browse/TEZ-4660) about this
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] [DRAFT] TEZ-4007: Zookeeper based FrameworkServices and AmExtensions (3/3) - initial patch [tez]
Posted by "tez-yetus (via GitHub)" <gi...@apache.org>.
tez-yetus commented on PR #427:
URL: https://github.com/apache/tez/pull/427#issuecomment-3401317087
:broken_heart: **-1 overall**
| Vote | Subsystem | Runtime | Logfile | Comment |
|:----:|----------:|--------:|:--------:|:-------:|
| +0 :ok: | reexec | 0m 17s | | Docker mode activated. |
|||| _ Prechecks _ |
| +1 :green_heart: | dupname | 0m 1s | | No case conflicting files found. |
| +0 :ok: | detsecrets | 0m 0s | | detect-secrets was not available. |
| +0 :ok: | xmllint | 0m 0s | | xmllint was not available. |
| +1 :green_heart: | @author | 0m 0s | | The patch does not contain any @author tags. |
| +1 :green_heart: | test4tests | 0m 0s | | The patch appears to include 10 new or modified test files. |
|||| _ master Compile Tests _ |
| +0 :ok: | mvndep | 2m 44s | | Maven dependency ordering for branch |
| +1 :green_heart: | mvninstall | 9m 57s | | master passed |
| +1 :green_heart: | compile | 6m 5s | | master passed |
| +1 :green_heart: | checkstyle | 3m 31s | | master passed |
| +1 :green_heart: | javadoc | 4m 30s | | master passed |
| +0 :ok: | spotbugs | 2m 8s | | tez-api in master has 610 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 0m 20s | | branch/tez-build-tools no spotbugs output file (spotbugsXml.xml) |
| +0 :ok: | spotbugs | 0m 45s | | tez-common in master has 13 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 1m 11s | | tez-runtime-library in master has 241 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 0m 56s | | tez-mapreduce in master has 129 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 0m 42s | | tez-examples in master has 2 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 1m 49s | | tez-dag in master has 785 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 7m 26s | | root in master has 2066 extant spotbugs warnings. |
|||| _ Patch Compile Tests _ |
| +0 :ok: | mvndep | 0m 16s | | Maven dependency ordering for patch |
| +1 :green_heart: | mvninstall | 7m 23s | | the patch passed |
| +1 :green_heart: | codespell | 0m 59s | | No new issues. |
| +1 :green_heart: | compile | 6m 17s | | the patch passed |
| +1 :green_heart: | javac | 6m 17s | | the patch passed |
| +1 :green_heart: | blanks | 0m 0s | | The patch has no blanks issues. |
| -0 :warning: | checkstyle | 0m 23s | [/results-checkstyle-tez-api.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/6/artifact/out/results-checkstyle-tez-api.txt) | tez-api: The patch generated 23 new + 112 unchanged - 11 fixed = 135 total (was 123) |
| +1 :green_heart: | checkstyle | 0m 19s | | The patch passed checkstyle in tez-build-tools |
| +1 :green_heart: | checkstyle | 0m 21s | | tez-common: The patch generated 0 new + 2 unchanged - 51 fixed = 2 total (was 53) |
| +1 :green_heart: | checkstyle | 0m 25s | | The patch passed checkstyle in tez-runtime-library |
| +1 :green_heart: | checkstyle | 0m 21s | | The patch passed checkstyle in tez-mapreduce |
| -0 :warning: | checkstyle | 0m 23s | [/results-checkstyle-tez-examples.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/6/artifact/out/results-checkstyle-tez-examples.txt) | tez-examples: The patch generated 6 new + 0 unchanged - 0 fixed = 6 total (was 0) |
| -0 :warning: | checkstyle | 0m 32s | [/results-checkstyle-tez-dag.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/6/artifact/out/results-checkstyle-tez-dag.txt) | tez-dag: The patch generated 243 new + 576 unchanged - 286 fixed = 819 total (was 862) |
| -0 :warning: | checkstyle | 0m 46s | [/results-checkstyle-root.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/6/artifact/out/results-checkstyle-root.txt) | root: The patch generated 268 new + 694 unchanged - 344 fixed = 962 total (was 1038) |
| +1 :green_heart: | javadoc | 4m 28s | | the patch passed |
| +0 :ok: | spotbugs | 0m 23s | | tez-build-tools has no data from spotbugs |
| -1 :x: | spotbugs | 2m 1s | [/new-spotbugs-tez-dag.html](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/6/artifact/out/new-spotbugs-tez-dag.html) | tez-dag generated 1 new + 785 unchanged - 0 fixed = 786 total (was 785) |
| -1 :x: | spotbugs | 7m 19s | [/new-spotbugs-root.html](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/6/artifact/out/new-spotbugs-root.html) | root generated 1 new + 2066 unchanged - 0 fixed = 2067 total (was 2066) |
|||| _ Other Tests _ |
| +1 :green_heart: | unit | 2m 30s | | tez-api in the patch passed. |
| +1 :green_heart: | unit | 0m 24s | | tez-build-tools in the patch passed. |
| +1 :green_heart: | unit | 0m 44s | | tez-common in the patch passed. |
| +1 :green_heart: | unit | 6m 6s | | tez-runtime-library in the patch passed. |
| +1 :green_heart: | unit | 1m 35s | | tez-mapreduce in the patch passed. |
| +1 :green_heart: | unit | 0m 31s | | tez-examples in the patch passed. |
| -1 :x: | unit | 5m 21s | [/patch-unit-tez-dag.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/6/artifact/out/patch-unit-tez-dag.txt) | tez-dag in the patch passed. |
| -1 :x: | unit | 72m 57s | [/patch-unit-root.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/6/artifact/out/patch-unit-root.txt) | root in the patch passed. |
| -1 :x: | asflicense | 3m 35s | [/results-asflicense.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/6/artifact/out/results-asflicense.txt) | The patch generated 2 ASF License warnings. |
| | | 180m 1s | | |
| Reason | Tests |
|-------:|:------|
| SpotBugs | module:tez-dag |
| | Found reliance on default encoding in org.apache.tez.dag.api.client.registry.zookeeper.ZkAMRegistry.add(AMRecord):in org.apache.tez.dag.api.client.registry.zookeeper.ZkAMRegistry.add(AMRecord): String.getBytes() At ZkAMRegistry.java:[line 107] |
| SpotBugs | module:root |
| | Found reliance on default encoding in org.apache.tez.dag.api.client.registry.zookeeper.ZkAMRegistry.add(AMRecord):in org.apache.tez.dag.api.client.registry.zookeeper.ZkAMRegistry.add(AMRecord): String.getBytes() At ZkAMRegistry.java:[line 107] |
| Failed junit tests | tez.dag.app.dag.impl.TestDAGRecovery |
| | tez.dag.app.dag.impl.TestDAGRecovery |
| Subsystem | Report/Notes |
|----------:|:-------------|
| Docker | ClientAPI=1.51 ServerAPI=1.51 base: https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/6/artifact/out/Dockerfile |
| GITHUB PR | https://github.com/apache/tez/pull/427 |
| Optional Tests | dupname asflicense javac javadoc unit codespell detsecrets xmllint compile spotbugs checkstyle |
| uname | Linux 5a8c06b81951 5.15.0-152-generic #162-Ubuntu SMP Wed Jul 23 09:48:42 UTC 2025 x86_64 x86_64 x86_64 GNU/Linux |
| Build tool | maven |
| Personality | /home/jenkins/jenkins-home/workspace/tez-multibranch_PR-427@2/src/.yetus/personality.sh |
| git revision | master / b3ae071a3d92b06a1e765b5db4284c9af840013b |
| Default Java | Ubuntu-21.0.8+9-Ubuntu-0ubuntu124.04.1 |
| Test Results | https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/6/testReport/ |
| Max. process+thread count | 1385 (vs. ulimit of 5500) |
| modules | C: tez-api tez-build-tools tez-common tez-runtime-library tez-mapreduce tez-examples tez-dag . U: . |
| Console output | https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/6/console |
| versions | git=2.43.0 maven=3.8.7 spotbugs=4.9.3 codespell=2.0.0 |
| Powered by | Apache Yetus 0.15.1 https://yetus.apache.org |
This message was automatically generated.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] TEZ-4007: Introduce AmExtensions and Zookeeper-based FrameworkServices [tez]
Posted by "tez-yetus (via GitHub)" <gi...@apache.org>.
tez-yetus commented on PR #427:
URL: https://github.com/apache/tez/pull/427#issuecomment-3555045182
:broken_heart: **-1 overall**
| Vote | Subsystem | Runtime | Logfile | Comment |
|:----:|----------:|--------:|:--------:|:-------:|
| +0 :ok: | reexec | 23m 29s | | Docker mode activated. |
|||| _ Prechecks _ |
| +1 :green_heart: | dupname | 0m 0s | | No case conflicting files found. |
| +0 :ok: | detsecrets | 0m 0s | | detect-secrets was not available. |
| +0 :ok: | xmllint | 0m 0s | | xmllint was not available. |
| +1 :green_heart: | @author | 0m 0s | | The patch does not contain any @author tags. |
| +1 :green_heart: | test4tests | 0m 0s | | The patch appears to include 11 new or modified test files. |
|||| _ master Compile Tests _ |
| +0 :ok: | mvndep | 2m 41s | | Maven dependency ordering for branch |
| +1 :green_heart: | mvninstall | 10m 37s | | master passed |
| +1 :green_heart: | compile | 5m 1s | | master passed |
| +1 :green_heart: | checkstyle | 2m 50s | | master passed |
| +1 :green_heart: | javadoc | 3m 35s | | master passed |
| +0 :ok: | spotbugs | 2m 10s | | tez-api in master has 610 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 0m 43s | | tez-common in master has 13 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 1m 8s | | tez-runtime-library in master has 241 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 0m 41s | | tez-examples in master has 2 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 1m 41s | | tez-dag in master has 785 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 6m 57s | | root in master has 2066 extant spotbugs warnings. |
| -0 :warning: | patch | 7m 40s | | Used diff version of patch file. Binary files and potentially other changes not applied. Please rebase and squash commits if necessary. |
|||| _ Patch Compile Tests _ |
| +0 :ok: | mvndep | 0m 16s | | Maven dependency ordering for patch |
| +1 :green_heart: | mvninstall | 6m 35s | | the patch passed |
| +1 :green_heart: | codespell | 0m 52s | | No new issues. |
| +1 :green_heart: | compile | 5m 2s | | the patch passed |
| +1 :green_heart: | javac | 5m 2s | | the patch passed |
| +1 :green_heart: | blanks | 0m 1s | | The patch has no blanks issues. |
| +1 :green_heart: | checkstyle | 0m 22s | | tez-api: The patch generated 0 new + 118 unchanged - 3 fixed = 118 total (was 121) |
| +1 :green_heart: | checkstyle | 0m 18s | | The patch passed checkstyle in tez-common |
| +1 :green_heart: | checkstyle | 0m 23s | | The patch passed checkstyle in tez-runtime-library |
| +1 :green_heart: | checkstyle | 0m 19s | | tez-examples: The patch generated 0 new + 10 unchanged - 1 fixed = 10 total (was 11) |
| -0 :warning: | checkstyle | 0m 29s | [/results-checkstyle-tez-dag.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/21/artifact/out/results-checkstyle-tez-dag.txt) | tez-dag: The patch generated 1 new + 610 unchanged - 2 fixed = 611 total (was 612) |
| -0 :warning: | checkstyle | 0m 41s | [/results-checkstyle-root.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/21/artifact/out/results-checkstyle-root.txt) | root: The patch generated 1 new + 740 unchanged - 6 fixed = 741 total (was 746) |
| +1 :green_heart: | javadoc | 3m 30s | | the patch passed |
| -1 :x: | spotbugs | 1m 58s | [/new-spotbugs-tez-dag.html](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/21/artifact/out/new-spotbugs-tez-dag.html) | tez-dag generated 1 new + 785 unchanged - 0 fixed = 786 total (was 785) |
| -1 :x: | spotbugs | 7m 17s | [/new-spotbugs-root.html](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/21/artifact/out/new-spotbugs-root.html) | root generated 1 new + 2066 unchanged - 0 fixed = 2067 total (was 2066) |
|||| _ Other Tests _ |
| +1 :green_heart: | unit | 2m 32s | | tez-api in the patch passed. |
| +1 :green_heart: | unit | 0m 39s | | tez-common in the patch passed. |
| +1 :green_heart: | unit | 5m 56s | | tez-runtime-library in the patch passed. |
| +1 :green_heart: | unit | 0m 27s | | tez-examples in the patch passed. |
| -1 :x: | unit | 5m 17s | [/patch-unit-tez-dag.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/21/artifact/out/patch-unit-tez-dag.txt) | tez-dag in the patch passed. |
| -1 :x: | unit | 69m 48s | [/patch-unit-root.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/21/artifact/out/patch-unit-root.txt) | root in the patch passed. |
| +1 :green_heart: | asflicense | 2m 11s | | The patch does not generate ASF License warnings. |
| | | 186m 13s | | |
| Reason | Tests |
|-------:|:------|
| SpotBugs | module:tez-dag |
| | org.apache.tez.frameworkplugins.zookeeper.ZkStandaloneServerFrameworkService.getAMRegistry(Configuration) may expose internal representation by returning ZkStandaloneServerFrameworkService.amRegistry At ZkStandaloneServerFrameworkService.java:by returning ZkStandaloneServerFrameworkService.amRegistry At ZkStandaloneServerFrameworkService.java:[line 48] |
| SpotBugs | module:root |
| | org.apache.tez.frameworkplugins.zookeeper.ZkStandaloneServerFrameworkService.getAMRegistry(Configuration) may expose internal representation by returning ZkStandaloneServerFrameworkService.amRegistry At ZkStandaloneServerFrameworkService.java:by returning ZkStandaloneServerFrameworkService.amRegistry At ZkStandaloneServerFrameworkService.java:[line 48] |
| Failed junit tests | tez.client.registry.zookeeper.TestZkAMRegistryClient |
| | tez.dag.history.recovery.TestRecoveryService |
| | tez.client.registry.zookeeper.TestZkAMRegistryClient |
| | tez.dag.history.recovery.TestRecoveryService |
| Subsystem | Report/Notes |
|----------:|:-------------|
| Docker | ClientAPI=1.52 ServerAPI=1.52 base: https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/21/artifact/out/Dockerfile |
| GITHUB PR | https://github.com/apache/tez/pull/427 |
| Optional Tests | dupname asflicense javac javadoc unit codespell detsecrets xmllint compile spotbugs checkstyle |
| uname | Linux 9ab4eb996e3a 5.15.0-161-generic #171-Ubuntu SMP Sat Oct 11 08:17:01 UTC 2025 x86_64 x86_64 x86_64 GNU/Linux |
| Build tool | maven |
| Personality | /home/jenkins/jenkins-home/workspace/tez-multibranch_PR-427/src/.yetus/personality.sh |
| git revision | master / 5794226e167b8082a4bee6d2dfcde2d7e701c745 |
| Default Java | Ubuntu-21.0.8+9-Ubuntu-0ubuntu124.04.1 |
| Test Results | https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/21/testReport/ |
| Max. process+thread count | 1353 (vs. ulimit of 5500) |
| modules | C: tez-api tez-common tez-runtime-library tez-examples tez-dag . U: . |
| Console output | https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/21/console |
| versions | git=2.43.0 maven=3.8.7 spotbugs=4.9.3 codespell=2.0.0 |
| Powered by | Apache Yetus 0.15.1 https://yetus.apache.org |
This message was automatically generated.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] TEZ-4007: Introduce AmExtensions and Zookeeper-based FrameworkServices [tez]
Posted by "abstractdog (via GitHub)" <gi...@apache.org>.
abstractdog commented on code in PR #427:
URL: https://github.com/apache/tez/pull/427#discussion_r2681539466
##########
tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java:
##########
@@ -513,19 +514,21 @@ protected void serviceInit(final Configuration conf) throws Exception {
jobTokenSecretManager = new JobTokenSecretManager(amConf);
- sessionToken =
- TokenCache.getSessionToken(amCredentials);
+ sessionToken = frameworkService.getAMExtensions().getSessionToken(
+ appAttemptID, jobTokenSecretManager, amCredentials);
+
if (sessionToken == null) {
- throw new RuntimeException("Could not find session token in AM Credentials");
+ sessionToken = TokenCache.getSessionToken(amCredentials);
Review Comment:
right, so the extra attempt to fetch it from session token doesn't make sense, let me change this to:
```
if (sessionToken == null) {
throw new RuntimeException("Could not find session token in AM Credentials");
}
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] TEZ-4007: Introduce AmExtensions and Zookeeper-based FrameworkServices [tez]
Posted by "zhangbutao (via GitHub)" <gi...@apache.org>.
zhangbutao commented on PR #427:
URL: https://github.com/apache/tez/pull/427#issuecomment-3748958720
Thank you! @abstractdog I can't wait to experience the features of Tez on Cloud. Let's move on to the next step!
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] TEZ-4007: Introduce AmExtensions and Zookeeper-based FrameworkServices [tez]
Posted by "okumin (via GitHub)" <gi...@apache.org>.
okumin commented on code in PR #427:
URL: https://github.com/apache/tez/pull/427#discussion_r2681187034
##########
tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java:
##########
@@ -513,19 +514,21 @@ protected void serviceInit(final Configuration conf) throws Exception {
jobTokenSecretManager = new JobTokenSecretManager(amConf);
- sessionToken =
- TokenCache.getSessionToken(amCredentials);
+ sessionToken = frameworkService.getAMExtensions().getSessionToken(
+ appAttemptID, jobTokenSecretManager, amCredentials);
+
if (sessionToken == null) {
- throw new RuntimeException("Could not find session token in AM Credentials");
+ sessionToken = TokenCache.getSessionToken(amCredentials);
Review Comment:
When does this double-check help?
- YARN -> When `sessionToken` is null, `TokenCache.getSessionToken(amCredentials)` is probably always null
- ZK -> `sessionToken` is not always null
##########
tez-api/src/main/java/org/apache/tez/frameworkplugins/zookeeper/ZkStandaloneClientFrameworkService.java:
##########
@@ -0,0 +1,29 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tez.frameworkplugins.zookeeper;
+
+
+import org.apache.tez.client.FrameworkClient;
+import org.apache.tez.client.registry.zookeeper.ZkFrameworkClient;
+import org.apache.tez.frameworkplugins.ClientFrameworkService;
+
+public class ZkStandaloneClientFrameworkService implements ClientFrameworkService {
+ @Override public FrameworkClient newFrameworkClient() {
Review Comment:
```suggestion
@Override
public FrameworkClient newFrameworkClient() {
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] TEZ-4007: Introduce AmExtensions and Zookeeper-based FrameworkServices [tez]
Posted by "abstractdog (via GitHub)" <gi...@apache.org>.
abstractdog commented on code in PR #427:
URL: https://github.com/apache/tez/pull/427#discussion_r2563596644
##########
tez-api/src/main/java/org/apache/tez/client/registry/zookeeper/ZkConfig.java:
##########
@@ -0,0 +1,128 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.client.registry.zookeeper;
+
+import org.apache.curator.RetryPolicy;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.CuratorFrameworkFactory;
+import org.apache.curator.retry.ExponentialBackoffRetry;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.tez.dag.api.TezConfiguration;
+
+import com.google.common.base.Preconditions;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class ZkConfig {
+ private static final Logger LOG = LoggerFactory.getLogger(ZkConfig.class);
+
+ // if namespace defined in config is 'foo' and COMPUTE_GROUP_NAME env is 'bar' then the zkpaths will be of format
+ // /tez-external-sessions/foo/bar
+ private final static String ZK_NAMESPACE_PREFIX = "/tez-external-sessions";
+ public final static String COMPUTE_GROUP_NAME_ENV = "COMPUTE_GROUP_NAME";
+ public final static String DEFAULT_COMPUTE_GROUP_NAME = "default-compute";
+
+ private final String zkQuorum;
+ private final String zkNamespace;
+ private final int curatorBackoffSleep;
+ private final int curatorMaxRetries;
+ private final int sessionTimeoutMs;
+ private final int connectionTimeoutMs;
+
+ public ZkConfig(Configuration conf) {
+ zkQuorum = conf.get(TezConfiguration.TEZ_AM_ZOOKEEPER_QUORUM);
+ Preconditions.checkNotNull(zkQuorum);
+
+ String fullZkNamespace = ZK_NAMESPACE_PREFIX;
+
+ String namespace = conf.get(TezConfiguration.TEZ_AM_REGISTRY_NAMESPACE,
+ TezConfiguration.TEZ_AM_REGISTRY_NAMESPACE_DEFAULT);
+ Preconditions.checkNotNull(namespace);
+ if (namespace.startsWith(Path.SEPARATOR)) {
+ fullZkNamespace += namespace;
+ } else {
+ fullZkNamespace = fullZkNamespace + Path.SEPARATOR + namespace;
+ }
+
+ boolean enableComputeGroups = conf.getBoolean(TezConfiguration.TEZ_AM_REGISTRY_ENABLE_COMPUTE_GROUPS,
+ TezConfiguration.TEZ_AM_REGISTRY_ENABLE_COMPUTE_GROUPS_DEFAULT);
+ if (enableComputeGroups) {
+ final String subNamespace = System.getenv(COMPUTE_GROUP_NAME_ENV);
Review Comment:
this regex replaces leading slashes with an empty string, which leads to this exception in ZK:
`java.lang.IllegalArgumentException: Path must start with / character`
in ZK, a node should start with a slash
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] TEZ-4007: Introduce AmExtensions and Zookeeper-based FrameworkServices [tez]
Posted by "abstractdog (via GitHub)" <gi...@apache.org>.
abstractdog commented on code in PR #427:
URL: https://github.com/apache/tez/pull/427#discussion_r2564918638
##########
tez-api/src/main/java/org/apache/tez/frameworkplugins/FrameworkUtils.java:
##########
@@ -0,0 +1,96 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tez.frameworkplugins;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.ServiceLoader;
+
+import javax.annotation.Nullable;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.tez.common.ReflectionUtils;
+import org.apache.tez.dag.api.TezConfiguration;
+import org.apache.tez.dag.api.TezConstants;
+import org.apache.tez.dag.api.TezReflectionException;
+
+public final class FrameworkUtils {
+
+ private FrameworkUtils() {}
+
+ /*
+ Searches for a FrameworkService provider which implements a target interface.
+ The interface should be either ClientFrameworkService or ServerFrameworkService.
+ Depending on which interface is used, either the client or server class of a
+ matching FrameworkMode will be used as the implementation.
+
+ NOTE: Layering of FrameworkServices in a decorator-style is currently not supported
+
+ An implementation is searched in the following order:
+ 1. If conf is not null and the parameter TEZ_FRAMEWORK_MODE is set:
+ the value of TEZ_FRAMEWORK_MODE from the conf will be used
+ 2. If conf is null or the parameter TEZ_FRAMEWORK_MODE is not set
+ and the environment var TEZ_FRAMEWORK_MODE is not empty:
+ the value of the environment var will be used
+ 3. Otherwise:
+ the default java.util.ServiceLoader behavior will be used,
+ i.e. the implementation classname should appear in a file on the classpath at the location
+ META-INF/services/org.apache.tez.frameworkplugins.ClientFrameworkService
+ or META-INF/services/org.apache.tez.frameworkplugins.ServerFrameworkService
+ */
+ public static <T extends FrameworkService> T get(Class<T> interfaze, @Nullable Configuration conf) {
+ try {
+ if ((conf != null) && (conf.get(TezConfiguration.TEZ_FRAMEWORK_MODE) != null)) {
+ return getByMode(interfaze, conf.get(TezConfiguration.TEZ_FRAMEWORK_MODE));
+ } else if (System.getenv(TezConstants.TEZ_FRAMEWORK_MODE) != null) {
+ return getByMode(interfaze, System.getenv(TezConstants.TEZ_FRAMEWORK_MODE));
+ } else {
+ return getByServiceLoader(interfaze);
+ }
+ } catch (TezReflectionException e) {
+ throw new RuntimeException("Failed to load framework service for interface: " + interfaze.getName(), e);
+ }
+ }
+
+ private static <T extends FrameworkService> T getByServiceLoader(Class<T> interfaze) {
+ List<T> services = new ArrayList<>();
+ ServiceLoader<T> frameworkService = ServiceLoader.load(interfaze);
+ for (T service : frameworkService) {
+ services.add(service);
+ }
+ if (services.isEmpty()) {
+ return null;
Review Comment:
finally, I decided to remove getByServiceLoader, and make the users use it by getMode properly
let's keep it simple
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] TEZ-4007: Introduce AmExtensions and Zookeeper-based FrameworkServices [tez]
Posted by "abstractdog (via GitHub)" <gi...@apache.org>.
abstractdog commented on code in PR #427:
URL: https://github.com/apache/tez/pull/427#discussion_r2567607890
##########
tez-dag/src/main/java/org/apache/tez/frameworkplugins/zookeeper/ZkStandaloneServerFrameworkService.java:
##########
@@ -0,0 +1,55 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tez.frameworkplugins.zookeeper;
+
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.tez.client.registry.AMRegistry;
+import org.apache.tez.dag.api.TezConstants;
+import org.apache.tez.dag.api.client.registry.zookeeper.ZkAMRegistry;
+import org.apache.tez.frameworkplugins.AMExtensions;
+import org.apache.tez.frameworkplugins.ServerFrameworkService;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class ZkStandaloneServerFrameworkService implements ServerFrameworkService {
+ private static final Logger LOG = LoggerFactory.getLogger(ZkStandaloneServerFrameworkService.class);
+ private ZkAMRegistry amRegistry;
+
+ @Override
+ public synchronized AMRegistry getAMRegistry(Configuration conf) {
+ if (amRegistry == null) {
+ try {
+ final String externalID = System.getenv(TezConstants.TEZ_AM_EXTERNAL_ID);
Review Comment:
right, let me give some thoughts for the user as below:
```
final String externalId = System.getenv(TezConstants.TEZ_AM_EXTERNAL_ID);
if (externalId == null) {
throw new IllegalStateException(
TezConstants.TEZ_AM_EXTERNAL_ID + " environment variable is not set for standalone AM");
}
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] [DRAFT] TEZ-4007: Zookeeper based FrameworkServices and AmExtensions (3/3) - initial patch [tez]
Posted by "tez-yetus (via GitHub)" <gi...@apache.org>.
tez-yetus commented on PR #427:
URL: https://github.com/apache/tez/pull/427#issuecomment-3397815298
:broken_heart: **-1 overall**
| Vote | Subsystem | Runtime | Logfile | Comment |
|:----:|----------:|--------:|:--------:|:-------:|
| +0 :ok: | reexec | 24m 0s | | Docker mode activated. |
|||| _ Prechecks _ |
| +1 :green_heart: | dupname | 0m 1s | | No case conflicting files found. |
| +0 :ok: | detsecrets | 0m 0s | | detect-secrets was not available. |
| +0 :ok: | xmllint | 0m 0s | | xmllint was not available. |
| +1 :green_heart: | @author | 0m 0s | | The patch does not contain any @author tags. |
| +1 :green_heart: | test4tests | 0m 0s | | The patch appears to include 8 new or modified test files. |
|||| _ master Compile Tests _ |
| +0 :ok: | mvndep | 2m 46s | | Maven dependency ordering for branch |
| +1 :green_heart: | mvninstall | 9m 56s | | master passed |
| +1 :green_heart: | compile | 5m 53s | | master passed |
| +1 :green_heart: | checkstyle | 3m 44s | | master passed |
| +1 :green_heart: | javadoc | 4m 37s | | master passed |
| +0 :ok: | spotbugs | 2m 12s | | tez-api in master has 610 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 0m 47s | | tez-common in master has 13 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 1m 10s | | tez-runtime-library in master has 241 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 0m 58s | | tez-mapreduce in master has 129 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 0m 43s | | tez-examples in master has 2 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 1m 47s | | tez-dag in master has 785 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 6m 54s | | root in master has 2066 extant spotbugs warnings. |
|||| _ Patch Compile Tests _ |
| +0 :ok: | mvndep | 0m 16s | | Maven dependency ordering for patch |
| +1 :green_heart: | mvninstall | 6m 41s | | the patch passed |
| +1 :green_heart: | codespell | 0m 50s | | No new issues. |
| +1 :green_heart: | compile | 5m 54s | | the patch passed |
| +1 :green_heart: | javac | 5m 54s | | the patch passed |
| -1 :x: | blanks | 0m 0s | [/blanks-eol.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/3/artifact/out/blanks-eol.txt) | The patch has 1 line(s) that end in blanks. Use git apply --whitespace=fix <<patch_file>>. Refer https://git-scm.com/docs/git-apply |
| -0 :warning: | checkstyle | 0m 25s | [/results-checkstyle-tez-api.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/3/artifact/out/results-checkstyle-tez-api.txt) | tez-api: The patch generated 48 new + 112 unchanged - 11 fixed = 160 total (was 123) |
| -0 :warning: | checkstyle | 0m 28s | [/results-checkstyle-tez-runtime-library.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/3/artifact/out/results-checkstyle-tez-runtime-library.txt) | tez-runtime-library: The patch generated 1 new + 0 unchanged - 0 fixed = 1 total (was 0) |
| -0 :warning: | checkstyle | 0m 22s | [/results-checkstyle-tez-examples.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/3/artifact/out/results-checkstyle-tez-examples.txt) | tez-examples: The patch generated 6 new + 0 unchanged - 0 fixed = 6 total (was 0) |
| -0 :warning: | checkstyle | 0m 36s | [/results-checkstyle-tez-dag.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/3/artifact/out/results-checkstyle-tez-dag.txt) | tez-dag: The patch generated 28 new + 740 unchanged - 1 fixed = 768 total (was 741) |
| -0 :warning: | checkstyle | 0m 47s | [/results-checkstyle-root.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/3/artifact/out/results-checkstyle-root.txt) | root: The patch generated 83 new + 905 unchanged - 12 fixed = 988 total (was 917) |
| +1 :green_heart: | javadoc | 4m 29s | | the patch passed |
| -1 :x: | spotbugs | 1m 45s | [/new-spotbugs-tez-api.html](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/3/artifact/out/new-spotbugs-tez-api.html) | tez-api generated 4 new + 610 unchanged - 0 fixed = 614 total (was 610) |
| -1 :x: | spotbugs | 2m 2s | [/new-spotbugs-tez-dag.html](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/3/artifact/out/new-spotbugs-tez-dag.html) | tez-dag generated 4 new + 785 unchanged - 0 fixed = 789 total (was 785) |
| -1 :x: | spotbugs | 7m 17s | [/new-spotbugs-root.html](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/3/artifact/out/new-spotbugs-root.html) | root generated 8 new + 2066 unchanged - 0 fixed = 2074 total (was 2066) |
|||| _ Other Tests _ |
| -1 :x: | unit | 2m 30s | [/patch-unit-tez-api.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/3/artifact/out/patch-unit-tez-api.txt) | tez-api in the patch passed. |
| +1 :green_heart: | unit | 0m 44s | | tez-common in the patch passed. |
| +1 :green_heart: | unit | 6m 8s | | tez-runtime-library in the patch passed. |
| +1 :green_heart: | unit | 1m 35s | | tez-mapreduce in the patch passed. |
| +1 :green_heart: | unit | 0m 32s | | tez-examples in the patch passed. |
| -1 :x: | unit | 5m 6s | [/patch-unit-tez-dag.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/3/artifact/out/patch-unit-tez-dag.txt) | tez-dag in the patch passed. |
| -1 :x: | unit | 72m 5s | [/patch-unit-root.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/3/artifact/out/patch-unit-root.txt) | root in the patch passed. |
| -1 :x: | asflicense | 3m 13s | [/results-asflicense.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/3/artifact/out/results-asflicense.txt) | The patch generated 9 ASF License warnings. |
| | | 199m 39s | | |
| Reason | Tests |
|-------:|:------|
| SpotBugs | module:tez-api |
| | Found reliance on default encoding in org.apache.tez.client.registry.zookeeper.ZkAMRegistryClient.getAMRecord(ChildData):in org.apache.tez.client.registry.zookeeper.ZkAMRegistryClient.getAMRecord(ChildData): new String(byte[]) At ZkAMRegistryClient.java:[line 107] |
| | Shared primitive variable "amPort" in one thread may not yield the value of the most recent write from another thread At ZkFrameworkClient.java:one thread may not yield the value of the most recent write from another thread At ZkFrameworkClient.java:[line 107] |
| | Shared primitive variable "isRunning" in one thread may not yield the value of the most recent write from another thread At ZkFrameworkClient.java:one thread may not yield the value of the most recent write from another thread At ZkFrameworkClient.java:[line 63] |
| | Shared primitive variable "isRunning" in one thread may not yield the value of the most recent write from another thread At ZkFrameworkClient.java:one thread may not yield the value of the most recent write from another thread At ZkFrameworkClient.java:[line 67] |
| SpotBugs | module:tez-dag |
| | Found reliance on default encoding in org.apache.tez.dag.api.client.registry.zookeeper.ZkAMRegistry.add(AMRecord):in org.apache.tez.dag.api.client.registry.zookeeper.ZkAMRegistry.add(AMRecord): String.getBytes() At ZkAMRegistry.java:[line 102] |
| | Found reliance on default encoding in org.apache.tez.dag.app.DAGAppMaster.getConfigurationProtoFromText():in org.apache.tez.dag.app.DAGAppMaster.getConfigurationProtoFromText(): new java.io.InputStreamReader(InputStream) At DAGAppMaster.java:[line 2723] |
| | Found reliance on default encoding in org.apache.tez.dag.app.DAGAppMaster.getConfigurationProtoFromText():in org.apache.tez.dag.app.DAGAppMaster.getConfigurationProtoFromText(): String.getBytes() At DAGAppMaster.java:[line 2731] |
| | org.apache.tez.dag.app.DAGAppMaster.getConfigurationProtoFromText() may fail to close stream At DAGAppMaster.java:stream At DAGAppMaster.java:[line 2723] |
| SpotBugs | module:root |
| | Found reliance on default encoding in org.apache.tez.client.registry.zookeeper.ZkAMRegistryClient.getAMRecord(ChildData):in org.apache.tez.client.registry.zookeeper.ZkAMRegistryClient.getAMRecord(ChildData): new String(byte[]) At ZkAMRegistryClient.java:[line 107] |
| | Shared primitive variable "amPort" in one thread may not yield the value of the most recent write from another thread At ZkFrameworkClient.java:one thread may not yield the value of the most recent write from another thread At ZkFrameworkClient.java:[line 107] |
| | Shared primitive variable "isRunning" in one thread may not yield the value of the most recent write from another thread At ZkFrameworkClient.java:one thread may not yield the value of the most recent write from another thread At ZkFrameworkClient.java:[line 63] |
| | Shared primitive variable "isRunning" in one thread may not yield the value of the most recent write from another thread At ZkFrameworkClient.java:one thread may not yield the value of the most recent write from another thread At ZkFrameworkClient.java:[line 67] |
| | Found reliance on default encoding in org.apache.tez.dag.api.client.registry.zookeeper.ZkAMRegistry.add(AMRecord):in org.apache.tez.dag.api.client.registry.zookeeper.ZkAMRegistry.add(AMRecord): String.getBytes() At ZkAMRegistry.java:[line 102] |
| | Found reliance on default encoding in org.apache.tez.dag.app.DAGAppMaster.getConfigurationProtoFromText():in org.apache.tez.dag.app.DAGAppMaster.getConfigurationProtoFromText(): new java.io.InputStreamReader(InputStream) At DAGAppMaster.java:[line 2723] |
| | Found reliance on default encoding in org.apache.tez.dag.app.DAGAppMaster.getConfigurationProtoFromText():in org.apache.tez.dag.app.DAGAppMaster.getConfigurationProtoFromText(): String.getBytes() At DAGAppMaster.java:[line 2731] |
| | org.apache.tez.dag.app.DAGAppMaster.getConfigurationProtoFromText() may fail to close stream At DAGAppMaster.java:stream At DAGAppMaster.java:[line 2723] |
| Failed junit tests | tez.client.TestTezClientUtils |
| | tez.dag.api.TestTezConfiguration |
| | tez.dag.app.dag.impl.TestDAGRecovery |
| | tez.dag.app.dag.impl.TestDAGImpl |
| | tez.dag.app.rm.TestTaskScheduler |
| | tez.dag.app.dag.impl.TestCommit |
| | tez.client.TestTezClientUtils |
| | tez.dag.api.TestTezConfiguration |
| | tez.dag.app.dag.impl.TestDAGRecovery |
| | tez.dag.app.dag.impl.TestDAGImpl |
| | tez.dag.app.rm.TestTaskScheduler |
| | tez.dag.app.dag.impl.TestCommit |
| Subsystem | Report/Notes |
|----------:|:-------------|
| Docker | ClientAPI=1.51 ServerAPI=1.51 base: https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/3/artifact/out/Dockerfile |
| GITHUB PR | https://github.com/apache/tez/pull/427 |
| Optional Tests | dupname asflicense javac javadoc unit codespell detsecrets xmllint compile spotbugs checkstyle |
| uname | Linux e1a78377e4ff 5.15.0-152-generic #162-Ubuntu SMP Wed Jul 23 09:48:42 UTC 2025 x86_64 x86_64 x86_64 GNU/Linux |
| Build tool | maven |
| Personality | /home/jenkins/jenkins-home/workspace/tez-multibranch_PR-427/src/.yetus/personality.sh |
| git revision | master / fbe40c86816fff674dd75fa288d090998ebbd78e |
| Default Java | Ubuntu-21.0.8+9-Ubuntu-0ubuntu124.04.1 |
| Test Results | https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/3/testReport/ |
| Max. process+thread count | 1323 (vs. ulimit of 5500) |
| modules | C: tez-api tez-common tez-runtime-library tez-mapreduce tez-examples tez-dag . U: . |
| Console output | https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/3/console |
| versions | git=2.43.0 maven=3.8.7 spotbugs=4.9.3 codespell=2.0.0 |
| Powered by | Apache Yetus 0.15.1 https://yetus.apache.org |
This message was automatically generated.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] TEZ-4007: Introduce AmExtensions and Zookeeper-based FrameworkServices [tez]
Posted by "abstractdog (via GitHub)" <gi...@apache.org>.
abstractdog commented on code in PR #427:
URL: https://github.com/apache/tez/pull/427#discussion_r2671729361
##########
tez-api/src/main/java/org/apache/tez/client/registry/AMRecord.java:
##########
@@ -125,9 +154,11 @@ public boolean equals(Object other) {
}
if (other instanceof AMRecord otherRecord) {
return appId.equals(otherRecord.appId)
- && host.equals(otherRecord.host)
+ && hostName.equals(otherRecord.hostName)
Review Comment:
this is initialized from DAGAppMaster:
https://github.com/apache/tez/pull/427/files#diff-54ba4a2af15261379079ed5a9c1f9eea52da7bdd3f1109fe7b96d4abf07f6173R677-R678
```
AMRecord amRecord = amRegistry.createAmRecord(appId, rpcServerAddress.getHostName(),
rpcServerAddress.getAddress().getHostAddress(), rpcServerAddress.getPort(), computeName);
```
hostName is not supposed to be null, so if it's null, it's already a fatal issue, which is revealed by NPE, I don't think we make this better
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] TEZ-4007: Introduce AmExtensions and Zookeeper-based FrameworkServices [tez]
Posted by "abstractdog (via GitHub)" <gi...@apache.org>.
abstractdog commented on code in PR #427:
URL: https://github.com/apache/tez/pull/427#discussion_r2671856385
##########
tez-api/src/main/java/org/apache/tez/client/registry/AMRegistryClient.java:
##########
@@ -0,0 +1,118 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.client.registry;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Client-side interface for discovering Application Master (AM) instances
+ * registered in the AM registry.
+ *
+ * <p>Implementations are responsible for locating AM endpoints and returning
+ * their metadata. This API is used by client components to discover running
+ * Tez AMs.</p>
+ *
+ * <p>Listeners may be registered to receive notifications when AM records
+ * appear or are removed.</p>
+ */
+public abstract class AMRegistryClient implements Closeable {
+ private static final Logger LOG = LoggerFactory.getLogger(AMRegistryClient.class);
+
+ private final List<AMRegistryClientListener> listeners = new ArrayList<>();
+
+ /**
+ * Lookup AM metadata for the given application ID.
+ *
+ * @param appId the application ID
+ * @return the AM record if found, otherwise {@code null}
+ * @throws IOException if the lookup fails
+ */
+ public abstract AMRecord getRecord(ApplicationId appId) throws IOException;
+
+ /**
+ * Retrieve all AM records known in the registry.
+ *
+ * @return a list of AM records (possibly empty)
+ * @throws IOException if the fetch fails
+ */
+ public abstract List<AMRecord> getAllRecords() throws IOException;
+
+ /**
+ * Register a listener for AM registry events.
+ * The listener will be notified when AM records are added or removed.
+ *
+ * @param listener the listener to add
+ */
+ public synchronized void addListener(AMRegistryClientListener listener) {
+ listeners.add(listener);
+ }
+
+ /**
+ * Notify listeners of a newly added AM record.
+ *
+ * @param record the added AM record
+ */
+ protected synchronized void notifyOnAdded(AMRecord record) {
+ for (AMRegistryClientListener listener : listeners) {
+ try {
+ listener.onAdd(record);
+ } catch (Exception e) {
+ LOG.warn("Exception while calling AM add listener, AM record {}", record, e);
+ }
+ }
+ }
+
+ /**
+ * Notify listeners of an updated AM record.
+ *
+ * @param record the updated AM record
+ */
+ protected synchronized void notifyOnUpdated(AMRecord record) {
+ for (AMRegistryClientListener listener : listeners) {
+ try {
+ listener.onUpdate(record);
+ } catch (Exception e) {
+ LOG.warn("Exception while calling AM update listener, AM record {}", record, e);
Review Comment:
yeah, and similar change can be done for the other messages in this class
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] TEZ-4007: Introduce AmExtensions and Zookeeper-based FrameworkServices [tez]
Posted by "abstractdog (via GitHub)" <gi...@apache.org>.
abstractdog commented on code in PR #427:
URL: https://github.com/apache/tez/pull/427#discussion_r2675939284
##########
tez-api/src/main/java/org/apache/tez/client/registry/zookeeper/ZkAMRegistryClient.java:
##########
@@ -0,0 +1,209 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.client.registry.zookeeper;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.imps.CuratorFrameworkState;
+import org.apache.curator.framework.recipes.cache.ChildData;
+import org.apache.curator.framework.recipes.cache.TreeCache;
+import org.apache.curator.framework.recipes.cache.TreeCacheEvent;
+import org.apache.curator.framework.recipes.cache.TreeCacheListener;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.registry.client.types.ServiceRecord;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.tez.client.registry.AMRecord;
+import org.apache.tez.client.registry.AMRegistryClient;
+import org.apache.tez.client.registry.AMRegistryUtils;
+import org.apache.tez.dag.api.TezConfiguration;
+
+import com.fasterxml.jackson.core.JsonParseException;
+import com.google.common.base.Preconditions;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Curator/Zookeeper implementation of {@link AMRegistryClient}.
+ */
+@InterfaceAudience.Public
+public final class ZkAMRegistryClient extends AMRegistryClient {
+ private static final Logger LOG = LoggerFactory.getLogger(ZkAMRegistryClient.class);
+ private static final Map<String, ZkAMRegistryClient> INSTANCES = new HashMap<>();
+
+ private final Configuration conf;
+ // Cache of known AMs
+ private final ConcurrentHashMap<ApplicationId, AMRecord> amRecordCache = new ConcurrentHashMap<>();
+
+ private CuratorFramework client;
+ private TreeCache cache;
+ private ZkRegistryListener listener;
+
+ private ZkAMRegistryClient(final Configuration conf) {
+ this.conf = conf;
+ }
+
+ public static synchronized ZkAMRegistryClient getClient(final Configuration conf) {
+ String namespace = conf.get(TezConfiguration.TEZ_AM_REGISTRY_NAMESPACE);
+ ZkAMRegistryClient registry = INSTANCES.get(namespace);
+ if (registry == null) {
+ registry = new ZkAMRegistryClient(conf);
+ INSTANCES.put(namespace, registry);
+ }
+ LOG.info("Returning tez AM registry ({}) for namespace '{}'", System.identityHashCode(registry), namespace);
+ return registry;
+ }
+
+ /**
+ * Deserializes a {@link ServiceRecord} from ZooKeeper data and converts it into an {@link AMRecord}
+ * for caching.
+ *
+ * @param childData the ZooKeeper node data containing a serialized {@link ServiceRecord}
+ * @return an {@link AMRecord} constructed from the deserialized {@link ServiceRecord}, or {@code null}
+ * if no data is present
+ * @throws IOException if the data cannot be deserialized into a {@link ServiceRecord}
+ */
+ public static AMRecord getAMRecord(final ChildData childData) throws IOException {
+ // Not a leaf path. Only leaf path contains AMRecord.
+ if (!childData.getPath().contains(ApplicationId.appIdStrPrefix)) {
+ return null;
+ }
+ byte[] data = childData.getData();
+ // only the path appeared, there is no data yet
+ if (data.length == 0) {
+ return null;
+ }
+ String value = new String(data, StandardCharsets.UTF_8);
+ try {
+ return AMRegistryUtils.jsonStringToRecord(value);
+ } catch (JsonParseException e) {
+ //Not a json AMRecord (SRV), could be some other data.
+ LOG.warn("Non-json data received while de-serializing AMRecord: {}. Ignoring...", value);
+ return null;
+ }
+ }
+
+ public void start() throws Exception {
+ ZkConfig zkConf = new ZkConfig(this.conf);
+ client = zkConf.createCuratorFramework();
+ cache = new TreeCache(client, zkConf.getZkNamespace());
+ client.start();
+ cache.start();
+ listener = new ZkRegistryListener();
+ cache.getListenable().addListener(listener);
+ }
+
+ @Override
+ public AMRecord getRecord(ApplicationId appId) {
+ AMRecord rec = amRecordCache.get(appId);
+ // Return a copy.
+ return rec == null ? null : new AMRecord(rec);
+ }
+
+ @Override
+ public List<AMRecord> getAllRecords() {
+ return amRecordCache.values().stream().map(AMRecord::new).collect(Collectors.toList());
+ }
+
+ @Override
+ public void close() {
+ IOUtils.closeQuietly(cache);
+ IOUtils.closeQuietly(client);
Review Comment:
solutiont is in this commit:
https://github.com/apache/tez/pull/427/commits/ad531a86ca6c8005e98413226cf4c6f0cd6884f0
investigated the downstream distro, and it turned out that Hive currently doesn't call TezClient.close on external sessions, and according to code comments, it's by design
I attempted and fix this in the above comment
##########
tez-api/src/main/java/org/apache/tez/client/registry/zookeeper/ZkAMRegistryClient.java:
##########
@@ -0,0 +1,209 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.client.registry.zookeeper;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.imps.CuratorFrameworkState;
+import org.apache.curator.framework.recipes.cache.ChildData;
+import org.apache.curator.framework.recipes.cache.TreeCache;
+import org.apache.curator.framework.recipes.cache.TreeCacheEvent;
+import org.apache.curator.framework.recipes.cache.TreeCacheListener;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.registry.client.types.ServiceRecord;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.tez.client.registry.AMRecord;
+import org.apache.tez.client.registry.AMRegistryClient;
+import org.apache.tez.client.registry.AMRegistryUtils;
+import org.apache.tez.dag.api.TezConfiguration;
+
+import com.fasterxml.jackson.core.JsonParseException;
+import com.google.common.base.Preconditions;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Curator/Zookeeper implementation of {@link AMRegistryClient}.
+ */
+@InterfaceAudience.Public
+public final class ZkAMRegistryClient extends AMRegistryClient {
+ private static final Logger LOG = LoggerFactory.getLogger(ZkAMRegistryClient.class);
+ private static final Map<String, ZkAMRegistryClient> INSTANCES = new HashMap<>();
+
+ private final Configuration conf;
+ // Cache of known AMs
+ private final ConcurrentHashMap<ApplicationId, AMRecord> amRecordCache = new ConcurrentHashMap<>();
+
+ private CuratorFramework client;
+ private TreeCache cache;
+ private ZkRegistryListener listener;
+
+ private ZkAMRegistryClient(final Configuration conf) {
+ this.conf = conf;
+ }
+
+ public static synchronized ZkAMRegistryClient getClient(final Configuration conf) {
+ String namespace = conf.get(TezConfiguration.TEZ_AM_REGISTRY_NAMESPACE);
+ ZkAMRegistryClient registry = INSTANCES.get(namespace);
+ if (registry == null) {
+ registry = new ZkAMRegistryClient(conf);
+ INSTANCES.put(namespace, registry);
+ }
+ LOG.info("Returning tez AM registry ({}) for namespace '{}'", System.identityHashCode(registry), namespace);
+ return registry;
+ }
+
+ /**
+ * Deserializes a {@link ServiceRecord} from ZooKeeper data and converts it into an {@link AMRecord}
+ * for caching.
+ *
+ * @param childData the ZooKeeper node data containing a serialized {@link ServiceRecord}
+ * @return an {@link AMRecord} constructed from the deserialized {@link ServiceRecord}, or {@code null}
+ * if no data is present
+ * @throws IOException if the data cannot be deserialized into a {@link ServiceRecord}
+ */
+ public static AMRecord getAMRecord(final ChildData childData) throws IOException {
+ // Not a leaf path. Only leaf path contains AMRecord.
+ if (!childData.getPath().contains(ApplicationId.appIdStrPrefix)) {
+ return null;
+ }
+ byte[] data = childData.getData();
+ // only the path appeared, there is no data yet
+ if (data.length == 0) {
+ return null;
+ }
+ String value = new String(data, StandardCharsets.UTF_8);
+ try {
+ return AMRegistryUtils.jsonStringToRecord(value);
+ } catch (JsonParseException e) {
+ //Not a json AMRecord (SRV), could be some other data.
+ LOG.warn("Non-json data received while de-serializing AMRecord: {}. Ignoring...", value);
+ return null;
+ }
+ }
+
+ public void start() throws Exception {
+ ZkConfig zkConf = new ZkConfig(this.conf);
+ client = zkConf.createCuratorFramework();
+ cache = new TreeCache(client, zkConf.getZkNamespace());
+ client.start();
+ cache.start();
+ listener = new ZkRegistryListener();
+ cache.getListenable().addListener(listener);
+ }
+
+ @Override
+ public AMRecord getRecord(ApplicationId appId) {
+ AMRecord rec = amRecordCache.get(appId);
+ // Return a copy.
+ return rec == null ? null : new AMRecord(rec);
+ }
+
+ @Override
+ public List<AMRecord> getAllRecords() {
+ return amRecordCache.values().stream().map(AMRecord::new).collect(Collectors.toList());
+ }
+
+ @Override
+ public void close() {
+ IOUtils.closeQuietly(cache);
+ IOUtils.closeQuietly(client);
Review Comment:
solution is in this commit:
https://github.com/apache/tez/pull/427/commits/ad531a86ca6c8005e98413226cf4c6f0cd6884f0
investigated the downstream distro, and it turned out that Hive currently doesn't call TezClient.close on external sessions, and according to code comments, it's by design
I attempted and fix this in the above comment
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] TEZ-4007: Introduce AmExtensions and Zookeeper-based FrameworkServices [tez]
Posted by "abstractdog (via GitHub)" <gi...@apache.org>.
abstractdog commented on code in PR #427:
URL: https://github.com/apache/tez/pull/427#discussion_r2563630356
##########
tez-api/src/main/java/org/apache/tez/frameworkplugins/FrameworkUtils.java:
##########
@@ -0,0 +1,96 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tez.frameworkplugins;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.ServiceLoader;
+
+import javax.annotation.Nullable;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.tez.common.ReflectionUtils;
+import org.apache.tez.dag.api.TezConfiguration;
+import org.apache.tez.dag.api.TezConstants;
+import org.apache.tez.dag.api.TezReflectionException;
+
+public final class FrameworkUtils {
+
+ private FrameworkUtils() {}
+
+ /*
+ Searches for a FrameworkService provider which implements a target interface.
+ The interface should be either ClientFrameworkService or ServerFrameworkService.
+ Depending on which interface is used, either the client or server class of a
+ matching FrameworkMode will be used as the implementation.
+
+ NOTE: Layering of FrameworkServices in a decorator-style is currently not supported
+
+ An implementation is searched in the following order:
+ 1. If conf is not null and the parameter TEZ_FRAMEWORK_MODE is set:
+ the value of TEZ_FRAMEWORK_MODE from the conf will be used
+ 2. If conf is null or the parameter TEZ_FRAMEWORK_MODE is not set
+ and the environment var TEZ_FRAMEWORK_MODE is not empty:
+ the value of the environment var will be used
+ 3. Otherwise:
+ the default java.util.ServiceLoader behavior will be used,
+ i.e. the implementation classname should appear in a file on the classpath at the location
+ META-INF/services/org.apache.tez.frameworkplugins.ClientFrameworkService
+ or META-INF/services/org.apache.tez.frameworkplugins.ServerFrameworkService
+ */
+ public static <T extends FrameworkService> T get(Class<T> interfaze, @Nullable Configuration conf) {
+ try {
+ if ((conf != null) && (conf.get(TezConfiguration.TEZ_FRAMEWORK_MODE) != null)) {
+ return getByMode(interfaze, conf.get(TezConfiguration.TEZ_FRAMEWORK_MODE));
+ } else if (System.getenv(TezConstants.TEZ_FRAMEWORK_MODE) != null) {
+ return getByMode(interfaze, System.getenv(TezConstants.TEZ_FRAMEWORK_MODE));
+ } else {
+ return getByServiceLoader(interfaze);
+ }
+ } catch (TezReflectionException e) {
+ throw new RuntimeException("Failed to load framework service for interface: " + interfaze.getName(), e);
+ }
+ }
+
+ private static <T extends FrameworkService> T getByServiceLoader(Class<T> interfaze) {
+ List<T> services = new ArrayList<>();
+ ServiceLoader<T> frameworkService = ServiceLoader.load(interfaze);
+ for (T service : frameworkService) {
+ services.add(service);
+ }
+ if (services.isEmpty()) {
+ return null;
Review Comment:
yeah, I was thinking about the same, which might look hacky, but maybe it's worth it
this problem is that this method is in tez-api, whereas the server side stuff is in tez-dag, and tez-api by design cannot depend on tez-dag, so instead of the null-checks, this might need at least 2 runtime workarounds:
1. check if interfaze is `ServerFrameworkService` or `ClientFrameworkService`, and return Yarn one accordingly
2. instantiate the `YarnServerFrameworkService` by reflection (reflection is needed here, becuase even if moving the `ServerFrameworkService` to tez-api fine, doing the same with `YarnServerFrameworkService` looks strange, as it's Tez AM specific implementation
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] TEZ-4007: Introduce AmExtensions and Zookeeper-based FrameworkServices [tez]
Posted by "abstractdog (via GitHub)" <gi...@apache.org>.
abstractdog commented on code in PR #427:
URL: https://github.com/apache/tez/pull/427#discussion_r2565261793
##########
tez-dag/src/main/java/org/apache/tez/dag/api/client/registry/zookeeper/ZkAMRegistry.java:
##########
@@ -0,0 +1,175 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.api.client.registry.zookeeper;
+
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.curator.RetryLoop;
+import org.apache.curator.RetryPolicy;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.registry.client.binding.RegistryUtils;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.tez.client.registry.AMRecord;
+import org.apache.tez.client.registry.AMRegistry;
+import org.apache.tez.client.registry.zookeeper.ZkConfig;
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.data.Stat;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Curator/Zookeeper impl of AMRegistry (for internal use only)
+ * Clients should use org.apache.tez.dag.api.client.registry.zookeeper.ZkAMRegistryClient instead.
+ */
+@InterfaceAudience.Private
+public class ZkAMRegistry extends AMRegistry {
+
+ private static final Logger LOG = LoggerFactory.getLogger(ZkAMRegistry.class);
+
+ private final List<AMRecord> amRecords = new ArrayList<>();
+ private final String externalId;
+
+ private CuratorFramework client = null;
+ private String namespace = null;
+ private ZkConfig zkConfig = null;
+ private boolean started = false;
+
+ public ZkAMRegistry(String externalId) {
+ super("ZkAMRegistry");
+ this.externalId = externalId;
+ }
+
+ @Override
+ public void serviceInit(Configuration conf) {
+ if (zkConfig == null) {
+ zkConfig = new ZkConfig(conf);
+ this.client = zkConfig.createCuratorFramework();
+ this.namespace = zkConfig.getZkNamespace();
+ LOG.info("AMRegistryZkImpl initialized");
+ }
+ }
+
+ @Override
+ public void serviceStart() throws Exception {
+ if (!started) {
+ client.start();
+ started = true;
+ LOG.info("AMRegistryZkImpl started");
+ }
+ }
+
+ //Deletes from Zookeeper AMRecords that were added by this instance
+ @Override
+ public void serviceStop() throws Exception {
+ List<AMRecord> records = new ArrayList<>(amRecords);
+ for (AMRecord amRecord : records) {
+ remove(amRecord);
+ }
+ client.close();
+ LOG.info("AMRegistryZkImpl shutdown");
+ }
+
+ //Serialize AMRecord to ServiceRecord and deliver the JSON bytes to
+ //zkNode at the path: <TEZ_AM_REGISTRY_NAMESPACE>/<appId>
+ @Override
+ public void add(AMRecord server) throws Exception {
+ RegistryUtils.ServiceRecordMarshal marshal = new RegistryUtils.ServiceRecordMarshal();
+ String json = marshal.toJson(server.toServiceRecord());
+ try {
+ final String path = namespace + "/" + server.getApplicationId().toString();
+ client.setData().forPath(path, json.getBytes(StandardCharsets.UTF_8));
+ LOG.info("Added AMRecord to zkpath {}", path);
+ } catch (KeeperException.NoNodeException nne) {
+ client.create().creatingParentContainersIfNeeded().withMode(CreateMode.EPHEMERAL)
+ .forPath(namespace + "/" + server.getApplicationId().toString(), json.getBytes(StandardCharsets.UTF_8));
+ }
+ amRecords.add(server);
+ }
+
+ @Override
+ public void remove(AMRecord server) throws Exception {
+ amRecords.remove(server);
+ final String path = namespace + "/" + server.getApplicationId().toString();
+ client.delete().forPath(path);
+ LOG.info("Deleted AMRecord from zkpath {}", path);
+ }
+
+ @Override
+ public ApplicationId generateNewId() throws Exception {
+ createNamespaceIfNotExists();
+ long namespaceCreationTime = getNamespaceCreationTime();
+
+ boolean success = false;
+ long startTime = System.currentTimeMillis();
+ RetryPolicy retryPolicy = zkConfig.getRetryPolicy();
+ int tryId = 0;
+ for (int i = 0; (i < zkConfig.getCuratorMaxRetries()) && !success; i++) {
+ List<String> children = client.getChildren().forPath(namespace);
+ if (children != null && !children.isEmpty()) {
+ children.sort(Collections.reverseOrder());
+ String last = children.getFirst();
+ ApplicationId lastAppId = ApplicationId.fromString(last);
+ tryId = lastAppId.getId() + 1;
Review Comment:
thread safety is indeed concerning, I was able to reproduce the problem with a unit test case
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] TEZ-4007: Introduce AmExtensions and Zookeeper-based FrameworkServices [tez]
Posted by "tez-yetus (via GitHub)" <gi...@apache.org>.
tez-yetus commented on PR #427:
URL: https://github.com/apache/tez/pull/427#issuecomment-3557195271
:confetti_ball: **+1 overall**
| Vote | Subsystem | Runtime | Logfile | Comment |
|:----:|----------:|--------:|:--------:|:-------:|
| +0 :ok: | reexec | 0m 28s | | Docker mode activated. |
|||| _ Prechecks _ |
| +1 :green_heart: | dupname | 0m 0s | | No case conflicting files found. |
| +0 :ok: | detsecrets | 0m 0s | | detect-secrets was not available. |
| +0 :ok: | xmllint | 0m 0s | | xmllint was not available. |
| +1 :green_heart: | @author | 0m 0s | | The patch does not contain any @author tags. |
| +1 :green_heart: | test4tests | 0m 0s | | The patch appears to include 11 new or modified test files. |
|||| _ master Compile Tests _ |
| +0 :ok: | mvndep | 2m 37s | | Maven dependency ordering for branch |
| +1 :green_heart: | mvninstall | 10m 19s | | master passed |
| +1 :green_heart: | compile | 4m 58s | | master passed |
| +1 :green_heart: | checkstyle | 2m 49s | | master passed |
| +1 :green_heart: | javadoc | 3m 39s | | master passed |
| +0 :ok: | spotbugs | 2m 9s | | tez-api in master has 610 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 0m 42s | | tez-common in master has 13 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 1m 8s | | tez-runtime-library in master has 241 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 0m 40s | | tez-examples in master has 2 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 1m 37s | | tez-dag in master has 785 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 7m 0s | | root in master has 2066 extant spotbugs warnings. |
| -0 :warning: | patch | 7m 43s | | Used diff version of patch file. Binary files and potentially other changes not applied. Please rebase and squash commits if necessary. |
|||| _ Patch Compile Tests _ |
| +0 :ok: | mvndep | 0m 15s | | Maven dependency ordering for patch |
| +1 :green_heart: | mvninstall | 6m 32s | | the patch passed |
| +1 :green_heart: | codespell | 0m 56s | | No new issues. |
| +1 :green_heart: | compile | 5m 2s | | the patch passed |
| +1 :green_heart: | javac | 5m 2s | | the patch passed |
| +1 :green_heart: | blanks | 0m 0s | | The patch has no blanks issues. |
| +1 :green_heart: | checkstyle | 0m 22s | | tez-api: The patch generated 0 new + 118 unchanged - 3 fixed = 118 total (was 121) |
| +1 :green_heart: | checkstyle | 0m 19s | | The patch passed checkstyle in tez-common |
| +1 :green_heart: | checkstyle | 0m 23s | | The patch passed checkstyle in tez-runtime-library |
| +1 :green_heart: | checkstyle | 0m 18s | | tez-examples: The patch generated 0 new + 10 unchanged - 1 fixed = 10 total (was 11) |
| -0 :warning: | checkstyle | 0m 29s | [/results-checkstyle-tez-dag.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/23/artifact/out/results-checkstyle-tez-dag.txt) | tez-dag: The patch generated 1 new + 610 unchanged - 2 fixed = 611 total (was 612) |
| -0 :warning: | checkstyle | 0m 40s | [/results-checkstyle-root.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/23/artifact/out/results-checkstyle-root.txt) | root: The patch generated 1 new + 740 unchanged - 6 fixed = 741 total (was 746) |
| +1 :green_heart: | javadoc | 3m 32s | | the patch passed |
| +1 :green_heart: | spotbugs | 14m 2s | | the patch passed |
|||| _ Other Tests _ |
| +1 :green_heart: | unit | 2m 30s | | tez-api in the patch passed. |
| +1 :green_heart: | unit | 0m 39s | | tez-common in the patch passed. |
| +1 :green_heart: | unit | 5m 59s | | tez-runtime-library in the patch passed. |
| +1 :green_heart: | unit | 0m 27s | | tez-examples in the patch passed. |
| +1 :green_heart: | unit | 5m 17s | | tez-dag in the patch passed. |
| +1 :green_heart: | unit | 71m 42s | | root in the patch passed. |
| +1 :green_heart: | asflicense | 2m 13s | | The patch does not generate ASF License warnings. |
| | | 164m 42s | | |
| Subsystem | Report/Notes |
|----------:|:-------------|
| Docker | ClientAPI=1.52 ServerAPI=1.52 base: https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/23/artifact/out/Dockerfile |
| GITHUB PR | https://github.com/apache/tez/pull/427 |
| Optional Tests | dupname asflicense javac javadoc unit codespell detsecrets xmllint compile spotbugs checkstyle |
| uname | Linux 30c717337d59 5.15.0-161-generic #171-Ubuntu SMP Sat Oct 11 08:17:01 UTC 2025 x86_64 x86_64 x86_64 GNU/Linux |
| Build tool | maven |
| Personality | /home/jenkins/jenkins-home/workspace/tez-multibranch_PR-427/src/.yetus/personality.sh |
| git revision | master / 32070fc74786ccc7f9b67bf2655b6a57dc9eff7c |
| Default Java | Ubuntu-21.0.8+9-Ubuntu-0ubuntu124.04.1 |
| Test Results | https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/23/testReport/ |
| Max. process+thread count | 2107 (vs. ulimit of 5500) |
| modules | C: tez-api tez-common tez-runtime-library tez-examples tez-dag . U: . |
| Console output | https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/23/console |
| versions | git=2.43.0 maven=3.8.7 spotbugs=4.9.3 codespell=2.0.0 |
| Powered by | Apache Yetus 0.15.1 https://yetus.apache.org |
This message was automatically generated.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] TEZ-4007: Introduce AmExtensions and Zookeeper-based FrameworkServices [tez]
Posted by "tez-yetus (via GitHub)" <gi...@apache.org>.
tez-yetus commented on PR #427:
URL: https://github.com/apache/tez/pull/427#issuecomment-3585542172
:broken_heart: **-1 overall**
| Vote | Subsystem | Runtime | Logfile | Comment |
|:----:|----------:|--------:|:--------:|:-------:|
| +0 :ok: | reexec | 0m 31s | | Docker mode activated. |
|||| _ Prechecks _ |
| +1 :green_heart: | dupname | 0m 1s | | No case conflicting files found. |
| +0 :ok: | detsecrets | 0m 0s | | detect-secrets was not available. |
| +0 :ok: | xmllint | 0m 0s | | xmllint was not available. |
| +1 :green_heart: | @author | 0m 0s | | The patch does not contain any @author tags. |
| +1 :green_heart: | test4tests | 0m 0s | | The patch appears to include 14 new or modified test files. |
|||| _ master Compile Tests _ |
| +0 :ok: | mvndep | 2m 37s | | Maven dependency ordering for branch |
| +1 :green_heart: | mvninstall | 10m 8s | | master passed |
| +1 :green_heart: | compile | 5m 23s | | master passed |
| +1 :green_heart: | checkstyle | 2m 57s | | master passed |
| +1 :green_heart: | javadoc | 3m 47s | | master passed |
| +0 :ok: | spotbugs | 2m 10s | | tez-api in master has 610 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 0m 46s | | tez-common in master has 13 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 1m 11s | | tez-runtime-library in master has 241 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 0m 41s | | tez-examples in master has 2 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 1m 46s | | tez-dag in master has 785 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 7m 29s | | root in master has 2066 extant spotbugs warnings. |
| -0 :warning: | patch | 8m 10s | | Used diff version of patch file. Binary files and potentially other changes not applied. Please rebase and squash commits if necessary. |
|||| _ Patch Compile Tests _ |
| +0 :ok: | mvndep | 0m 16s | | Maven dependency ordering for patch |
| +1 :green_heart: | mvninstall | 6m 55s | | the patch passed |
| +1 :green_heart: | codespell | 1m 0s | | No new issues. |
| +1 :green_heart: | compile | 5m 18s | | the patch passed |
| +1 :green_heart: | javac | 5m 18s | | the patch passed |
| +1 :green_heart: | blanks | 0m 0s | | The patch has no blanks issues. |
| -0 :warning: | checkstyle | 0m 21s | [/results-checkstyle-tez-api.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/28/artifact/out/results-checkstyle-tez-api.txt) | tez-api: The patch generated 1 new + 118 unchanged - 3 fixed = 119 total (was 121) |
| +1 :green_heart: | checkstyle | 0m 20s | | The patch passed checkstyle in tez-common |
| +1 :green_heart: | checkstyle | 0m 23s | | The patch passed checkstyle in tez-runtime-library |
| +1 :green_heart: | checkstyle | 0m 18s | | tez-examples: The patch generated 0 new + 10 unchanged - 1 fixed = 10 total (was 11) |
| -0 :warning: | checkstyle | 0m 32s | [/results-checkstyle-tez-dag.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/28/artifact/out/results-checkstyle-tez-dag.txt) | tez-dag: The patch generated 1 new + 610 unchanged - 2 fixed = 611 total (was 612) |
| -0 :warning: | checkstyle | 0m 43s | [/results-checkstyle-root.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/28/artifact/out/results-checkstyle-root.txt) | root: The patch generated 2 new + 740 unchanged - 6 fixed = 742 total (was 746) |
| -1 :x: | javadoc | 0m 39s | [/results-javadoc-javadoc-tez-api.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/28/artifact/out/results-javadoc-javadoc-tez-api.txt) | tez-api generated 1 new + 10 unchanged - 0 fixed = 11 total (was 10) |
| -1 :x: | javadoc | 1m 21s | [/results-javadoc-javadoc-root.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/28/artifact/out/results-javadoc-javadoc-root.txt) | root generated 1 new + 19 unchanged - 0 fixed = 20 total (was 19) |
| -1 :x: | spotbugs | 1m 46s | [/new-spotbugs-tez-api.html](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/28/artifact/out/new-spotbugs-tez-api.html) | tez-api generated 5 new + 610 unchanged - 0 fixed = 615 total (was 610) |
| -1 :x: | spotbugs | 7m 31s | [/new-spotbugs-root.html](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/28/artifact/out/new-spotbugs-root.html) | root generated 5 new + 2066 unchanged - 0 fixed = 2071 total (was 2066) |
|||| _ Other Tests _ |
| -1 :x: | unit | 2m 25s | [/patch-unit-tez-api.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/28/artifact/out/patch-unit-tez-api.txt) | tez-api in the patch passed. |
| +1 :green_heart: | unit | 0m 43s | | tez-common in the patch passed. |
| +1 :green_heart: | unit | 6m 10s | | tez-runtime-library in the patch passed. |
| +1 :green_heart: | unit | 0m 31s | | tez-examples in the patch passed. |
| -1 :x: | unit | 5m 50s | [/patch-unit-tez-dag.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/28/artifact/out/patch-unit-tez-dag.txt) | tez-dag in the patch passed. |
| -1 :x: | unit | 73m 13s | [/patch-unit-root.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/28/artifact/out/patch-unit-root.txt) | root in the patch passed. |
| +1 :green_heart: | asflicense | 2m 23s | | The patch does not generate ASF License warnings. |
| | | 170m 1s | | |
| Reason | Tests |
|-------:|:------|
| SpotBugs | module:tez-api |
| | org.apache.tez.client.registry.AMRecord.toServiceRecord() may expose internal representation by returning AMRecord.serviceRecord At AMRecord.java:by returning AMRecord.serviceRecord At AMRecord.java:[line 183] |
| | Load of known null value in org.apache.tez.frameworkplugins.FrameworkUtils.get(Class, Configuration, Class) At FrameworkUtils.java:in org.apache.tez.frameworkplugins.FrameworkUtils.get(Class, Configuration, Class) At FrameworkUtils.java:[line 65] |
| | Load of known null value in org.apache.tez.frameworkplugins.FrameworkUtils.get(Class, Configuration, Class) At FrameworkUtils.java:in org.apache.tez.frameworkplugins.FrameworkUtils.get(Class, Configuration, Class) At FrameworkUtils.java:[line 65] |
| | Load of known null value in org.apache.tez.frameworkplugins.FrameworkUtils.get(Class, Configuration, Class) At FrameworkUtils.java:in org.apache.tez.frameworkplugins.FrameworkUtils.get(Class, Configuration, Class) At FrameworkUtils.java:[line 65] |
| | Public method org.apache.tez.frameworkplugins.FrameworkUtils.get(Class, Configuration, Class) uses reflection to create a class it gets in its parameter which could increase the accessibility of any class At FrameworkUtils.java:uses reflection to create a class it gets in its parameter which could increase the accessibility of any class At FrameworkUtils.java:[line 63] |
| SpotBugs | module:root |
| | org.apache.tez.client.registry.AMRecord.toServiceRecord() may expose internal representation by returning AMRecord.serviceRecord At AMRecord.java:by returning AMRecord.serviceRecord At AMRecord.java:[line 183] |
| | Load of known null value in org.apache.tez.frameworkplugins.FrameworkUtils.get(Class, Configuration, Class) At FrameworkUtils.java:in org.apache.tez.frameworkplugins.FrameworkUtils.get(Class, Configuration, Class) At FrameworkUtils.java:[line 65] |
| | Load of known null value in org.apache.tez.frameworkplugins.FrameworkUtils.get(Class, Configuration, Class) At FrameworkUtils.java:in org.apache.tez.frameworkplugins.FrameworkUtils.get(Class, Configuration, Class) At FrameworkUtils.java:[line 65] |
| | Load of known null value in org.apache.tez.frameworkplugins.FrameworkUtils.get(Class, Configuration, Class) At FrameworkUtils.java:in org.apache.tez.frameworkplugins.FrameworkUtils.get(Class, Configuration, Class) At FrameworkUtils.java:[line 65] |
| | Public method org.apache.tez.frameworkplugins.FrameworkUtils.get(Class, Configuration, Class) uses reflection to create a class it gets in its parameter which could increase the accessibility of any class At FrameworkUtils.java:uses reflection to create a class it gets in its parameter which could increase the accessibility of any class At FrameworkUtils.java:[line 63] |
| Failed junit tests | tez.dag.api.client.rpc.TestDAGClient |
| | tez.client.registry.zookeeper.TestZkAMRegistryClient |
| | tez.dag.api.client.rpc.TestDAGClient |
| | tez.client.registry.zookeeper.TestZkAMRegistryClient |
| Subsystem | Report/Notes |
|----------:|:-------------|
| Docker | ClientAPI=1.52 ServerAPI=1.52 base: https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/28/artifact/out/Dockerfile |
| GITHUB PR | https://github.com/apache/tez/pull/427 |
| Optional Tests | dupname asflicense javac javadoc unit codespell detsecrets xmllint compile spotbugs checkstyle |
| uname | Linux 924cde8bbe42 5.15.0-161-generic #171-Ubuntu SMP Sat Oct 11 08:17:01 UTC 2025 x86_64 x86_64 x86_64 GNU/Linux |
| Build tool | maven |
| Personality | /home/jenkins/jenkins-home/workspace/tez-multibranch_PR-427@2/src/.yetus/personality.sh |
| git revision | master / 7f72e4aa33348837aa3e99b55ce15eae095cd1f6 |
| Default Java | Ubuntu-21.0.8+9-Ubuntu-0ubuntu124.04.1 |
| Test Results | https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/28/testReport/ |
| Max. process+thread count | 1356 (vs. ulimit of 5500) |
| modules | C: tez-api tez-common tez-runtime-library tez-examples tez-dag . U: . |
| Console output | https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/28/console |
| versions | git=2.43.0 maven=3.8.7 spotbugs=4.9.3 codespell=2.0.0 |
| Powered by | Apache Yetus 0.15.1 https://yetus.apache.org |
This message was automatically generated.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] [DRAFT] TEZ-4007: Zookeeper based FrameworkServices and AmExtensions (3/3) + refactor [tez]
Posted by "tez-yetus (via GitHub)" <gi...@apache.org>.
tez-yetus commented on PR #427:
URL: https://github.com/apache/tez/pull/427#issuecomment-3415594549
:broken_heart: **-1 overall**
| Vote | Subsystem | Runtime | Logfile | Comment |
|:----:|----------:|--------:|:--------:|:-------:|
| +0 :ok: | reexec | 0m 16s | | Docker mode activated. |
|||| _ Prechecks _ |
| +1 :green_heart: | dupname | 0m 1s | | No case conflicting files found. |
| +0 :ok: | detsecrets | 0m 0s | | detect-secrets was not available. |
| +0 :ok: | xmllint | 0m 0s | | xmllint was not available. |
| +1 :green_heart: | @author | 0m 0s | | The patch does not contain any @author tags. |
| +1 :green_heart: | test4tests | 0m 0s | | The patch appears to include 10 new or modified test files. |
|||| _ master Compile Tests _ |
| +0 :ok: | mvndep | 2m 41s | | Maven dependency ordering for branch |
| +1 :green_heart: | mvninstall | 9m 54s | | master passed |
| +1 :green_heart: | compile | 6m 0s | | master passed |
| +1 :green_heart: | checkstyle | 3m 42s | | master passed |
| +1 :green_heart: | javadoc | 4m 40s | | master passed |
| +0 :ok: | spotbugs | 2m 12s | | tez-api in master has 610 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 0m 47s | | tez-common in master has 13 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 1m 12s | | tez-runtime-library in master has 241 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 0m 56s | | tez-mapreduce in master has 129 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 0m 44s | | tez-examples in master has 2 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 1m 43s | | tez-dag in master has 785 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 7m 1s | | root in master has 2066 extant spotbugs warnings. |
|||| _ Patch Compile Tests _ |
| +0 :ok: | mvndep | 0m 18s | | Maven dependency ordering for patch |
| +1 :green_heart: | mvninstall | 6m 45s | | the patch passed |
| +1 :green_heart: | codespell | 0m 51s | | No new issues. |
| +1 :green_heart: | compile | 6m 1s | | the patch passed |
| +1 :green_heart: | javac | 6m 1s | | the patch passed |
| +1 :green_heart: | blanks | 0m 0s | | The patch has no blanks issues. |
| -0 :warning: | checkstyle | 0m 27s | [/results-checkstyle-tez-api.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/11/artifact/out/results-checkstyle-tez-api.txt) | tez-api: The patch generated 3 new + 134 unchanged - 11 fixed = 137 total (was 145) |
| -0 :warning: | checkstyle | 0m 23s | [/results-checkstyle-tez-examples.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/11/artifact/out/results-checkstyle-tez-examples.txt) | tez-examples: The patch generated 6 new + 0 unchanged - 0 fixed = 6 total (was 0) |
| -0 :warning: | checkstyle | 0m 37s | [/results-checkstyle-tez-dag.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/11/artifact/out/results-checkstyle-tez-dag.txt) | tez-dag: The patch generated 1 new + 818 unchanged - 2 fixed = 819 total (was 820) |
| -0 :warning: | checkstyle | 0m 46s | [/results-checkstyle-root.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/11/artifact/out/results-checkstyle-root.txt) | root: The patch generated 10 new + 954 unchanged - 13 fixed = 964 total (was 967) |
| -1 :x: | javadoc | 0m 41s | [/results-javadoc-javadoc-tez-api.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/11/artifact/out/results-javadoc-javadoc-tez-api.txt) | tez-api generated 1 new + 10 unchanged - 0 fixed = 11 total (was 10) |
| -1 :x: | javadoc | 1m 21s | [/results-javadoc-javadoc-root.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/11/artifact/out/results-javadoc-javadoc-root.txt) | root generated 1 new + 19 unchanged - 0 fixed = 20 total (was 19) |
| -1 :x: | spotbugs | 1m 42s | [/new-spotbugs-tez-api.html](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/11/artifact/out/new-spotbugs-tez-api.html) | tez-api generated 1 new + 610 unchanged - 0 fixed = 611 total (was 610) |
| -1 :x: | spotbugs | 7m 16s | [/new-spotbugs-root.html](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/11/artifact/out/new-spotbugs-root.html) | root generated 1 new + 2066 unchanged - 0 fixed = 2067 total (was 2066) |
|||| _ Other Tests _ |
| +1 :green_heart: | unit | 0m 51s | | tez-api in the patch passed. |
| +1 :green_heart: | unit | 0m 47s | | tez-common in the patch passed. |
| +1 :green_heart: | unit | 6m 9s | | tez-runtime-library in the patch passed. |
| +1 :green_heart: | unit | 1m 38s | | tez-mapreduce in the patch passed. |
| +1 :green_heart: | unit | 0m 30s | | tez-examples in the patch passed. |
| +1 :green_heart: | unit | 0m 45s | | tez-dag in the patch passed. |
| -1 :x: | unit | 17m 41s | [/patch-unit-root.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/11/artifact/out/patch-unit-root.txt) | root in the patch passed. |
| -1 :x: | asflicense | 2m 25s | [/results-asflicense.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/11/artifact/out/results-asflicense.txt) | The patch generated 1 ASF License warnings. |
| | | 115m 0s | | |
| Reason | Tests |
|-------:|:------|
| SpotBugs | module:tez-api |
| | new org.apache.tez.client.TezYarnClient(YarnClient) may expose internal representation by storing an externally mutable object into TezYarnClient.yarnClient At TezYarnClient.java:representation by storing an externally mutable object into TezYarnClient.yarnClient At TezYarnClient.java:[line 47] |
| SpotBugs | module:root |
| | new org.apache.tez.client.TezYarnClient(YarnClient) may expose internal representation by storing an externally mutable object into TezYarnClient.yarnClient At TezYarnClient.java:representation by storing an externally mutable object into TezYarnClient.yarnClient At TezYarnClient.java:[line 47] |
| Failed junit tests | tez.dag.history.ats.acls.TestATSHistoryWithACLs |
| | tez.dag.history.ats.acls.TestATSHistoryV15 |
| | tez.auxservices.TestShuffleHandlerJobs |
| | tez.dag.history.logging.ats.TestATSHistoryWithMiniCluster |
| | tez.history.TestHistoryParser |
| | tez.mapreduce.TestMRRJobsDAGApi |
| | tez.test.TestAMRecovery |
| | tez.test.TestPipelinedShuffle |
| | tez.test.TestTaskErrorsUsingLocalMode |
| | tez.test.TestAMRecoveryAggregationBroadcast |
| | tez.test.TestDAGRecovery2 |
| | tez.test.TestRecovery |
| | tez.test.TestLocalMode |
| | tez.test.TestDAGRecovery |
| | tez.test.TestSecureShuffle |
| | tez.test.TestTezJobs |
| | tez.test.TestExceptionPropagation |
| | tez.test.TestFaultTolerance |
| | tez.mapreduce.TestMRRJobs |
| | tez.test.TestAM |
| | tez.analyzer.TestAnalyzer |
| | tez.tests.TestExtServicesWithLocalMode |
| | tez.tests.TestExternalTezServices |
| | tez.tests.TestExternalTezServicesErrors |
| Subsystem | Report/Notes |
|----------:|:-------------|
| Docker | ClientAPI=1.51 ServerAPI=1.51 base: https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/11/artifact/out/Dockerfile |
| GITHUB PR | https://github.com/apache/tez/pull/427 |
| Optional Tests | dupname asflicense javac javadoc unit codespell detsecrets xmllint compile spotbugs checkstyle |
| uname | Linux a01ab8ee0abb 5.15.0-152-generic #162-Ubuntu SMP Wed Jul 23 09:48:42 UTC 2025 x86_64 x86_64 x86_64 GNU/Linux |
| Build tool | maven |
| Personality | /home/jenkins/jenkins-home/workspace/tez-multibranch_PR-427@2/src/.yetus/personality.sh |
| git revision | master / ec303279a96cb6ef9da58d1e47cdf9d13e4a2b07 |
| Default Java | Ubuntu-21.0.8+9-Ubuntu-0ubuntu124.04.1 |
| Test Results | https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/11/testReport/ |
| Max. process+thread count | 1373 (vs. ulimit of 5500) |
| modules | C: tez-api tez-common tez-runtime-library tez-mapreduce tez-examples tez-dag . U: . |
| Console output | https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/11/console |
| versions | git=2.43.0 maven=3.8.7 spotbugs=4.9.3 codespell=2.0.0 |
| Powered by | Apache Yetus 0.15.1 https://yetus.apache.org |
This message was automatically generated.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] [DRAFT] TEZ-4007: Zookeeper based FrameworkServices and AmExtensions (3/3) - initial patch [tez]
Posted by "zhangbutao (via GitHub)" <gi...@apache.org>.
zhangbutao commented on code in PR #427:
URL: https://github.com/apache/tez/pull/427#discussion_r2365698334
##########
tez-api/src/main/java/org/apache/tez/client/registry/zookeeper/ZkAMRegistryClient.java:
##########
@@ -0,0 +1,188 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.tez.client.registry.zookeeper;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
+
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.imps.CuratorFrameworkState;
+import org.apache.curator.framework.recipes.cache.ChildData;
+import org.apache.curator.framework.recipes.cache.PathChildrenCache;
+import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent;
+import org.apache.curator.framework.recipes.cache.PathChildrenCacheListener;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.registry.client.binding.RegistryUtils;
+import org.apache.hadoop.registry.client.types.ServiceRecord;
+import org.apache.tez.client.registry.AMRecord;
+import org.apache.tez.client.registry.AMRegistryClient;
+import org.apache.tez.client.registry.AMRegistryClientListener;
+import org.apache.tez.dag.api.TezConfiguration;
+
+import com.google.common.base.Preconditions;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Curator/Zookeeper impl of AMRegistryClient
+*/
+@InterfaceAudience.Public
+public class ZkAMRegistryClient extends AMRegistryClient {
Review Comment:
Hi folks, sorry, I'd like to chime in with a question here. :)
From the description of [TEZ-3991](https://issues.apache.org/jira/browse/TEZ-3991), this feature seems to allow the AM to manage its own lifecycle, while clients such as HiveServer2 only need to discover available AMs via ZooKeeper (ZK) and submit tasks.
Based on my current limited understanding, the benefits of this feature should include reducing the burden on HS2 and potentially making task submission more efficient.
However, I have a few questions:
- Is this feature primarily beneficial for LLAP mode? Since LLAP requires long-running Tez AMs, and IMO this feature seems more useful for long-running tasks.
- Could this functionality serve as preliminary groundwork for decoupling the Tez framework from YARN? For example, could it pave the way for Tez to run directly on Kubernetes without relying on YARN for resource management?
Thanks.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] TEZ-4007: Introduce AmExtensions and Zookeeper-based FrameworkServices [tez]
Posted by "okumin (via GitHub)" <gi...@apache.org>.
okumin commented on code in PR #427:
URL: https://github.com/apache/tez/pull/427#discussion_r2520588552
##########
tez-api/src/main/java/org/apache/tez/client/registry/zookeeper/ZkFrameworkClient.java:
##########
@@ -0,0 +1,132 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.client.registry.zookeeper;
+
+import java.io.IOException;
+
+import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.client.api.YarnClientApplication;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.util.Records;
+import org.apache.tez.client.FrameworkClient;
+import org.apache.tez.client.registry.AMRecord;
+import org.apache.tez.dag.api.TezConfiguration;
+
+public class ZkFrameworkClient extends FrameworkClient {
+
+ private AMRecord amRecord;
+ private ZkAMRegistryClient amRegistryClient = null;
+ private volatile boolean isRunning = false;
+ private String amHost;
+ private int amPort;
+
+ @Override
+ public synchronized void init(TezConfiguration tezConf) {
+ if (this.amRegistryClient == null) {
+ try {
+ this.amRegistryClient = ZkAMRegistryClient.getClient(tezConf);
+ this.isRunning = true;
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ @Override
+ public void start() {
+ try {
+ this.amRegistryClient.start();
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ isRunning = true;
+ }
+
+ @Override
+ public void stop() {
+ isRunning = false;
+ amRegistryClient.close();
+ }
+
+ @Override
+ public void close() throws IOException {
+ amRegistryClient.close();
+ }
+
+ @Override
+ public YarnClientApplication createApplication() throws YarnException, IOException {
+ ApplicationSubmissionContext context = Records.newRecord(ApplicationSubmissionContext.class);
+ ApplicationId appId = amRecord.getApplicationId();
Review Comment:
I'm not understanding how Tez can bootstrap `amRecord`, which requires `getApplicationReport` to be invoked with an application id. Probably, I'm overlooking something
##########
tez-dag/src/main/java/org/apache/tez/frameworkplugins/yarn/YarnServerFrameworkService.java:
##########
@@ -0,0 +1,121 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tez.frameworkplugins.yarn;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.util.Map;
+import java.util.Optional;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.yarn.api.ApplicationConstants;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.util.ConverterUtils;
+import org.apache.tez.client.registry.AMRegistry;
+import org.apache.tez.common.TezUtilsInternal;
+import org.apache.tez.common.security.JobTokenIdentifier;
+import org.apache.tez.common.security.JobTokenSecretManager;
+import org.apache.tez.common.security.TokenCache;
+import org.apache.tez.dag.api.TezConstants;
+import org.apache.tez.dag.api.records.DAGProtos;
+import org.apache.tez.dag.app.ClusterInfo;
+import org.apache.tez.dag.app.dag.Vertex;
+import org.apache.tez.frameworkplugins.AmExtensions;
+import org.apache.tez.frameworkplugins.ServerFrameworkService;
+import org.apache.tez.frameworkplugins.TaskResourceException;
+
+/**
+ * YARN-based server framework service implementation.
+ * Provides default YARN framework server functionality with default implementations
+ * for all AmExtensions methods.
+ */
+public class YarnServerFrameworkService implements ServerFrameworkService {
+
+ private final YarnAmExtensions amExtensions = new YarnAmExtensions();
+
+ @Override
+ public Optional<AMRegistry> createOrGetAMRegistry(Configuration conf) {
+ // Return empty for now - YARN mode doesn't require a custom AM registry
+ return Optional.empty();
+ }
+
+ @Override
+ public AmExtensions createAmExtensions() {
+ return new YarnAmExtensions();
Review Comment:
```suggestion
return amExtensions;
```
or we can remote the private variable above.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] TEZ-4007: Introduce AmExtensions and Zookeeper-based FrameworkServices [tez]
Posted by "abstractdog (via GitHub)" <gi...@apache.org>.
abstractdog commented on code in PR #427:
URL: https://github.com/apache/tez/pull/427#discussion_r2549763920
##########
tez-api/src/main/java/org/apache/tez/client/registry/AMRegistryClient.java:
##########
@@ -0,0 +1,98 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.client.registry;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Client-side interface for discovering Application Master (AM) instances
+ * registered in the AM registry.
+ *
+ * <p>Implementations are responsible for locating AM endpoints and returning
+ * their metadata. This API is used by client components to discover running
+ * Tez AMs.</p>
+ *
+ * <p>Listeners may be registered to receive notifications when AM records
+ * appear or are removed.</p>
+ */
+public abstract class AMRegistryClient implements Closeable {
+
+ private final List<AMRegistryClientListener> listeners = new ArrayList<>();
+
+ /**
+ * Returns the current set of registered listeners.
+ *
+ * @return a mutable list of listeners
+ */
+ protected List<AMRegistryClientListener> getListeners() {
+ return listeners;
+ }
+
+ /**
+ * Lookup AM metadata for the given application ID.
+ *
+ * @param appId the application ID
+ * @return the AM record if found, otherwise {@code null}
+ * @throws IOException if the lookup fails
+ */
+ public abstract AMRecord getRecord(String appId) throws IOException;
Review Comment:
yeah, also, as ApplicationId's equals and hashCode are properly implemented, we can use it as the Map key type also in ZkAMRegistryClient
```
private final ConcurrentHashMap<ApplicationId, AMRecord> amRecordCache = new ConcurrentHashMap<>();
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] TEZ-4007: Introduce AmExtensions and Zookeeper-based FrameworkServices [tez]
Posted by "abstractdog (via GitHub)" <gi...@apache.org>.
abstractdog commented on code in PR #427:
URL: https://github.com/apache/tez/pull/427#discussion_r2549763920
##########
tez-api/src/main/java/org/apache/tez/client/registry/AMRegistryClient.java:
##########
@@ -0,0 +1,98 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.client.registry;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * Client-side interface for discovering Application Master (AM) instances
+ * registered in the AM registry.
+ *
+ * <p>Implementations are responsible for locating AM endpoints and returning
+ * their metadata. This API is used by client components to discover running
+ * Tez AMs.</p>
+ *
+ * <p>Listeners may be registered to receive notifications when AM records
+ * appear or are removed.</p>
+ */
+public abstract class AMRegistryClient implements Closeable {
+
+ private final List<AMRegistryClientListener> listeners = new ArrayList<>();
+
+ /**
+ * Returns the current set of registered listeners.
+ *
+ * @return a mutable list of listeners
+ */
+ protected List<AMRegistryClientListener> getListeners() {
+ return listeners;
+ }
+
+ /**
+ * Lookup AM metadata for the given application ID.
+ *
+ * @param appId the application ID
+ * @return the AM record if found, otherwise {@code null}
+ * @throws IOException if the lookup fails
+ */
+ public abstract AMRecord getRecord(String appId) throws IOException;
Review Comment:
yeah, also, as ApplicationId's equals and hashCode are properly implemented, we can use it as the Map key type also
```
private final ConcurrentHashMap<ApplicationId, AMRecord> amRecordCache = new ConcurrentHashMap<>();
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] TEZ-4007: Introduce AmExtensions and Zookeeper-based FrameworkServices [tez]
Posted by "tez-yetus (via GitHub)" <gi...@apache.org>.
tez-yetus commented on PR #427:
URL: https://github.com/apache/tez/pull/427#issuecomment-3562298246
:confetti_ball: **+1 overall**
| Vote | Subsystem | Runtime | Logfile | Comment |
|:----:|----------:|--------:|:--------:|:-------:|
| +0 :ok: | reexec | 0m 29s | | Docker mode activated. |
|||| _ Prechecks _ |
| +1 :green_heart: | dupname | 0m 1s | | No case conflicting files found. |
| +0 :ok: | detsecrets | 0m 0s | | detect-secrets was not available. |
| +0 :ok: | xmllint | 0m 0s | | xmllint was not available. |
| +1 :green_heart: | @author | 0m 0s | | The patch does not contain any @author tags. |
| +1 :green_heart: | test4tests | 0m 0s | | The patch appears to include 11 new or modified test files. |
|||| _ master Compile Tests _ |
| +0 :ok: | mvndep | 2m 42s | | Maven dependency ordering for branch |
| +1 :green_heart: | mvninstall | 10m 11s | | master passed |
| +1 :green_heart: | compile | 5m 18s | | master passed |
| +1 :green_heart: | checkstyle | 3m 2s | | master passed |
| +1 :green_heart: | javadoc | 3m 53s | | master passed |
| +0 :ok: | spotbugs | 2m 12s | | tez-api in master has 610 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 0m 46s | | tez-common in master has 13 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 1m 12s | | tez-runtime-library in master has 241 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 0m 44s | | tez-examples in master has 2 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 1m 45s | | tez-dag in master has 785 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 7m 31s | | root in master has 2066 extant spotbugs warnings. |
| -0 :warning: | patch | 8m 14s | | Used diff version of patch file. Binary files and potentially other changes not applied. Please rebase and squash commits if necessary. |
|||| _ Patch Compile Tests _ |
| +0 :ok: | mvndep | 0m 15s | | Maven dependency ordering for patch |
| +1 :green_heart: | mvninstall | 6m 46s | | the patch passed |
| +1 :green_heart: | codespell | 0m 55s | | No new issues. |
| +1 :green_heart: | compile | 5m 22s | | the patch passed |
| +1 :green_heart: | javac | 5m 22s | | the patch passed |
| +1 :green_heart: | blanks | 0m 0s | | The patch has no blanks issues. |
| +1 :green_heart: | checkstyle | 0m 24s | | tez-api: The patch generated 0 new + 118 unchanged - 3 fixed = 118 total (was 121) |
| +1 :green_heart: | checkstyle | 0m 17s | | The patch passed checkstyle in tez-common |
| +1 :green_heart: | checkstyle | 0m 26s | | The patch passed checkstyle in tez-runtime-library |
| +1 :green_heart: | checkstyle | 0m 20s | | tez-examples: The patch generated 0 new + 10 unchanged - 1 fixed = 10 total (was 11) |
| -0 :warning: | checkstyle | 0m 29s | [/results-checkstyle-tez-dag.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/26/artifact/out/results-checkstyle-tez-dag.txt) | tez-dag: The patch generated 1 new + 610 unchanged - 2 fixed = 611 total (was 612) |
| -0 :warning: | checkstyle | 0m 42s | [/results-checkstyle-root.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/26/artifact/out/results-checkstyle-root.txt) | root: The patch generated 1 new + 740 unchanged - 6 fixed = 741 total (was 746) |
| +1 :green_heart: | javadoc | 3m 40s | | the patch passed |
| +1 :green_heart: | spotbugs | 14m 35s | | the patch passed |
|||| _ Other Tests _ |
| +1 :green_heart: | unit | 2m 34s | | tez-api in the patch passed. |
| +1 :green_heart: | unit | 0m 41s | | tez-common in the patch passed. |
| +1 :green_heart: | unit | 5m 58s | | tez-runtime-library in the patch passed. |
| +1 :green_heart: | unit | 0m 31s | | tez-examples in the patch passed. |
| +1 :green_heart: | unit | 5m 26s | | tez-dag in the patch passed. |
| +1 :green_heart: | unit | 71m 57s | | root in the patch passed. |
| +1 :green_heart: | asflicense | 2m 41s | | The patch does not generate ASF License warnings. |
| | | 168m 44s | | |
| Subsystem | Report/Notes |
|----------:|:-------------|
| Docker | ClientAPI=1.52 ServerAPI=1.52 base: https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/26/artifact/out/Dockerfile |
| GITHUB PR | https://github.com/apache/tez/pull/427 |
| Optional Tests | dupname asflicense javac javadoc unit codespell detsecrets xmllint compile spotbugs checkstyle |
| uname | Linux 7640fc86df1a 5.15.0-161-generic #171-Ubuntu SMP Sat Oct 11 08:17:01 UTC 2025 x86_64 x86_64 x86_64 GNU/Linux |
| Build tool | maven |
| Personality | /home/jenkins/jenkins-home/workspace/tez-multibranch_PR-427/src/.yetus/personality.sh |
| git revision | master / b3ca9d5af65a6bc9b5fc51456cf748ecd2f6b1de |
| Default Java | Ubuntu-21.0.8+9-Ubuntu-0ubuntu124.04.1 |
| Test Results | https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/26/testReport/ |
| Max. process+thread count | 1403 (vs. ulimit of 5500) |
| modules | C: tez-api tez-common tez-runtime-library tez-examples tez-dag . U: . |
| Console output | https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/26/console |
| versions | git=2.43.0 maven=3.8.7 spotbugs=4.9.3 codespell=2.0.0 |
| Powered by | Apache Yetus 0.15.1 https://yetus.apache.org |
This message was automatically generated.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] TEZ-4007: Introduce AmExtensions and Zookeeper-based FrameworkServices [tez]
Posted by "abstractdog (via GitHub)" <gi...@apache.org>.
abstractdog commented on code in PR #427:
URL: https://github.com/apache/tez/pull/427#discussion_r2549815047
##########
tez-api/src/main/java/org/apache/tez/client/registry/zookeeper/ZkAMRegistryClient.java:
##########
@@ -0,0 +1,205 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.client.registry.zookeeper;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
+
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.imps.CuratorFrameworkState;
+import org.apache.curator.framework.recipes.cache.ChildData;
+import org.apache.curator.framework.recipes.cache.TreeCache;
+import org.apache.curator.framework.recipes.cache.TreeCacheEvent;
+import org.apache.curator.framework.recipes.cache.TreeCacheListener;
+import org.apache.curator.shaded.com.google.common.base.Charsets;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.registry.client.binding.RegistryUtils;
+import org.apache.hadoop.registry.client.types.ServiceRecord;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.tez.client.registry.AMRecord;
+import org.apache.tez.client.registry.AMRegistryClient;
+import org.apache.tez.client.registry.AMRegistryClientListener;
+import org.apache.tez.dag.api.TezConfiguration;
+
+import com.fasterxml.jackson.core.JsonParseException;
+import com.google.common.base.Preconditions;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Curator/Zookeeper implementation of {@link AMRegistryClient}.
+ */
+@InterfaceAudience.Public
+public final class ZkAMRegistryClient extends AMRegistryClient {
+ private static final Logger LOG = LoggerFactory.getLogger(ZkAMRegistryClient.class);
+ private static final Map<String, ZkAMRegistryClient> INSTANCES = new HashMap<>();
+
+ private final Configuration conf;
+ //Cache of known AMs
+ private final ConcurrentHashMap<String, AMRecord> amRecordCache = new ConcurrentHashMap<>();
+ private CuratorFramework client;
+
+ private ZkAMRegistryClient(final Configuration conf) {
+ this.conf = conf;
+ }
+
+ public static synchronized ZkAMRegistryClient getClient(final Configuration conf) {
+ String namespace = conf.get(TezConfiguration.TEZ_AM_REGISTRY_NAMESPACE);
+ ZkAMRegistryClient registry = INSTANCES.get(namespace);
+ if (registry == null) {
+ registry = new ZkAMRegistryClient(conf);
+ INSTANCES.put(namespace, registry);
+ }
+ LOG.info("Returning tez AM registry ({}) for namespace '{}'", System.identityHashCode(registry), namespace);
+ return registry;
+ }
+
+ /**
+ * Deserializes a {@link ServiceRecord} from ZooKeeper data and converts it into an {@link AMRecord}
+ * for caching.
+ *
+ * @param childData the ZooKeeper node data containing a serialized {@link ServiceRecord}
+ * @return an {@link AMRecord} constructed from the deserialized {@link ServiceRecord}, or {@code null}
+ * if no data is present
+ * @throws IOException if the data cannot be deserialized into a {@link ServiceRecord}
+ */
+ public static AMRecord getAMRecord(final ChildData childData) throws IOException {
+ // not a leaf path. Only leaf path contains AMRecord
+ if (!childData.getPath().contains(ApplicationId.appIdStrPrefix)) {
+ return null;
+ }
+ byte[] data = childData.getData();
+ // only the path appeared, there is no data yet
+ if (data.length == 0) {
+ return null;
+ }
+ String value = new String(data, Charsets.UTF_8);
+ try {
+ RegistryUtils.ServiceRecordMarshal marshal = new RegistryUtils.ServiceRecordMarshal();
+ ServiceRecord serviceRecord = marshal.fromJson(value);
+ return new AMRecord(serviceRecord);
+ } catch (JsonParseException e) {
+ //Not a json AMRecord (SRV), could be some other data.
+ LOG.warn("Non-json data received while de-serializing AMRecord: {}. Ignoring...", value);
+ return null;
+ }
+ }
+
+ public void start() throws Exception {
+ ZkConfig zkConf = new ZkConfig(this.conf);
+ client = zkConf.createCuratorFramework();
+ final TreeCache cache = new TreeCache(client, zkConf.getZkNamespace());
+ client.start();
+ cache.start();
+ cache.getListenable().addListener(new ZkRegistryListener());
+ }
+
+ @Override
+ public AMRecord getRecord(String appId) {
+ if (amRecordCache.get(appId) == null) {
+ return null;
+ }
+ //Return a copy
+ return new AMRecord(amRecordCache.get(appId));
+ }
+
+ @Override
+ public List<AMRecord> getAllRecords() {
+ return amRecordCache.values().stream().map(AMRecord::new).collect(Collectors.toList());
+ }
+
+ @Override
+ public synchronized void addListener(AMRegistryClientListener listener) {
+ getListeners().add(listener);
+ }
+
+ @Override
+ public void close() {
+ client.close();
+ }
+
+ /**
+ * Callback listener for ZooKeeper events that updates the local cache
+ * when child nodes under the monitored path change.
+ */
+ private class ZkRegistryListener implements TreeCacheListener {
+
+ @Override
+ public void childEvent(final CuratorFramework clientParam, final TreeCacheEvent event) throws Exception {
+ Preconditions.checkArgument(clientParam != null && clientParam.getState() == CuratorFrameworkState.STARTED,
+ "Curator client is not started");
+
+ ChildData childData = event.getData();
+ switch (event.getType()) {
+ case NODE_ADDED:
+ if (isEmpty(childData)) {
+ LOG.info("AppId allocated: {}", childData.getPath());
+ } else {
+ AMRecord amRecord = getAMRecord(childData);
+ if (amRecord != null) {
+ LOG.info("AM registered with data: {}. Notifying {} listeners.", amRecord, getListeners().size());
+ amRecordCache.put(amRecord.getApplicationId().toString(), amRecord);
+ notifyOnAdded(amRecord);
+ }
+ }
+ break;
+ case NODE_UPDATED:
+ if (isEmpty(childData)) {
+ throw new RuntimeException("AM updated with empty data");
+ } else {
+ AMRecord amRecord = getAMRecord(childData);
+ if (amRecord != null) {
+ LOG.info("AM updated data: {}. Notifying {} listeners.", amRecord, getListeners().size());
+ amRecordCache.put(amRecord.getApplicationId().toString(), amRecord);
+ notifyOnAdded(amRecord);
+ }
+ }
+ break;
+ case NODE_REMOVED:
+ if (isEmpty(childData)) {
+ LOG.info("Unused AppId unregistered: {}", childData.getPath());
+ } else {
+ AMRecord amRecord = getAMRecord(childData);
+ if (amRecord != null) {
+ LOG.info("AM removed: {}. Notifying {} listeners.", amRecord, getListeners().size());
+ amRecordCache.remove(amRecord.getApplicationId().toString(), amRecord);
Review Comment:
refactored amRecordCache collection key to ApplicationId according to other comments, so this problem is resolved
no matter the AMRecord instance, the encapsulated ApplicationId.equals() takes care of the correct behavior:
```
public boolean equals(Object obj) {
if (this == obj)
return true;
if (obj == null)
return false;
if (getClass() != obj.getClass())
return false;
ApplicationId other = (ApplicationId) obj;
if (this.getClusterTimestamp() != other.getClusterTimestamp())
return false;
if (this.getId() != other.getId())
return false;
return true;
}
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] TEZ-4007: Zookeeper based FrameworkServices and AmExtensions (3/3) - initial patch [tez]
Posted by "okumin (via GitHub)" <gi...@apache.org>.
okumin commented on code in PR #427:
URL: https://github.com/apache/tez/pull/427#discussion_r2338352708
##########
tez-api/src/main/java/org/apache/tez/client/registry/zookeeper/ZkAMRegistryClient.java:
##########
@@ -0,0 +1,188 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+*/
+
+package org.apache.tez.client.registry.zookeeper;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
+
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.imps.CuratorFrameworkState;
+import org.apache.curator.framework.recipes.cache.ChildData;
+import org.apache.curator.framework.recipes.cache.PathChildrenCache;
+import org.apache.curator.framework.recipes.cache.PathChildrenCacheEvent;
+import org.apache.curator.framework.recipes.cache.PathChildrenCacheListener;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.registry.client.binding.RegistryUtils;
+import org.apache.hadoop.registry.client.types.ServiceRecord;
+import org.apache.tez.client.registry.AMRecord;
+import org.apache.tez.client.registry.AMRegistryClient;
+import org.apache.tez.client.registry.AMRegistryClientListener;
+import org.apache.tez.dag.api.TezConfiguration;
+
+import com.google.common.base.Preconditions;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Curator/Zookeeper impl of AMRegistryClient
+*/
+@InterfaceAudience.Public
+public class ZkAMRegistryClient extends AMRegistryClient {
Review Comment:
Just a question. What is the duty of ZooKeeper here? When I read this patch a while ago, I presumed ZooKeeper would be used for two reasons. My memory might be wrong.
1. Generate a unique application ID
2. Service discovery for application masters
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] TEZ-4007: Introduce AmExtensions and Zookeeper-based FrameworkServices [tez]
Posted by "tez-yetus (via GitHub)" <gi...@apache.org>.
tez-yetus commented on PR #427:
URL: https://github.com/apache/tez/pull/427#issuecomment-3591711761
:confetti_ball: **+1 overall**
| Vote | Subsystem | Runtime | Logfile | Comment |
|:----:|----------:|--------:|:--------:|:-------:|
| +0 :ok: | reexec | 0m 29s | | Docker mode activated. |
|||| _ Prechecks _ |
| +1 :green_heart: | dupname | 0m 1s | | No case conflicting files found. |
| +0 :ok: | detsecrets | 0m 0s | | detect-secrets was not available. |
| +0 :ok: | xmllint | 0m 0s | | xmllint was not available. |
| +1 :green_heart: | @author | 0m 0s | | The patch does not contain any @author tags. |
| +1 :green_heart: | test4tests | 0m 0s | | The patch appears to include 14 new or modified test files. |
|||| _ master Compile Tests _ |
| +0 :ok: | mvndep | 2m 38s | | Maven dependency ordering for branch |
| +1 :green_heart: | mvninstall | 9m 52s | | master passed |
| +1 :green_heart: | compile | 5m 24s | | master passed |
| +1 :green_heart: | checkstyle | 3m 16s | | master passed |
| +1 :green_heart: | javadoc | 4m 4s | | master passed |
| +0 :ok: | spotbugs | 2m 10s | | tez-api in master has 610 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 0m 45s | | tez-common in master has 13 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 1m 11s | | tez-runtime-library in master has 241 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 0m 43s | | tez-examples in master has 2 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 1m 44s | | tez-dag in master has 785 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 6m 56s | | root in master has 2066 extant spotbugs warnings. |
| -0 :warning: | patch | 7m 40s | | Used diff version of patch file. Binary files and potentially other changes not applied. Please rebase and squash commits if necessary. |
|||| _ Patch Compile Tests _ |
| +0 :ok: | mvndep | 0m 16s | | Maven dependency ordering for patch |
| +1 :green_heart: | mvninstall | 6m 19s | | the patch passed |
| +1 :green_heart: | codespell | 0m 50s | | No new issues. |
| +1 :green_heart: | compile | 5m 17s | | the patch passed |
| +1 :green_heart: | javac | 5m 17s | | the patch passed |
| +1 :green_heart: | blanks | 0m 0s | | The patch has no blanks issues. |
| +1 :green_heart: | checkstyle | 0m 24s | | tez-api: The patch generated 0 new + 118 unchanged - 3 fixed = 118 total (was 121) |
| +1 :green_heart: | checkstyle | 0m 21s | | The patch passed checkstyle in tez-common |
| +1 :green_heart: | checkstyle | 0m 26s | | The patch passed checkstyle in tez-runtime-library |
| +1 :green_heart: | checkstyle | 0m 21s | | tez-examples: The patch generated 0 new + 10 unchanged - 1 fixed = 10 total (was 11) |
| -0 :warning: | checkstyle | 0m 33s | [/results-checkstyle-tez-dag.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/30/artifact/out/results-checkstyle-tez-dag.txt) | tez-dag: The patch generated 1 new + 610 unchanged - 2 fixed = 611 total (was 612) |
| -0 :warning: | checkstyle | 0m 44s | [/results-checkstyle-root.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/30/artifact/out/results-checkstyle-root.txt) | root: The patch generated 1 new + 740 unchanged - 6 fixed = 741 total (was 746) |
| +1 :green_heart: | javadoc | 3m 55s | | the patch passed |
| +1 :green_heart: | spotbugs | 14m 10s | | the patch passed |
|||| _ Other Tests _ |
| +1 :green_heart: | unit | 2m 37s | | tez-api in the patch passed. |
| +1 :green_heart: | unit | 0m 44s | | tez-common in the patch passed. |
| +1 :green_heart: | unit | 6m 8s | | tez-runtime-library in the patch passed. |
| +1 :green_heart: | unit | 0m 32s | | tez-examples in the patch passed. |
| +1 :green_heart: | unit | 6m 28s | | tez-dag in the patch passed. |
| +1 :green_heart: | unit | 74m 14s | | root in the patch passed. |
| +1 :green_heart: | asflicense | 2m 46s | | The patch does not generate ASF License warnings. |
| | | 171m 18s | | |
| Subsystem | Report/Notes |
|----------:|:-------------|
| Docker | ClientAPI=1.52 ServerAPI=1.52 base: https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/30/artifact/out/Dockerfile |
| GITHUB PR | https://github.com/apache/tez/pull/427 |
| Optional Tests | dupname asflicense javac javadoc unit codespell detsecrets xmllint compile spotbugs checkstyle |
| uname | Linux de6817b7aa0a 5.15.0-161-generic #171-Ubuntu SMP Sat Oct 11 08:17:01 UTC 2025 x86_64 x86_64 x86_64 GNU/Linux |
| Build tool | maven |
| Personality | /home/jenkins/jenkins-home/workspace/tez-multibranch_PR-427/src/.yetus/personality.sh |
| git revision | master / 731c4df45a297bca8c594332e174463b2863ce76 |
| Default Java | Ubuntu-21.0.8+9-Ubuntu-0ubuntu124.04.1 |
| Test Results | https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/30/testReport/ |
| Max. process+thread count | 1322 (vs. ulimit of 5500) |
| modules | C: tez-api tez-common tez-runtime-library tez-examples tez-dag . U: . |
| Console output | https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/30/console |
| versions | git=2.43.0 maven=3.8.7 spotbugs=4.9.3 codespell=2.0.0 |
| Powered by | Apache Yetus 0.15.1 https://yetus.apache.org |
This message was automatically generated.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] TEZ-4007: Introduce AmExtensions and Zookeeper-based FrameworkServices [tez]
Posted by "tez-yetus (via GitHub)" <gi...@apache.org>.
tez-yetus commented on PR #427:
URL: https://github.com/apache/tez/pull/427#issuecomment-3601689298
:confetti_ball: **+1 overall**
| Vote | Subsystem | Runtime | Logfile | Comment |
|:----:|----------:|--------:|:--------:|:-------:|
| +0 :ok: | reexec | 0m 30s | | Docker mode activated. |
|||| _ Prechecks _ |
| +1 :green_heart: | dupname | 0m 0s | | No case conflicting files found. |
| +0 :ok: | detsecrets | 0m 1s | | detect-secrets was not available. |
| +0 :ok: | xmllint | 0m 1s | | xmllint was not available. |
| +1 :green_heart: | @author | 0m 0s | | The patch does not contain any @author tags. |
| +1 :green_heart: | test4tests | 0m 0s | | The patch appears to include 14 new or modified test files. |
|||| _ master Compile Tests _ |
| +0 :ok: | mvndep | 2m 38s | | Maven dependency ordering for branch |
| +1 :green_heart: | mvninstall | 9m 51s | | master passed |
| +1 :green_heart: | compile | 5m 23s | | master passed |
| +1 :green_heart: | checkstyle | 3m 14s | | master passed |
| +1 :green_heart: | javadoc | 4m 3s | | master passed |
| +0 :ok: | spotbugs | 2m 9s | | tez-api in master has 610 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 0m 47s | | tez-common in master has 13 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 1m 10s | | tez-runtime-library in master has 241 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 0m 45s | | tez-examples in master has 2 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 1m 44s | | tez-dag in master has 785 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 6m 51s | | root in master has 2066 extant spotbugs warnings. |
| -0 :warning: | patch | 7m 34s | | Used diff version of patch file. Binary files and potentially other changes not applied. Please rebase and squash commits if necessary. |
|||| _ Patch Compile Tests _ |
| +0 :ok: | mvndep | 0m 17s | | Maven dependency ordering for patch |
| +1 :green_heart: | mvninstall | 6m 21s | | the patch passed |
| +1 :green_heart: | codespell | 0m 50s | | No new issues. |
| +1 :green_heart: | compile | 5m 19s | | the patch passed |
| +1 :green_heart: | javac | 5m 19s | | the patch passed |
| +1 :green_heart: | blanks | 0m 0s | | The patch has no blanks issues. |
| +1 :green_heart: | checkstyle | 0m 26s | | tez-api: The patch generated 0 new + 118 unchanged - 3 fixed = 118 total (was 121) |
| +1 :green_heart: | checkstyle | 0m 23s | | The patch passed checkstyle in tez-common |
| +1 :green_heart: | checkstyle | 0m 27s | | The patch passed checkstyle in tez-runtime-library |
| +1 :green_heart: | checkstyle | 0m 23s | | tez-examples: The patch generated 0 new + 10 unchanged - 1 fixed = 10 total (was 11) |
| -0 :warning: | checkstyle | 0m 33s | [/results-checkstyle-tez-dag.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/36/artifact/out/results-checkstyle-tez-dag.txt) | tez-dag: The patch generated 2 new + 610 unchanged - 2 fixed = 612 total (was 612) |
| -0 :warning: | checkstyle | 0m 45s | [/results-checkstyle-root.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/36/artifact/out/results-checkstyle-root.txt) | root: The patch generated 2 new + 740 unchanged - 6 fixed = 742 total (was 746) |
| +1 :green_heart: | javadoc | 3m 59s | | the patch passed |
| +1 :green_heart: | spotbugs | 14m 13s | | the patch passed |
|||| _ Other Tests _ |
| +1 :green_heart: | unit | 2m 36s | | tez-api in the patch passed. |
| +1 :green_heart: | unit | 0m 42s | | tez-common in the patch passed. |
| +1 :green_heart: | unit | 6m 9s | | tez-runtime-library in the patch passed. |
| +1 :green_heart: | unit | 0m 32s | | tez-examples in the patch passed. |
| +1 :green_heart: | unit | 6m 7s | | tez-dag in the patch passed. |
| +1 :green_heart: | unit | 73m 45s | | root in the patch passed. |
| +1 :green_heart: | asflicense | 2m 46s | | The patch does not generate ASF License warnings. |
| | | 170m 36s | | |
| Subsystem | Report/Notes |
|----------:|:-------------|
| Docker | ClientAPI=1.52 ServerAPI=1.52 base: https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/36/artifact/out/Dockerfile |
| GITHUB PR | https://github.com/apache/tez/pull/427 |
| Optional Tests | dupname asflicense javac javadoc unit codespell detsecrets xmllint compile spotbugs checkstyle |
| uname | Linux e3f0c21725bf 5.15.0-161-generic #171-Ubuntu SMP Sat Oct 11 08:17:01 UTC 2025 x86_64 x86_64 x86_64 GNU/Linux |
| Build tool | maven |
| Personality | /home/jenkins/jenkins-home/workspace/tez-multibranch_PR-427/src/.yetus/personality.sh |
| git revision | master / 39387810bdb01e63d49522712e0b098cf39ad739 |
| Default Java | Ubuntu-21.0.8+9-Ubuntu-0ubuntu124.04.1 |
| Test Results | https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/36/testReport/ |
| Max. process+thread count | 1379 (vs. ulimit of 5500) |
| modules | C: tez-api tez-common tez-runtime-library tez-examples tez-dag . U: . |
| Console output | https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/36/console |
| versions | git=2.43.0 maven=3.8.7 spotbugs=4.9.3 codespell=2.0.0 |
| Powered by | Apache Yetus 0.15.1 https://yetus.apache.org |
This message was automatically generated.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] TEZ-4007: Introduce AmExtensions and Zookeeper-based FrameworkServices [tez]
Posted by "tez-yetus (via GitHub)" <gi...@apache.org>.
tez-yetus commented on PR #427:
URL: https://github.com/apache/tez/pull/427#issuecomment-3598106074
:confetti_ball: **+1 overall**
| Vote | Subsystem | Runtime | Logfile | Comment |
|:----:|----------:|--------:|:--------:|:-------:|
| +0 :ok: | reexec | 0m 24s | | Docker mode activated. |
|||| _ Prechecks _ |
| +1 :green_heart: | dupname | 0m 0s | | No case conflicting files found. |
| +0 :ok: | detsecrets | 0m 0s | | detect-secrets was not available. |
| +0 :ok: | xmllint | 0m 0s | | xmllint was not available. |
| +1 :green_heart: | @author | 0m 0s | | The patch does not contain any @author tags. |
| +1 :green_heart: | test4tests | 0m 0s | | The patch appears to include 14 new or modified test files. |
|||| _ master Compile Tests _ |
| +0 :ok: | mvndep | 2m 27s | | Maven dependency ordering for branch |
| +1 :green_heart: | mvninstall | 8m 1s | | master passed |
| +1 :green_heart: | compile | 2m 50s | | master passed |
| +1 :green_heart: | checkstyle | 1m 34s | | master passed |
| +1 :green_heart: | javadoc | 2m 3s | | master passed |
| +0 :ok: | spotbugs | 1m 29s | | tez-api in master has 610 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 0m 24s | | tez-common in master has 13 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 0m 38s | | tez-runtime-library in master has 241 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 0m 23s | | tez-examples in master has 2 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 0m 51s | | tez-dag in master has 785 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 3m 47s | | root in master has 2066 extant spotbugs warnings. |
| -0 :warning: | patch | 4m 15s | | Used diff version of patch file. Binary files and potentially other changes not applied. Please rebase and squash commits if necessary. |
|||| _ Patch Compile Tests _ |
| +0 :ok: | mvndep | 0m 10s | | Maven dependency ordering for patch |
| +1 :green_heart: | mvninstall | 3m 54s | | the patch passed |
| +1 :green_heart: | codespell | 0m 31s | | No new issues. |
| +1 :green_heart: | compile | 2m 49s | | the patch passed |
| +1 :green_heart: | javac | 2m 49s | | the patch passed |
| +1 :green_heart: | blanks | 0m 0s | | The patch has no blanks issues. |
| +1 :green_heart: | checkstyle | 0m 12s | | tez-api: The patch generated 0 new + 117 unchanged - 3 fixed = 117 total (was 120) |
| +1 :green_heart: | checkstyle | 0m 11s | | The patch passed checkstyle in tez-common |
| +1 :green_heart: | checkstyle | 0m 13s | | The patch passed checkstyle in tez-runtime-library |
| +1 :green_heart: | checkstyle | 0m 11s | | tez-examples: The patch generated 0 new + 11 unchanged - 1 fixed = 11 total (was 12) |
| -0 :warning: | checkstyle | 0m 16s | [/results-checkstyle-tez-dag.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/35/artifact/out/results-checkstyle-tez-dag.txt) | tez-dag: The patch generated 2 new + 610 unchanged - 2 fixed = 612 total (was 612) |
| -0 :warning: | checkstyle | 0m 22s | [/results-checkstyle-root.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/35/artifact/out/results-checkstyle-root.txt) | root: The patch generated 2 new + 740 unchanged - 6 fixed = 742 total (was 746) |
| +1 :green_heart: | javadoc | 2m 1s | | the patch passed |
| +1 :green_heart: | spotbugs | 7m 39s | | the patch passed |
|||| _ Other Tests _ |
| +1 :green_heart: | unit | 2m 1s | | tez-api in the patch passed. |
| +1 :green_heart: | unit | 0m 25s | | tez-common in the patch passed. |
| +1 :green_heart: | unit | 4m 6s | | tez-runtime-library in the patch passed. |
| +1 :green_heart: | unit | 0m 20s | | tez-examples in the patch passed. |
| +1 :green_heart: | unit | 4m 33s | | tez-dag in the patch passed. |
| +1 :green_heart: | unit | 60m 3s | | root in the patch passed. |
| +1 :green_heart: | asflicense | 1m 28s | | The patch does not generate ASF License warnings. |
| | | 119m 27s | | |
| Subsystem | Report/Notes |
|----------:|:-------------|
| Docker | ClientAPI=1.52 ServerAPI=1.52 base: https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/35/artifact/out/Dockerfile |
| GITHUB PR | https://github.com/apache/tez/pull/427 |
| Optional Tests | dupname asflicense javac javadoc unit codespell detsecrets xmllint compile spotbugs checkstyle |
| uname | Linux b839cd43c9e4 5.15.0-156-generic #166-Ubuntu SMP Sat Aug 9 00:02:46 UTC 2025 x86_64 x86_64 x86_64 GNU/Linux |
| Build tool | maven |
| Personality | /home/jenkins/jenkins-home/workspace/tez-multibranch_PR-427/src/.yetus/personality.sh |
| git revision | master / f30ec0002c761ecba03fa8565f7b53f68d8a1c4a |
| Default Java | Ubuntu-21.0.9+10-Ubuntu-124.04 |
| Test Results | https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/35/testReport/ |
| Max. process+thread count | 2110 (vs. ulimit of 5500) |
| modules | C: tez-api tez-common tez-runtime-library tez-examples tez-dag . U: . |
| Console output | https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/35/console |
| versions | git=2.43.0 maven=3.8.7 spotbugs=4.9.3 codespell=2.0.0 |
| Powered by | Apache Yetus 0.15.1 https://yetus.apache.org |
This message was automatically generated.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] TEZ-4007: Introduce AmExtensions and Zookeeper-based FrameworkServices [tez]
Posted by "abstractdog (via GitHub)" <gi...@apache.org>.
abstractdog commented on code in PR #427:
URL: https://github.com/apache/tez/pull/427#discussion_r2551031624
##########
tez-api/src/main/java/org/apache/tez/client/registry/zookeeper/ZkAMRegistryClient.java:
##########
@@ -0,0 +1,205 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.client.registry.zookeeper;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
+
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.imps.CuratorFrameworkState;
+import org.apache.curator.framework.recipes.cache.ChildData;
+import org.apache.curator.framework.recipes.cache.TreeCache;
+import org.apache.curator.framework.recipes.cache.TreeCacheEvent;
+import org.apache.curator.framework.recipes.cache.TreeCacheListener;
+import org.apache.curator.shaded.com.google.common.base.Charsets;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.registry.client.binding.RegistryUtils;
+import org.apache.hadoop.registry.client.types.ServiceRecord;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.tez.client.registry.AMRecord;
+import org.apache.tez.client.registry.AMRegistryClient;
+import org.apache.tez.client.registry.AMRegistryClientListener;
+import org.apache.tez.dag.api.TezConfiguration;
+
+import com.fasterxml.jackson.core.JsonParseException;
+import com.google.common.base.Preconditions;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Curator/Zookeeper implementation of {@link AMRegistryClient}.
+ */
+@InterfaceAudience.Public
+public final class ZkAMRegistryClient extends AMRegistryClient {
+ private static final Logger LOG = LoggerFactory.getLogger(ZkAMRegistryClient.class);
+ private static final Map<String, ZkAMRegistryClient> INSTANCES = new HashMap<>();
+
+ private final Configuration conf;
+ //Cache of known AMs
+ private final ConcurrentHashMap<String, AMRecord> amRecordCache = new ConcurrentHashMap<>();
+ private CuratorFramework client;
+
+ private ZkAMRegistryClient(final Configuration conf) {
+ this.conf = conf;
+ }
+
+ public static synchronized ZkAMRegistryClient getClient(final Configuration conf) {
+ String namespace = conf.get(TezConfiguration.TEZ_AM_REGISTRY_NAMESPACE);
+ ZkAMRegistryClient registry = INSTANCES.get(namespace);
+ if (registry == null) {
+ registry = new ZkAMRegistryClient(conf);
+ INSTANCES.put(namespace, registry);
+ }
+ LOG.info("Returning tez AM registry ({}) for namespace '{}'", System.identityHashCode(registry), namespace);
+ return registry;
+ }
+
+ /**
+ * Deserializes a {@link ServiceRecord} from ZooKeeper data and converts it into an {@link AMRecord}
+ * for caching.
+ *
+ * @param childData the ZooKeeper node data containing a serialized {@link ServiceRecord}
+ * @return an {@link AMRecord} constructed from the deserialized {@link ServiceRecord}, or {@code null}
+ * if no data is present
+ * @throws IOException if the data cannot be deserialized into a {@link ServiceRecord}
+ */
+ public static AMRecord getAMRecord(final ChildData childData) throws IOException {
+ // not a leaf path. Only leaf path contains AMRecord
+ if (!childData.getPath().contains(ApplicationId.appIdStrPrefix)) {
+ return null;
+ }
+ byte[] data = childData.getData();
+ // only the path appeared, there is no data yet
+ if (data.length == 0) {
+ return null;
+ }
+ String value = new String(data, Charsets.UTF_8);
+ try {
+ RegistryUtils.ServiceRecordMarshal marshal = new RegistryUtils.ServiceRecordMarshal();
+ ServiceRecord serviceRecord = marshal.fromJson(value);
+ return new AMRecord(serviceRecord);
+ } catch (JsonParseException e) {
+ //Not a json AMRecord (SRV), could be some other data.
+ LOG.warn("Non-json data received while de-serializing AMRecord: {}. Ignoring...", value);
+ return null;
+ }
+ }
+
+ public void start() throws Exception {
+ ZkConfig zkConf = new ZkConfig(this.conf);
+ client = zkConf.createCuratorFramework();
+ final TreeCache cache = new TreeCache(client, zkConf.getZkNamespace());
+ client.start();
+ cache.start();
+ cache.getListenable().addListener(new ZkRegistryListener());
+ }
+
+ @Override
+ public AMRecord getRecord(String appId) {
+ if (amRecordCache.get(appId) == null) {
+ return null;
+ }
+ //Return a copy
+ return new AMRecord(amRecordCache.get(appId));
+ }
+
+ @Override
+ public List<AMRecord> getAllRecords() {
+ return amRecordCache.values().stream().map(AMRecord::new).collect(Collectors.toList());
+ }
+
+ @Override
+ public synchronized void addListener(AMRegistryClientListener listener) {
+ getListeners().add(listener);
+ }
+
+ @Override
+ public void close() {
+ client.close();
+ }
+
+ /**
+ * Callback listener for ZooKeeper events that updates the local cache
+ * when child nodes under the monitored path change.
+ */
+ private class ZkRegistryListener implements TreeCacheListener {
+
+ @Override
+ public void childEvent(final CuratorFramework clientParam, final TreeCacheEvent event) throws Exception {
+ Preconditions.checkArgument(clientParam != null && clientParam.getState() == CuratorFrameworkState.STARTED,
+ "Curator client is not started");
+
+ ChildData childData = event.getData();
+ switch (event.getType()) {
+ case NODE_ADDED:
+ if (isEmpty(childData)) {
+ LOG.info("AppId allocated: {}", childData.getPath());
+ } else {
+ AMRecord amRecord = getAMRecord(childData);
+ if (amRecord != null) {
+ LOG.info("AM registered with data: {}. Notifying {} listeners.", amRecord, getListeners().size());
Review Comment:
removing the number of listeners from this message altogether, as getListeners was exposed just for logging reasons, and it caused another problems too
##########
tez-api/src/main/java/org/apache/tez/client/registry/zookeeper/ZkAMRegistryClient.java:
##########
@@ -0,0 +1,205 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.client.registry.zookeeper;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
+
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.imps.CuratorFrameworkState;
+import org.apache.curator.framework.recipes.cache.ChildData;
+import org.apache.curator.framework.recipes.cache.TreeCache;
+import org.apache.curator.framework.recipes.cache.TreeCacheEvent;
+import org.apache.curator.framework.recipes.cache.TreeCacheListener;
+import org.apache.curator.shaded.com.google.common.base.Charsets;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.registry.client.binding.RegistryUtils;
+import org.apache.hadoop.registry.client.types.ServiceRecord;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.tez.client.registry.AMRecord;
+import org.apache.tez.client.registry.AMRegistryClient;
+import org.apache.tez.client.registry.AMRegistryClientListener;
+import org.apache.tez.dag.api.TezConfiguration;
+
+import com.fasterxml.jackson.core.JsonParseException;
+import com.google.common.base.Preconditions;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Curator/Zookeeper implementation of {@link AMRegistryClient}.
+ */
+@InterfaceAudience.Public
+public final class ZkAMRegistryClient extends AMRegistryClient {
+ private static final Logger LOG = LoggerFactory.getLogger(ZkAMRegistryClient.class);
+ private static final Map<String, ZkAMRegistryClient> INSTANCES = new HashMap<>();
+
+ private final Configuration conf;
+ //Cache of known AMs
+ private final ConcurrentHashMap<String, AMRecord> amRecordCache = new ConcurrentHashMap<>();
+ private CuratorFramework client;
+
+ private ZkAMRegistryClient(final Configuration conf) {
+ this.conf = conf;
+ }
+
+ public static synchronized ZkAMRegistryClient getClient(final Configuration conf) {
+ String namespace = conf.get(TezConfiguration.TEZ_AM_REGISTRY_NAMESPACE);
+ ZkAMRegistryClient registry = INSTANCES.get(namespace);
+ if (registry == null) {
+ registry = new ZkAMRegistryClient(conf);
+ INSTANCES.put(namespace, registry);
+ }
+ LOG.info("Returning tez AM registry ({}) for namespace '{}'", System.identityHashCode(registry), namespace);
+ return registry;
+ }
+
+ /**
+ * Deserializes a {@link ServiceRecord} from ZooKeeper data and converts it into an {@link AMRecord}
+ * for caching.
+ *
+ * @param childData the ZooKeeper node data containing a serialized {@link ServiceRecord}
+ * @return an {@link AMRecord} constructed from the deserialized {@link ServiceRecord}, or {@code null}
+ * if no data is present
+ * @throws IOException if the data cannot be deserialized into a {@link ServiceRecord}
+ */
+ public static AMRecord getAMRecord(final ChildData childData) throws IOException {
+ // not a leaf path. Only leaf path contains AMRecord
+ if (!childData.getPath().contains(ApplicationId.appIdStrPrefix)) {
+ return null;
+ }
+ byte[] data = childData.getData();
+ // only the path appeared, there is no data yet
+ if (data.length == 0) {
+ return null;
+ }
+ String value = new String(data, Charsets.UTF_8);
+ try {
+ RegistryUtils.ServiceRecordMarshal marshal = new RegistryUtils.ServiceRecordMarshal();
+ ServiceRecord serviceRecord = marshal.fromJson(value);
+ return new AMRecord(serviceRecord);
+ } catch (JsonParseException e) {
+ //Not a json AMRecord (SRV), could be some other data.
+ LOG.warn("Non-json data received while de-serializing AMRecord: {}. Ignoring...", value);
+ return null;
+ }
+ }
+
+ public void start() throws Exception {
+ ZkConfig zkConf = new ZkConfig(this.conf);
+ client = zkConf.createCuratorFramework();
+ final TreeCache cache = new TreeCache(client, zkConf.getZkNamespace());
+ client.start();
+ cache.start();
+ cache.getListenable().addListener(new ZkRegistryListener());
+ }
+
+ @Override
+ public AMRecord getRecord(String appId) {
+ if (amRecordCache.get(appId) == null) {
+ return null;
+ }
+ //Return a copy
+ return new AMRecord(amRecordCache.get(appId));
+ }
+
+ @Override
+ public List<AMRecord> getAllRecords() {
+ return amRecordCache.values().stream().map(AMRecord::new).collect(Collectors.toList());
+ }
+
+ @Override
+ public synchronized void addListener(AMRegistryClientListener listener) {
+ getListeners().add(listener);
+ }
+
+ @Override
+ public void close() {
+ client.close();
+ }
+
+ /**
+ * Callback listener for ZooKeeper events that updates the local cache
+ * when child nodes under the monitored path change.
+ */
+ private class ZkRegistryListener implements TreeCacheListener {
+
+ @Override
+ public void childEvent(final CuratorFramework clientParam, final TreeCacheEvent event) throws Exception {
+ Preconditions.checkArgument(clientParam != null && clientParam.getState() == CuratorFrameworkState.STARTED,
+ "Curator client is not started");
+
+ ChildData childData = event.getData();
+ switch (event.getType()) {
+ case NODE_ADDED:
+ if (isEmpty(childData)) {
+ LOG.info("AppId allocated: {}", childData.getPath());
+ } else {
+ AMRecord amRecord = getAMRecord(childData);
+ if (amRecord != null) {
+ LOG.info("AM registered with data: {}. Notifying {} listeners.", amRecord, getListeners().size());
Review Comment:
removing the number of listeners from this message altogether, as getListeners was exposed just for logging purposes, and it caused another problems too
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] TEZ-4007: Introduce AmExtensions and Zookeeper-based FrameworkServices [tez]
Posted by "tez-yetus (via GitHub)" <gi...@apache.org>.
tez-yetus commented on PR #427:
URL: https://github.com/apache/tez/pull/427#issuecomment-3585935418
:broken_heart: **-1 overall**
| Vote | Subsystem | Runtime | Logfile | Comment |
|:----:|----------:|--------:|:--------:|:-------:|
| +0 :ok: | reexec | 0m 31s | | Docker mode activated. |
|||| _ Prechecks _ |
| +1 :green_heart: | dupname | 0m 0s | | No case conflicting files found. |
| +0 :ok: | detsecrets | 0m 1s | | detect-secrets was not available. |
| +0 :ok: | xmllint | 0m 1s | | xmllint was not available. |
| +1 :green_heart: | @author | 0m 0s | | The patch does not contain any @author tags. |
| +1 :green_heart: | test4tests | 0m 0s | | The patch appears to include 14 new or modified test files. |
|||| _ master Compile Tests _ |
| +0 :ok: | mvndep | 2m 33s | | Maven dependency ordering for branch |
| +1 :green_heart: | mvninstall | 10m 13s | | master passed |
| +1 :green_heart: | compile | 5m 10s | | master passed |
| +1 :green_heart: | checkstyle | 3m 3s | | master passed |
| +1 :green_heart: | javadoc | 3m 55s | | master passed |
| +0 :ok: | spotbugs | 2m 12s | | tez-api in master has 610 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 0m 45s | | tez-common in master has 13 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 1m 11s | | tez-runtime-library in master has 241 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 0m 43s | | tez-examples in master has 2 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 1m 49s | | tez-dag in master has 785 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 7m 31s | | root in master has 2066 extant spotbugs warnings. |
| -0 :warning: | patch | 8m 17s | | Used diff version of patch file. Binary files and potentially other changes not applied. Please rebase and squash commits if necessary. |
|||| _ Patch Compile Tests _ |
| +0 :ok: | mvndep | 0m 16s | | Maven dependency ordering for patch |
| +1 :green_heart: | mvninstall | 6m 46s | | the patch passed |
| +1 :green_heart: | codespell | 0m 53s | | No new issues. |
| +1 :green_heart: | compile | 5m 32s | | the patch passed |
| +1 :green_heart: | javac | 5m 32s | | the patch passed |
| +1 :green_heart: | blanks | 0m 0s | | The patch has no blanks issues. |
| +1 :green_heart: | checkstyle | 0m 23s | | tez-api: The patch generated 0 new + 118 unchanged - 3 fixed = 118 total (was 121) |
| +1 :green_heart: | checkstyle | 0m 20s | | The patch passed checkstyle in tez-common |
| +1 :green_heart: | checkstyle | 0m 26s | | The patch passed checkstyle in tez-runtime-library |
| +1 :green_heart: | checkstyle | 0m 20s | | tez-examples: The patch generated 0 new + 10 unchanged - 1 fixed = 10 total (was 11) |
| -0 :warning: | checkstyle | 0m 30s | [/results-checkstyle-tez-dag.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/29/artifact/out/results-checkstyle-tez-dag.txt) | tez-dag: The patch generated 1 new + 610 unchanged - 2 fixed = 611 total (was 612) |
| -0 :warning: | checkstyle | 0m 43s | [/results-checkstyle-root.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/29/artifact/out/results-checkstyle-root.txt) | root: The patch generated 1 new + 740 unchanged - 6 fixed = 741 total (was 746) |
| +1 :green_heart: | javadoc | 3m 38s | | the patch passed |
| -1 :x: | spotbugs | 1m 43s | [/new-spotbugs-tez-api.html](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/29/artifact/out/new-spotbugs-tez-api.html) | tez-api generated 5 new + 610 unchanged - 0 fixed = 615 total (was 610) |
| -1 :x: | spotbugs | 7m 45s | [/new-spotbugs-root.html](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/29/artifact/out/new-spotbugs-root.html) | root generated 5 new + 2066 unchanged - 0 fixed = 2071 total (was 2066) |
|||| _ Other Tests _ |
| -1 :x: | unit | 2m 24s | [/patch-unit-tez-api.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/29/artifact/out/patch-unit-tez-api.txt) | tez-api in the patch passed. |
| +1 :green_heart: | unit | 0m 40s | | tez-common in the patch passed. |
| +1 :green_heart: | unit | 6m 4s | | tez-runtime-library in the patch passed. |
| +1 :green_heart: | unit | 0m 31s | | tez-examples in the patch passed. |
| -1 :x: | unit | 6m 2s | [/patch-unit-tez-dag.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/29/artifact/out/patch-unit-tez-dag.txt) | tez-dag in the patch passed. |
| -1 :x: | unit | 74m 41s | [/patch-unit-root.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/29/artifact/out/patch-unit-root.txt) | root in the patch passed. |
| +1 :green_heart: | asflicense | 2m 46s | | The patch does not generate ASF License warnings. |
| | | 172m 27s | | |
| Reason | Tests |
|-------:|:------|
| SpotBugs | module:tez-api |
| | org.apache.tez.client.registry.AMRecord.toServiceRecord() may expose internal representation by returning AMRecord.serviceRecord At AMRecord.java:by returning AMRecord.serviceRecord At AMRecord.java:[line 183] |
| | Load of known null value in org.apache.tez.frameworkplugins.FrameworkUtils.get(Class, Configuration, Class) At FrameworkUtils.java:in org.apache.tez.frameworkplugins.FrameworkUtils.get(Class, Configuration, Class) At FrameworkUtils.java:[line 65] |
| | Load of known null value in org.apache.tez.frameworkplugins.FrameworkUtils.get(Class, Configuration, Class) At FrameworkUtils.java:in org.apache.tez.frameworkplugins.FrameworkUtils.get(Class, Configuration, Class) At FrameworkUtils.java:[line 65] |
| | Load of known null value in org.apache.tez.frameworkplugins.FrameworkUtils.get(Class, Configuration, Class) At FrameworkUtils.java:in org.apache.tez.frameworkplugins.FrameworkUtils.get(Class, Configuration, Class) At FrameworkUtils.java:[line 65] |
| | Public method org.apache.tez.frameworkplugins.FrameworkUtils.get(Class, Configuration, Class) uses reflection to create a class it gets in its parameter which could increase the accessibility of any class At FrameworkUtils.java:uses reflection to create a class it gets in its parameter which could increase the accessibility of any class At FrameworkUtils.java:[line 63] |
| SpotBugs | module:root |
| | org.apache.tez.client.registry.AMRecord.toServiceRecord() may expose internal representation by returning AMRecord.serviceRecord At AMRecord.java:by returning AMRecord.serviceRecord At AMRecord.java:[line 183] |
| | Load of known null value in org.apache.tez.frameworkplugins.FrameworkUtils.get(Class, Configuration, Class) At FrameworkUtils.java:in org.apache.tez.frameworkplugins.FrameworkUtils.get(Class, Configuration, Class) At FrameworkUtils.java:[line 65] |
| | Load of known null value in org.apache.tez.frameworkplugins.FrameworkUtils.get(Class, Configuration, Class) At FrameworkUtils.java:in org.apache.tez.frameworkplugins.FrameworkUtils.get(Class, Configuration, Class) At FrameworkUtils.java:[line 65] |
| | Load of known null value in org.apache.tez.frameworkplugins.FrameworkUtils.get(Class, Configuration, Class) At FrameworkUtils.java:in org.apache.tez.frameworkplugins.FrameworkUtils.get(Class, Configuration, Class) At FrameworkUtils.java:[line 65] |
| | Public method org.apache.tez.frameworkplugins.FrameworkUtils.get(Class, Configuration, Class) uses reflection to create a class it gets in its parameter which could increase the accessibility of any class At FrameworkUtils.java:uses reflection to create a class it gets in its parameter which could increase the accessibility of any class At FrameworkUtils.java:[line 63] |
| Failed junit tests | tez.dag.api.client.rpc.TestDAGClient |
| | tez.client.registry.zookeeper.TestZkAMRegistryClient |
| | tez.dag.api.client.rpc.TestDAGClient |
| | tez.client.registry.zookeeper.TestZkAMRegistryClient |
| Subsystem | Report/Notes |
|----------:|:-------------|
| Docker | ClientAPI=1.52 ServerAPI=1.52 base: https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/29/artifact/out/Dockerfile |
| GITHUB PR | https://github.com/apache/tez/pull/427 |
| Optional Tests | dupname asflicense javac javadoc unit codespell detsecrets xmllint compile spotbugs checkstyle |
| uname | Linux d1c7fb3d3fdc 5.15.0-161-generic #171-Ubuntu SMP Sat Oct 11 08:17:01 UTC 2025 x86_64 x86_64 x86_64 GNU/Linux |
| Build tool | maven |
| Personality | /home/jenkins/jenkins-home/workspace/tez-multibranch_PR-427/src/.yetus/personality.sh |
| git revision | master / 4c9ee7fefa6c27c90b11f54e99d65d6a64fd742e |
| Default Java | Ubuntu-21.0.8+9-Ubuntu-0ubuntu124.04.1 |
| Test Results | https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/29/testReport/ |
| Max. process+thread count | 2109 (vs. ulimit of 5500) |
| modules | C: tez-api tez-common tez-runtime-library tez-examples tez-dag . U: . |
| Console output | https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/29/console |
| versions | git=2.43.0 maven=3.8.7 spotbugs=4.9.3 codespell=2.0.0 |
| Powered by | Apache Yetus 0.15.1 https://yetus.apache.org |
This message was automatically generated.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] TEZ-4007: Introduce AmExtensions and Zookeeper-based FrameworkServices [tez]
Posted by "abstractdog (via GitHub)" <gi...@apache.org>.
abstractdog commented on code in PR #427:
URL: https://github.com/apache/tez/pull/427#discussion_r2565040247
##########
tez-api/src/test/java/org/apache/tez/client/TestTezClient.java:
##########
@@ -447,6 +457,94 @@ public TezClientForTest testTezClient(boolean isSession, boolean shouldStop, Str
return client;
}
+ public void testTezClientReconnect(boolean isSession) throws Exception {
+ //Setup 1
+ Map<String, LocalResource> lrs = Maps.newHashMap();
+ String lrName1 = "LR1";
+ lrs.put(lrName1, LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test"),
+ LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1));
+
+ //Client 1
+ TezClientForTest client = configureAndCreateTezClient(lrs, isSession, null);
+
+ //Submission Context 1
+ ArgumentCaptor<ApplicationSubmissionContext> captor = ArgumentCaptor.forClass(ApplicationSubmissionContext.class);
+ when(client.mockYarnClient.getApplicationReport(client.mockAppId).getYarnApplicationState())
+ .thenReturn(YarnApplicationState.RUNNING);
+
+ //Client 1 start
+ client.start();
Review Comment:
yeah, dagClient can and should be closed
I cannot see separate close method in TezClient
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] TEZ-4007: Introduce AmExtensions and Zookeeper-based FrameworkServices [tez]
Posted by "abstractdog (via GitHub)" <gi...@apache.org>.
abstractdog commented on code in PR #427:
URL: https://github.com/apache/tez/pull/427#discussion_r2565261793
##########
tez-dag/src/main/java/org/apache/tez/dag/api/client/registry/zookeeper/ZkAMRegistry.java:
##########
@@ -0,0 +1,175 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.dag.api.client.registry.zookeeper;
+
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.List;
+
+import org.apache.curator.RetryLoop;
+import org.apache.curator.RetryPolicy;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.registry.client.binding.RegistryUtils;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.tez.client.registry.AMRecord;
+import org.apache.tez.client.registry.AMRegistry;
+import org.apache.tez.client.registry.zookeeper.ZkConfig;
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.KeeperException;
+import org.apache.zookeeper.data.Stat;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Curator/Zookeeper impl of AMRegistry (for internal use only)
+ * Clients should use org.apache.tez.dag.api.client.registry.zookeeper.ZkAMRegistryClient instead.
+ */
+@InterfaceAudience.Private
+public class ZkAMRegistry extends AMRegistry {
+
+ private static final Logger LOG = LoggerFactory.getLogger(ZkAMRegistry.class);
+
+ private final List<AMRecord> amRecords = new ArrayList<>();
+ private final String externalId;
+
+ private CuratorFramework client = null;
+ private String namespace = null;
+ private ZkConfig zkConfig = null;
+ private boolean started = false;
+
+ public ZkAMRegistry(String externalId) {
+ super("ZkAMRegistry");
+ this.externalId = externalId;
+ }
+
+ @Override
+ public void serviceInit(Configuration conf) {
+ if (zkConfig == null) {
+ zkConfig = new ZkConfig(conf);
+ this.client = zkConfig.createCuratorFramework();
+ this.namespace = zkConfig.getZkNamespace();
+ LOG.info("AMRegistryZkImpl initialized");
+ }
+ }
+
+ @Override
+ public void serviceStart() throws Exception {
+ if (!started) {
+ client.start();
+ started = true;
+ LOG.info("AMRegistryZkImpl started");
+ }
+ }
+
+ //Deletes from Zookeeper AMRecords that were added by this instance
+ @Override
+ public void serviceStop() throws Exception {
+ List<AMRecord> records = new ArrayList<>(amRecords);
+ for (AMRecord amRecord : records) {
+ remove(amRecord);
+ }
+ client.close();
+ LOG.info("AMRegistryZkImpl shutdown");
+ }
+
+ //Serialize AMRecord to ServiceRecord and deliver the JSON bytes to
+ //zkNode at the path: <TEZ_AM_REGISTRY_NAMESPACE>/<appId>
+ @Override
+ public void add(AMRecord server) throws Exception {
+ RegistryUtils.ServiceRecordMarshal marshal = new RegistryUtils.ServiceRecordMarshal();
+ String json = marshal.toJson(server.toServiceRecord());
+ try {
+ final String path = namespace + "/" + server.getApplicationId().toString();
+ client.setData().forPath(path, json.getBytes(StandardCharsets.UTF_8));
+ LOG.info("Added AMRecord to zkpath {}", path);
+ } catch (KeeperException.NoNodeException nne) {
+ client.create().creatingParentContainersIfNeeded().withMode(CreateMode.EPHEMERAL)
+ .forPath(namespace + "/" + server.getApplicationId().toString(), json.getBytes(StandardCharsets.UTF_8));
+ }
+ amRecords.add(server);
+ }
+
+ @Override
+ public void remove(AMRecord server) throws Exception {
+ amRecords.remove(server);
+ final String path = namespace + "/" + server.getApplicationId().toString();
+ client.delete().forPath(path);
+ LOG.info("Deleted AMRecord from zkpath {}", path);
+ }
+
+ @Override
+ public ApplicationId generateNewId() throws Exception {
+ createNamespaceIfNotExists();
+ long namespaceCreationTime = getNamespaceCreationTime();
+
+ boolean success = false;
+ long startTime = System.currentTimeMillis();
+ RetryPolicy retryPolicy = zkConfig.getRetryPolicy();
+ int tryId = 0;
+ for (int i = 0; (i < zkConfig.getCuratorMaxRetries()) && !success; i++) {
+ List<String> children = client.getChildren().forPath(namespace);
+ if (children != null && !children.isEmpty()) {
+ children.sort(Collections.reverseOrder());
+ String last = children.getFirst();
+ ApplicationId lastAppId = ApplicationId.fromString(last);
+ tryId = lastAppId.getId() + 1;
Review Comment:
added a unit test for this (TestZkAmRegistry.testGenerateNewIdFromParallelThreads)
regarding lexiographical order, this is not the problem, as application IDs have leading zero padding, added log messages before and after
```
2025-11-27 08:22:52,222 INFO [pool-9-thread-13] zookeeper.ZkAMRegistry (ZkAMRegistry.java:generateNewId(141)) - Children before sort: [application_1764228156152_0010, application_1764228156152_0000, application_1764228156152_0011, application_1764228156152_0001, application_1764228156152_0012, application_1764228156152_0002, application_1764228156152_0013, application_1764228156152_0003, application_1764228156152_0014, application_1764228156152_0004, application_1764228156152_0015, application_1764228156152_0005, application_1764228156152_0016, application_1764228156152_0006, application_1764228156152_0017, application_1764228156152_0007, application_1764228156152_0018, application_1764228156152_0008, application_1764228156152_0009]
2025-11-27 08:22:52,222 INFO [pool-9-thread-13] zookeeper.ZkAMRegistry (ZkAMRegistry.java:generateNewId(145)) - Children after sort (last: application_1764228156152_0018): [application_1764228156152_0018, application_1764228156152_0017, application_1764228156152_0016, application_1764228156152_0015, application_1764228156152_0014, application_1764228156152_0013, application_1764228156152_0012, application_1764228156152_0011, application_1764228156152_0010, application_1764228156152_0009, application_1764228156152_0008, application_1764228156152_0007, application_1764228156152_0006, application_1764228156152_0005, application_1764228156152_0004, application_1764228156152_0003, application_1764228156152_0002, application_1764228156152_0001, application_1764228156152_0000]
```
I guess if this were a problem, it should swap "10" and "9" unexpectedly, doesn't even have to go for 99 vs 100
regarding _Retry logic increments tryId, but doesn't re-check children_, I wasn't able to reproduce the problem, which is because we rely on zookeeper here...if the tryId was taken in the meantime, we increment further
```
} catch (KeeperException.NodeExistsException nodeExists) {
LOG.info("Node already exists in ZK for application id {}", tryId);
...
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] TEZ-4007: Introduce AmExtensions and Zookeeper-based FrameworkServices [tez]
Posted by "abstractdog (via GitHub)" <gi...@apache.org>.
abstractdog commented on code in PR #427:
URL: https://github.com/apache/tez/pull/427#discussion_r2563630356
##########
tez-api/src/main/java/org/apache/tez/frameworkplugins/FrameworkUtils.java:
##########
@@ -0,0 +1,96 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tez.frameworkplugins;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.ServiceLoader;
+
+import javax.annotation.Nullable;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.tez.common.ReflectionUtils;
+import org.apache.tez.dag.api.TezConfiguration;
+import org.apache.tez.dag.api.TezConstants;
+import org.apache.tez.dag.api.TezReflectionException;
+
+public final class FrameworkUtils {
+
+ private FrameworkUtils() {}
+
+ /*
+ Searches for a FrameworkService provider which implements a target interface.
+ The interface should be either ClientFrameworkService or ServerFrameworkService.
+ Depending on which interface is used, either the client or server class of a
+ matching FrameworkMode will be used as the implementation.
+
+ NOTE: Layering of FrameworkServices in a decorator-style is currently not supported
+
+ An implementation is searched in the following order:
+ 1. If conf is not null and the parameter TEZ_FRAMEWORK_MODE is set:
+ the value of TEZ_FRAMEWORK_MODE from the conf will be used
+ 2. If conf is null or the parameter TEZ_FRAMEWORK_MODE is not set
+ and the environment var TEZ_FRAMEWORK_MODE is not empty:
+ the value of the environment var will be used
+ 3. Otherwise:
+ the default java.util.ServiceLoader behavior will be used,
+ i.e. the implementation classname should appear in a file on the classpath at the location
+ META-INF/services/org.apache.tez.frameworkplugins.ClientFrameworkService
+ or META-INF/services/org.apache.tez.frameworkplugins.ServerFrameworkService
+ */
+ public static <T extends FrameworkService> T get(Class<T> interfaze, @Nullable Configuration conf) {
+ try {
+ if ((conf != null) && (conf.get(TezConfiguration.TEZ_FRAMEWORK_MODE) != null)) {
+ return getByMode(interfaze, conf.get(TezConfiguration.TEZ_FRAMEWORK_MODE));
+ } else if (System.getenv(TezConstants.TEZ_FRAMEWORK_MODE) != null) {
+ return getByMode(interfaze, System.getenv(TezConstants.TEZ_FRAMEWORK_MODE));
+ } else {
+ return getByServiceLoader(interfaze);
+ }
+ } catch (TezReflectionException e) {
+ throw new RuntimeException("Failed to load framework service for interface: " + interfaze.getName(), e);
+ }
+ }
+
+ private static <T extends FrameworkService> T getByServiceLoader(Class<T> interfaze) {
+ List<T> services = new ArrayList<>();
+ ServiceLoader<T> frameworkService = ServiceLoader.load(interfaze);
+ for (T service : frameworkService) {
+ services.add(service);
+ }
+ if (services.isEmpty()) {
+ return null;
Review Comment:
**UPDATE (ignore things below): I just realized this is already handled in the `getByMode` before the `getByServiceLoader`, so this is another case, sorry for the noise, let me check this**
yeah, I was thinking about the same, which might look hacky, but maybe it's worth it
the problem is that this method is in `tez-api`, whereas the server-side stuff is in `tez-dag`, and `tez-api` by design cannot depend on `tez-dag`, so instead of the null-checks, this might need at least two runtime workarounds:
1. check if `interfaze` is `ServerFrameworkService` or `ClientFrameworkService`, and return Yarn one accordingly
2. instantiate the `YarnServerFrameworkService` by reflection (reflection is needed here, because even if moving the `ServerFrameworkService` to `tez-api` fine, doing the same with `YarnServerFrameworkService` looks strange, as it's Tez AM specific implementation
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] TEZ-4007: Introduce AmExtensions and Zookeeper-based FrameworkServices [tez]
Posted by "abstractdog (via GitHub)" <gi...@apache.org>.
abstractdog commented on code in PR #427:
URL: https://github.com/apache/tez/pull/427#discussion_r2565140913
##########
tez-api/src/test/java/org/apache/tez/client/registry/zookeeper/TestZkFrameworkClient.java:
##########
@@ -0,0 +1,243 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.client.registry.zookeeper;
+
+import static org.junit.Assert.*;
+
+import java.io.File;
+import java.nio.charset.StandardCharsets;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.test.TestingServer;
+import org.apache.hadoop.registry.client.binding.RegistryUtils;
+import org.apache.hadoop.registry.client.types.ServiceRecord;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.client.api.YarnClientApplication;
+import org.apache.tez.client.registry.AMRecord;
+import org.apache.tez.dag.api.TezConfiguration;
+import org.apache.zookeeper.CreateMode;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Unit tests for {@link ZkFrameworkClient}.
+ * <p>
+ * This test class validates the ZooKeeper-based framework client that discovers
+ * and communicates with Application Masters through ZooKeeper registry.
+ * </p>
+ */
+public class TestZkFrameworkClient {
+ private static final Logger LOG = LoggerFactory.getLogger(TestZkFrameworkClient.class);
+ private static final File TEST_DIR = new File(System.getProperty("test.build.data", "target"),
+ TestZkFrameworkClient.class.getName()).getAbsoluteFile();
+
+ private TestingServer zkServer;
+ private ZkFrameworkClient zkFrameworkClient;
+ private CuratorFramework curatorClient;
+
+ @Before
+ public void setup() throws Exception {
+ zkServer = new TestingServer(true);
+ LOG.info("Started ZooKeeper test server on port: {}", zkServer.getPort());
+ }
+
+ @After
+ public void teardown() throws Exception {
+ if (zkFrameworkClient != null) {
+ zkFrameworkClient.close();
+ }
+ IOUtils.closeQuietly(curatorClient);
+ IOUtils.closeQuietly(zkServer);
+ }
+
+ /**
+ * Tests initialization and lifecycle methods of ZkFrameworkClient.
+ */
+ @Test
+ public void testInitAndLifecycle() throws Exception {
+ TezConfiguration tezConf = createTezConf();
+
+ zkFrameworkClient = new ZkFrameworkClient();
+ zkFrameworkClient.init(tezConf);
+
+ assertTrue("Client should be running after init", zkFrameworkClient.isRunning());
+
+ zkFrameworkClient.start();
+ assertTrue("Client should be running after start", zkFrameworkClient.isRunning());
+
+ zkFrameworkClient.stop();
+ assertFalse("Client should not be running after stop", zkFrameworkClient.isRunning());
+ }
+
+ /**
+ * Tests retrieving application report when AM is registered in ZooKeeper.
+ */
+ @Test
+ public void testGetApplicationReportWithRegisteredAM() throws Exception {
+ TezConfiguration tezConf = createTezConf();
+
+ // Register a mock AM in ZooKeeper
+ ApplicationId appId = ApplicationId.newInstance(System.currentTimeMillis(), 1);
+ String testHostName = "test-host";
+ String testHostIp = "127.0.0.1";
+ int testPort = 12345;
+ registerMockAM(tezConf, appId, testHostName, testHostIp, testPort);
+
+ zkFrameworkClient = new ZkFrameworkClient();
+ zkFrameworkClient.init(tezConf);
+ zkFrameworkClient.start();
+
+ // Give time for ZK registry to initialize
+ Thread.sleep(500);
Review Comment:
ack, the ZK TreeCache INITIALIZED callback can be used for such purposes
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] TEZ-4007: Introduce AmExtensions and Zookeeper-based FrameworkServices [tez]
Posted by "ayushtkn (via GitHub)" <gi...@apache.org>.
ayushtkn commented on code in PR #427:
URL: https://github.com/apache/tez/pull/427#discussion_r2609634048
##########
tez-api/src/main/java/org/apache/tez/client/registry/AMRegistryClient.java:
##########
@@ -0,0 +1,118 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.client.registry;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Client-side interface for discovering Application Master (AM) instances
+ * registered in the AM registry.
+ *
+ * <p>Implementations are responsible for locating AM endpoints and returning
+ * their metadata. This API is used by client components to discover running
+ * Tez AMs.</p>
+ *
+ * <p>Listeners may be registered to receive notifications when AM records
+ * appear or are removed.</p>
+ */
+public abstract class AMRegistryClient implements Closeable {
Review Comment:
can we do ``implements AutoCloseable``?
##########
tez-api/src/main/java/org/apache/tez/client/registry/AMRecord.java:
##########
@@ -125,9 +154,11 @@ public boolean equals(Object other) {
}
if (other instanceof AMRecord otherRecord) {
return appId.equals(otherRecord.appId)
- && host.equals(otherRecord.host)
+ && hostName.equals(otherRecord.hostName)
Review Comment:
can this lead to NPE, we don't have any default here
```
this.hostName = serviceRecord.get(HOST_NAME_RECORD_KEY);
```
##########
tez-api/src/main/java/org/apache/tez/client/registry/zookeeper/ZkFrameworkClient.java:
##########
@@ -0,0 +1,170 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.client.registry.zookeeper;
+
+import java.io.IOException;
+
+import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.client.api.YarnClientApplication;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.util.Records;
+import org.apache.tez.client.FrameworkClient;
+import org.apache.tez.client.registry.AMRecord;
+import org.apache.tez.dag.api.TezConfiguration;
+
+import com.google.common.annotations.VisibleForTesting;
+
+public class ZkFrameworkClient extends FrameworkClient {
+
+ private AMRecord amRecord;
+ private ZkAMRegistryClient amRegistryClient = null;
+ private volatile boolean isRunning = false;
+ private String amHost;
+ private int amPort;
+
+ @Override
+ public synchronized void init(TezConfiguration tezConf) {
+ if (amRegistryClient == null) {
+ try {
+ amRegistryClient = ZkAMRegistryClient.getClient(tezConf);
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ @Override
+ public void start() {
+ try {
+ amRegistryClient.start();
Review Comment:
we should return if `isRunning` is ` true` here
##########
tez-api/src/main/java/org/apache/tez/client/registry/AMRecord.java:
##########
@@ -148,16 +179,27 @@ public boolean equals(Object other) {
* @return a {@link ServiceRecord} populated with the values of this {@code AMRecord}
*/
public ServiceRecord toServiceRecord() {
- ServiceRecord serviceRecord = new ServiceRecord();
+ if (serviceRecord != null) {
+ return serviceRecord;
+ }
+ serviceRecord = new ServiceRecord();
serviceRecord.set(APP_ID_RECORD_KEY, appId);
- serviceRecord.set(HOST_RECORD_KEY, host);
+ serviceRecord.set(HOST_NAME_RECORD_KEY, hostName);
+ serviceRecord.set(HOST_IP_RECORD_KEY, hostIp);
serviceRecord.set(PORT_RECORD_KEY, port);
- serviceRecord.set(OPAQUE_ID_KEY, id);
+ serviceRecord.set(EXTERNAL_ID_KEY, externalId);
+ serviceRecord.set(COMPUTE_GROUP_NAME_KEY, computeName);
+
return serviceRecord;
}
+ @Override
+ public String toString() {
+ return toServiceRecord().attributes().toString();
Review Comment:
`toString` calls `toServiceRecord`, So, it can lead to initializing the ``serviceRecord = new ServiceRecord();``, are we ok with it. This could lead to issues later, like someone logged it or so, before setting the values and then we have the service record cached as well
##########
tez-api/src/main/java/org/apache/tez/client/registry/AMRegistry.java:
##########
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.client.registry;
+
+
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+
+
+/**
+ * Base class for {@code AMRegistry} implementations.
+ *
+ * <p>The specific implementation is configured via the
+ * {@code tez.am.registry.class} property.</p>
+ *
+ * <p>Implementations are expected to provide appropriate service lifecycle
+ * behavior, including:
+ * <ul>
+ * <li>{@code init}</li>
+ * <li>{@code serviceStart}</li>
+ * <li>{@code serviceStop}</li>
+ * </ul>
+ * </p>
+ */
+public interface AMRegistry extends AutoCloseable {
+
+ void add(AMRecord server) throws Exception;
+
+ void remove(AMRecord server) throws Exception;
Review Comment:
should it be `server` or `record`
##########
tez-api/src/test/java/org/apache/tez/client/TestTezClient.java:
##########
@@ -447,6 +457,97 @@ public TezClientForTest testTezClient(boolean isSession, boolean shouldStop, Str
return client;
}
+ public void testTezClientReconnect(boolean isSession) throws Exception {
+ //Setup 1
+ Map<String, LocalResource> lrs = Maps.newHashMap();
+ String lrName1 = "LR1";
+ lrs.put(lrName1, LocalResource.newInstance(URL.newInstance("file", "localhost", 0, "/test"),
+ LocalResourceType.FILE, LocalResourceVisibility.PUBLIC, 1, 1));
+
+ //Client 1
+ TezClientForTest client = configureAndCreateTezClient(lrs, isSession, null);
+
+ //Submission Context 1
+ ArgumentCaptor<ApplicationSubmissionContext> captor = ArgumentCaptor.forClass(ApplicationSubmissionContext.class);
+ when(client.mockYarnClient.getApplicationReport(client.mockAppId).getYarnApplicationState())
+ .thenReturn(YarnApplicationState.RUNNING);
+
+ //Client 1 start
+ client.start();
Review Comment:
missing close?
##########
tez-api/src/main/java/org/apache/tez/client/registry/AMRegistryClient.java:
##########
@@ -0,0 +1,118 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.client.registry;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Client-side interface for discovering Application Master (AM) instances
+ * registered in the AM registry.
+ *
+ * <p>Implementations are responsible for locating AM endpoints and returning
+ * their metadata. This API is used by client components to discover running
+ * Tez AMs.</p>
+ *
+ * <p>Listeners may be registered to receive notifications when AM records
+ * appear or are removed.</p>
+ */
+public abstract class AMRegistryClient implements Closeable {
+ private static final Logger LOG = LoggerFactory.getLogger(AMRegistryClient.class);
+
+ private final List<AMRegistryClientListener> listeners = new ArrayList<>();
+
+ /**
+ * Lookup AM metadata for the given application ID.
+ *
+ * @param appId the application ID
+ * @return the AM record if found, otherwise {@code null}
+ * @throws IOException if the lookup fails
+ */
+ public abstract AMRecord getRecord(ApplicationId appId) throws IOException;
+
+ /**
+ * Retrieve all AM records known in the registry.
+ *
+ * @return a list of AM records (possibly empty)
+ * @throws IOException if the fetch fails
+ */
+ public abstract List<AMRecord> getAllRecords() throws IOException;
+
+ /**
+ * Register a listener for AM registry events.
+ * The listener will be notified when AM records are added or removed.
+ *
+ * @param listener the listener to add
+ */
+ public synchronized void addListener(AMRegistryClientListener listener) {
+ listeners.add(listener);
+ }
+
+ /**
+ * Notify listeners of a newly added AM record.
+ *
+ * @param record the added AM record
+ */
+ protected synchronized void notifyOnAdded(AMRecord record) {
+ for (AMRegistryClientListener listener : listeners) {
+ try {
+ listener.onAdd(record);
+ } catch (Exception e) {
+ LOG.warn("Exception while calling AM add listener, AM record {}", record, e);
+ }
+ }
+ }
+
+ /**
+ * Notify listeners of an updated AM record.
+ *
+ * @param record the updated AM record
+ */
+ protected synchronized void notifyOnUpdated(AMRecord record) {
+ for (AMRegistryClientListener listener : listeners) {
+ try {
+ listener.onUpdate(record);
+ } catch (Exception e) {
+ LOG.warn("Exception while calling AM update listener, AM record {}", record, e);
Review Comment:
nit
can we
```
"Exception while notifying AM update listener for record {}"
```
##########
tez-api/src/main/java/org/apache/tez/client/registry/zookeeper/ZkAMRegistryClient.java:
##########
@@ -0,0 +1,209 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.client.registry.zookeeper;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.imps.CuratorFrameworkState;
+import org.apache.curator.framework.recipes.cache.ChildData;
+import org.apache.curator.framework.recipes.cache.TreeCache;
+import org.apache.curator.framework.recipes.cache.TreeCacheEvent;
+import org.apache.curator.framework.recipes.cache.TreeCacheListener;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.registry.client.types.ServiceRecord;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.tez.client.registry.AMRecord;
+import org.apache.tez.client.registry.AMRegistryClient;
+import org.apache.tez.client.registry.AMRegistryUtils;
+import org.apache.tez.dag.api.TezConfiguration;
+
+import com.fasterxml.jackson.core.JsonParseException;
+import com.google.common.base.Preconditions;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Curator/Zookeeper implementation of {@link AMRegistryClient}.
+ */
+@InterfaceAudience.Public
+public final class ZkAMRegistryClient extends AMRegistryClient {
+ private static final Logger LOG = LoggerFactory.getLogger(ZkAMRegistryClient.class);
+ private static final Map<String, ZkAMRegistryClient> INSTANCES = new HashMap<>();
+
+ private final Configuration conf;
+ // Cache of known AMs
+ private final ConcurrentHashMap<ApplicationId, AMRecord> amRecordCache = new ConcurrentHashMap<>();
+
+ private CuratorFramework client;
+ private TreeCache cache;
+ private ZkRegistryListener listener;
+
+ private ZkAMRegistryClient(final Configuration conf) {
+ this.conf = conf;
+ }
+
+ public static synchronized ZkAMRegistryClient getClient(final Configuration conf) {
+ String namespace = conf.get(TezConfiguration.TEZ_AM_REGISTRY_NAMESPACE);
+ ZkAMRegistryClient registry = INSTANCES.get(namespace);
+ if (registry == null) {
+ registry = new ZkAMRegistryClient(conf);
+ INSTANCES.put(namespace, registry);
+ }
+ LOG.info("Returning tez AM registry ({}) for namespace '{}'", System.identityHashCode(registry), namespace);
+ return registry;
+ }
+
+ /**
+ * Deserializes a {@link ServiceRecord} from ZooKeeper data and converts it into an {@link AMRecord}
+ * for caching.
+ *
+ * @param childData the ZooKeeper node data containing a serialized {@link ServiceRecord}
+ * @return an {@link AMRecord} constructed from the deserialized {@link ServiceRecord}, or {@code null}
+ * if no data is present
+ * @throws IOException if the data cannot be deserialized into a {@link ServiceRecord}
+ */
+ public static AMRecord getAMRecord(final ChildData childData) throws IOException {
+ // Not a leaf path. Only leaf path contains AMRecord.
+ if (!childData.getPath().contains(ApplicationId.appIdStrPrefix)) {
+ return null;
+ }
+ byte[] data = childData.getData();
+ // only the path appeared, there is no data yet
+ if (data.length == 0) {
+ return null;
+ }
+ String value = new String(data, StandardCharsets.UTF_8);
+ try {
+ return AMRegistryUtils.jsonStringToRecord(value);
+ } catch (JsonParseException e) {
+ //Not a json AMRecord (SRV), could be some other data.
+ LOG.warn("Non-json data received while de-serializing AMRecord: {}. Ignoring...", value);
+ return null;
+ }
+ }
+
+ public void start() throws Exception {
+ ZkConfig zkConf = new ZkConfig(this.conf);
+ client = zkConf.createCuratorFramework();
+ cache = new TreeCache(client, zkConf.getZkNamespace());
+ client.start();
+ cache.start();
Review Comment:
shouldn't this be
```
client.start();
client.blockUntilConnected();
cache.start();
```
I think cache.start() normally expects the client to already be started and with an active connection.
##########
tez-api/src/main/java/org/apache/tez/client/registry/zookeeper/ZkFrameworkClient.java:
##########
@@ -0,0 +1,170 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.client.registry.zookeeper;
+
+import java.io.IOException;
+
+import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.client.api.YarnClientApplication;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.util.Records;
+import org.apache.tez.client.FrameworkClient;
+import org.apache.tez.client.registry.AMRecord;
+import org.apache.tez.dag.api.TezConfiguration;
+
+import com.google.common.annotations.VisibleForTesting;
+
+public class ZkFrameworkClient extends FrameworkClient {
+
+ private AMRecord amRecord;
Review Comment:
should this be `volatile`
##########
tez-api/src/main/java/org/apache/tez/client/registry/zookeeper/ZkAMRegistryClient.java:
##########
@@ -0,0 +1,209 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.client.registry.zookeeper;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.imps.CuratorFrameworkState;
+import org.apache.curator.framework.recipes.cache.ChildData;
+import org.apache.curator.framework.recipes.cache.TreeCache;
+import org.apache.curator.framework.recipes.cache.TreeCacheEvent;
+import org.apache.curator.framework.recipes.cache.TreeCacheListener;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.registry.client.types.ServiceRecord;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.tez.client.registry.AMRecord;
+import org.apache.tez.client.registry.AMRegistryClient;
+import org.apache.tez.client.registry.AMRegistryUtils;
+import org.apache.tez.dag.api.TezConfiguration;
+
+import com.fasterxml.jackson.core.JsonParseException;
+import com.google.common.base.Preconditions;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Curator/Zookeeper implementation of {@link AMRegistryClient}.
+ */
+@InterfaceAudience.Public
+public final class ZkAMRegistryClient extends AMRegistryClient {
+ private static final Logger LOG = LoggerFactory.getLogger(ZkAMRegistryClient.class);
+ private static final Map<String, ZkAMRegistryClient> INSTANCES = new HashMap<>();
+
+ private final Configuration conf;
+ // Cache of known AMs
+ private final ConcurrentHashMap<ApplicationId, AMRecord> amRecordCache = new ConcurrentHashMap<>();
+
+ private CuratorFramework client;
+ private TreeCache cache;
+ private ZkRegistryListener listener;
+
+ private ZkAMRegistryClient(final Configuration conf) {
+ this.conf = conf;
+ }
+
+ public static synchronized ZkAMRegistryClient getClient(final Configuration conf) {
+ String namespace = conf.get(TezConfiguration.TEZ_AM_REGISTRY_NAMESPACE);
+ ZkAMRegistryClient registry = INSTANCES.get(namespace);
+ if (registry == null) {
+ registry = new ZkAMRegistryClient(conf);
+ INSTANCES.put(namespace, registry);
+ }
+ LOG.info("Returning tez AM registry ({}) for namespace '{}'", System.identityHashCode(registry), namespace);
+ return registry;
+ }
+
+ /**
+ * Deserializes a {@link ServiceRecord} from ZooKeeper data and converts it into an {@link AMRecord}
+ * for caching.
+ *
+ * @param childData the ZooKeeper node data containing a serialized {@link ServiceRecord}
+ * @return an {@link AMRecord} constructed from the deserialized {@link ServiceRecord}, or {@code null}
+ * if no data is present
+ * @throws IOException if the data cannot be deserialized into a {@link ServiceRecord}
+ */
+ public static AMRecord getAMRecord(final ChildData childData) throws IOException {
+ // Not a leaf path. Only leaf path contains AMRecord.
+ if (!childData.getPath().contains(ApplicationId.appIdStrPrefix)) {
+ return null;
+ }
+ byte[] data = childData.getData();
+ // only the path appeared, there is no data yet
+ if (data.length == 0) {
+ return null;
+ }
+ String value = new String(data, StandardCharsets.UTF_8);
+ try {
+ return AMRegistryUtils.jsonStringToRecord(value);
+ } catch (JsonParseException e) {
+ //Not a json AMRecord (SRV), could be some other data.
+ LOG.warn("Non-json data received while de-serializing AMRecord: {}. Ignoring...", value);
+ return null;
+ }
+ }
+
+ public void start() throws Exception {
+ ZkConfig zkConf = new ZkConfig(this.conf);
+ client = zkConf.createCuratorFramework();
+ cache = new TreeCache(client, zkConf.getZkNamespace());
+ client.start();
+ cache.start();
+ listener = new ZkRegistryListener();
+ cache.getListenable().addListener(listener);
+ }
+
+ @Override
+ public AMRecord getRecord(ApplicationId appId) {
+ AMRecord rec = amRecordCache.get(appId);
+ // Return a copy.
+ return rec == null ? null : new AMRecord(rec);
+ }
+
+ @Override
+ public List<AMRecord> getAllRecords() {
+ return amRecordCache.values().stream().map(AMRecord::new).collect(Collectors.toList());
+ }
+
+ @Override
+ public void close() {
+ IOUtils.closeQuietly(cache);
+ IOUtils.closeQuietly(client);
+ }
+
+ public boolean isInitialized() {
+ return listener.initialized;
+ }
+
+ /**
+ * Callback listener for ZooKeeper events that updates the local cache
+ * when child nodes under the monitored path change.
+ */
+ private class ZkRegistryListener implements TreeCacheListener {
+
+ private boolean initialized = false;
+
+ @Override
+ public void childEvent(final CuratorFramework clientParam, final TreeCacheEvent event) throws Exception {
+ Preconditions.checkArgument(clientParam != null && clientParam.getState() == CuratorFrameworkState.STARTED,
+ "Curator client is not started");
+
+ ChildData childData = event.getData();
+ switch (event.getType()) {
+ case NODE_ADDED:
+ if (isEmpty(childData)) {
+ LOG.info("AppId allocated: {}", childData.getPath());
+ } else {
+ AMRecord amRecord = getAMRecord(childData);
+ if (amRecord != null) {
+ LOG.info("AM registered with data: {}. Notifying listeners.", amRecord);
+ amRecordCache.put(amRecord.getApplicationId(), amRecord);
+ notifyOnAdded(amRecord);
+ }
+ }
+ break;
+ case NODE_UPDATED:
+ if (isEmpty(childData)) {
+ throw new RuntimeException("AM updated with empty data");
+ } else {
+ AMRecord amRecord = getAMRecord(childData);
+ if (amRecord != null) {
+ LOG.info("AM updated data: {}. Notifying listeners.", amRecord);
+ amRecordCache.put(amRecord.getApplicationId(), amRecord);
+ notifyOnAdded(amRecord);
Review Comment:
shouldn't this call ` notifyOnUpdated(amRecord);` instead?
##########
tez-api/src/main/java/org/apache/tez/client/registry/AMRegistry.java:
##########
@@ -0,0 +1,52 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.client.registry;
+
+
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+
+
+/**
+ * Base class for {@code AMRegistry} implementations.
+ *
+ * <p>The specific implementation is configured via the
+ * {@code tez.am.registry.class} property.</p>
+ *
+ * <p>Implementations are expected to provide appropriate service lifecycle
+ * behavior, including:
+ * <ul>
+ * <li>{@code init}</li>
+ * <li>{@code serviceStart}</li>
+ * <li>{@code serviceStop}</li>
+ * </ul>
+ * </p>
+ */
+public interface AMRegistry extends AutoCloseable {
+
+ void add(AMRecord server) throws Exception;
+
+ void remove(AMRecord server) throws Exception;
+
+ ApplicationId generateNewId() throws Exception;
+
+ AMRecord createAmRecord(ApplicationId appId, String hostName, String hostIp, int port,
+ String computeName);
+
+ void close();
Review Comment:
`AutoCloseable` already defines `close`, do we need to define again here?
##########
tez-api/src/main/java/org/apache/tez/frameworkplugins/FrameworkUtils.java:
##########
@@ -0,0 +1,84 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tez.frameworkplugins;
+
+
+import javax.annotation.Nullable;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.tez.common.ReflectionUtils;
+import org.apache.tez.dag.api.TezConfiguration;
+import org.apache.tez.dag.api.TezConstants;
+import org.apache.tez.dag.api.TezReflectionException;
+
+public final class FrameworkUtils {
+
+ private static final String SERVER_FRAMEWORK_SERVICE_INTERFACE_NAME =
+ "org.apache.tez.frameworkplugins.ServerFrameworkService";
+
+ private FrameworkUtils() {}
+
+ /*
+ Searches for a FrameworkService provider which implements a target interface.
+ The interface should be either ClientFrameworkService or ServerFrameworkService.
+ Depending on which interface is used, either the client or server class of a
+ matching FrameworkMode will be used as the implementation.
+
+ NOTE: Layering of FrameworkServices in a decorator-style is currently not supported
+
+ An implementation is searched in the following order:
+ 1. If conf is not null and the parameter TEZ_FRAMEWORK_MODE is set:
+ the value of TEZ_FRAMEWORK_MODE from the conf will be used
+ 2. If conf is null or the parameter TEZ_FRAMEWORK_MODE is not set
+ and the environment var TEZ_FRAMEWORK_MODE is not empty:
+ the value of the environment var will be used
+ 3. Otherwise: the default class will be instantiated and returned
+ */
+ public static <T extends FrameworkService> T get(Class<T> interfaze, @Nullable Configuration conf,
+ Class<?> defaultClazz) {
+ String modeInConf = conf != null ? conf.get(TezConfiguration.TEZ_FRAMEWORK_MODE) : null;
+ String modeInEnv = System.getenv(TezConstants.TEZ_FRAMEWORK_MODE);
+ try {
+ if (modeInConf != null) {
+ return getByMode(interfaze, modeInConf);
+ } else if (modeInEnv != null) {
+ return getByMode(interfaze, modeInEnv);
+ } else if (defaultClazz != null) {
+ return (T) defaultClazz.newInstance();
Review Comment:
I think this is deprecated, does this work
```
defaultClazz.getDeclaredConstructor().newInstance();
```
##########
tez-api/src/main/java/org/apache/tez/client/registry/AMRegistryClient.java:
##########
@@ -0,0 +1,118 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.client.registry;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Client-side interface for discovering Application Master (AM) instances
+ * registered in the AM registry.
+ *
+ * <p>Implementations are responsible for locating AM endpoints and returning
+ * their metadata. This API is used by client components to discover running
+ * Tez AMs.</p>
+ *
+ * <p>Listeners may be registered to receive notifications when AM records
+ * appear or are removed.</p>
+ */
+public abstract class AMRegistryClient implements Closeable {
+ private static final Logger LOG = LoggerFactory.getLogger(AMRegistryClient.class);
+
+ private final List<AMRegistryClientListener> listeners = new ArrayList<>();
+
+ /**
+ * Lookup AM metadata for the given application ID.
+ *
+ * @param appId the application ID
+ * @return the AM record if found, otherwise {@code null}
+ * @throws IOException if the lookup fails
+ */
+ public abstract AMRecord getRecord(ApplicationId appId) throws IOException;
+
+ /**
+ * Retrieve all AM records known in the registry.
+ *
+ * @return a list of AM records (possibly empty)
+ * @throws IOException if the fetch fails
+ */
+ public abstract List<AMRecord> getAllRecords() throws IOException;
+
+ /**
+ * Register a listener for AM registry events.
+ * The listener will be notified when AM records are added or removed.
+ *
+ * @param listener the listener to add
+ */
+ public synchronized void addListener(AMRegistryClientListener listener) {
+ listeners.add(listener);
+ }
+
Review Comment:
why dont' we have `removeListener`?
##########
tez-api/src/main/java/org/apache/tez/client/registry/zookeeper/ZkAMRegistryClient.java:
##########
@@ -0,0 +1,209 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.client.registry.zookeeper;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.imps.CuratorFrameworkState;
+import org.apache.curator.framework.recipes.cache.ChildData;
+import org.apache.curator.framework.recipes.cache.TreeCache;
+import org.apache.curator.framework.recipes.cache.TreeCacheEvent;
+import org.apache.curator.framework.recipes.cache.TreeCacheListener;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.registry.client.types.ServiceRecord;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.tez.client.registry.AMRecord;
+import org.apache.tez.client.registry.AMRegistryClient;
+import org.apache.tez.client.registry.AMRegistryUtils;
+import org.apache.tez.dag.api.TezConfiguration;
+
+import com.fasterxml.jackson.core.JsonParseException;
+import com.google.common.base.Preconditions;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Curator/Zookeeper implementation of {@link AMRegistryClient}.
+ */
+@InterfaceAudience.Public
+public final class ZkAMRegistryClient extends AMRegistryClient {
+ private static final Logger LOG = LoggerFactory.getLogger(ZkAMRegistryClient.class);
+ private static final Map<String, ZkAMRegistryClient> INSTANCES = new HashMap<>();
+
+ private final Configuration conf;
+ // Cache of known AMs
+ private final ConcurrentHashMap<ApplicationId, AMRecord> amRecordCache = new ConcurrentHashMap<>();
+
+ private CuratorFramework client;
+ private TreeCache cache;
+ private ZkRegistryListener listener;
+
+ private ZkAMRegistryClient(final Configuration conf) {
+ this.conf = conf;
+ }
+
+ public static synchronized ZkAMRegistryClient getClient(final Configuration conf) {
+ String namespace = conf.get(TezConfiguration.TEZ_AM_REGISTRY_NAMESPACE);
+ ZkAMRegistryClient registry = INSTANCES.get(namespace);
+ if (registry == null) {
+ registry = new ZkAMRegistryClient(conf);
+ INSTANCES.put(namespace, registry);
+ }
+ LOG.info("Returning tez AM registry ({}) for namespace '{}'", System.identityHashCode(registry), namespace);
+ return registry;
+ }
+
+ /**
+ * Deserializes a {@link ServiceRecord} from ZooKeeper data and converts it into an {@link AMRecord}
+ * for caching.
+ *
+ * @param childData the ZooKeeper node data containing a serialized {@link ServiceRecord}
+ * @return an {@link AMRecord} constructed from the deserialized {@link ServiceRecord}, or {@code null}
+ * if no data is present
+ * @throws IOException if the data cannot be deserialized into a {@link ServiceRecord}
+ */
+ public static AMRecord getAMRecord(final ChildData childData) throws IOException {
+ // Not a leaf path. Only leaf path contains AMRecord.
+ if (!childData.getPath().contains(ApplicationId.appIdStrPrefix)) {
+ return null;
+ }
+ byte[] data = childData.getData();
+ // only the path appeared, there is no data yet
+ if (data.length == 0) {
+ return null;
+ }
+ String value = new String(data, StandardCharsets.UTF_8);
+ try {
+ return AMRegistryUtils.jsonStringToRecord(value);
+ } catch (JsonParseException e) {
+ //Not a json AMRecord (SRV), could be some other data.
+ LOG.warn("Non-json data received while de-serializing AMRecord: {}. Ignoring...", value);
+ return null;
+ }
+ }
+
+ public void start() throws Exception {
+ ZkConfig zkConf = new ZkConfig(this.conf);
+ client = zkConf.createCuratorFramework();
+ cache = new TreeCache(client, zkConf.getZkNamespace());
+ client.start();
+ cache.start();
+ listener = new ZkRegistryListener();
+ cache.getListenable().addListener(listener);
+ }
+
+ @Override
+ public AMRecord getRecord(ApplicationId appId) {
+ AMRecord rec = amRecordCache.get(appId);
+ // Return a copy.
+ return rec == null ? null : new AMRecord(rec);
+ }
+
+ @Override
+ public List<AMRecord> getAllRecords() {
+ return amRecordCache.values().stream().map(AMRecord::new).collect(Collectors.toList());
+ }
+
+ @Override
+ public void close() {
+ IOUtils.closeQuietly(cache);
+ IOUtils.closeQuietly(client);
Review Comment:
should we clean up ``INSTANCES`` here as well?. I am doubting a leak here
##########
tez-api/src/main/java/org/apache/tez/frameworkplugins/FrameworkUtils.java:
##########
@@ -0,0 +1,84 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tez.frameworkplugins;
+
+
+import javax.annotation.Nullable;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.tez.common.ReflectionUtils;
+import org.apache.tez.dag.api.TezConfiguration;
+import org.apache.tez.dag.api.TezConstants;
+import org.apache.tez.dag.api.TezReflectionException;
+
+public final class FrameworkUtils {
+
+ private static final String SERVER_FRAMEWORK_SERVICE_INTERFACE_NAME =
+ "org.apache.tez.frameworkplugins.ServerFrameworkService";
+
+ private FrameworkUtils() {}
+
+ /*
+ Searches for a FrameworkService provider which implements a target interface.
+ The interface should be either ClientFrameworkService or ServerFrameworkService.
+ Depending on which interface is used, either the client or server class of a
+ matching FrameworkMode will be used as the implementation.
+
+ NOTE: Layering of FrameworkServices in a decorator-style is currently not supported
+
+ An implementation is searched in the following order:
+ 1. If conf is not null and the parameter TEZ_FRAMEWORK_MODE is set:
+ the value of TEZ_FRAMEWORK_MODE from the conf will be used
+ 2. If conf is null or the parameter TEZ_FRAMEWORK_MODE is not set
+ and the environment var TEZ_FRAMEWORK_MODE is not empty:
+ the value of the environment var will be used
+ 3. Otherwise: the default class will be instantiated and returned
+ */
+ public static <T extends FrameworkService> T get(Class<T> interfaze, @Nullable Configuration conf,
+ Class<?> defaultClazz) {
+ String modeInConf = conf != null ? conf.get(TezConfiguration.TEZ_FRAMEWORK_MODE) : null;
+ String modeInEnv = System.getenv(TezConstants.TEZ_FRAMEWORK_MODE);
+ try {
+ if (modeInConf != null) {
Review Comment:
should it be StringUtils.isNotEmpty()?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] TEZ-4007: Introduce AmExtensions and Zookeeper-based FrameworkServices [tez]
Posted by "abstractdog (via GitHub)" <gi...@apache.org>.
abstractdog commented on code in PR #427:
URL: https://github.com/apache/tez/pull/427#discussion_r2669024497
##########
tez-api/src/main/java/org/apache/tez/client/registry/AMRecord.java:
##########
@@ -148,16 +179,27 @@ public boolean equals(Object other) {
* @return a {@link ServiceRecord} populated with the values of this {@code AMRecord}
*/
public ServiceRecord toServiceRecord() {
- ServiceRecord serviceRecord = new ServiceRecord();
+ if (serviceRecord != null) {
+ return serviceRecord;
+ }
+ serviceRecord = new ServiceRecord();
serviceRecord.set(APP_ID_RECORD_KEY, appId);
- serviceRecord.set(HOST_RECORD_KEY, host);
+ serviceRecord.set(HOST_NAME_RECORD_KEY, hostName);
+ serviceRecord.set(HOST_IP_RECORD_KEY, hostIp);
serviceRecord.set(PORT_RECORD_KEY, port);
- serviceRecord.set(OPAQUE_ID_KEY, id);
+ serviceRecord.set(EXTERNAL_ID_KEY, externalId);
+ serviceRecord.set(COMPUTE_GROUP_NAME_KEY, computeName);
+
return serviceRecord;
}
+ @Override
+ public String toString() {
+ return toServiceRecord().attributes().toString();
Review Comment:
I think there is no "before setting the values" scenario given only final fields
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] TEZ-4007: Introduce AmExtensions and Zookeeper-based FrameworkServices [tez]
Posted by "abstractdog (via GitHub)" <gi...@apache.org>.
abstractdog commented on code in PR #427:
URL: https://github.com/apache/tez/pull/427#discussion_r2672453303
##########
tez-api/src/main/java/org/apache/tez/frameworkplugins/FrameworkUtils.java:
##########
@@ -0,0 +1,84 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tez.frameworkplugins;
+
+
+import javax.annotation.Nullable;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.tez.common.ReflectionUtils;
+import org.apache.tez.dag.api.TezConfiguration;
+import org.apache.tez.dag.api.TezConstants;
+import org.apache.tez.dag.api.TezReflectionException;
+
+public final class FrameworkUtils {
+
+ private static final String SERVER_FRAMEWORK_SERVICE_INTERFACE_NAME =
+ "org.apache.tez.frameworkplugins.ServerFrameworkService";
+
+ private FrameworkUtils() {}
+
+ /*
+ Searches for a FrameworkService provider which implements a target interface.
+ The interface should be either ClientFrameworkService or ServerFrameworkService.
+ Depending on which interface is used, either the client or server class of a
+ matching FrameworkMode will be used as the implementation.
+
+ NOTE: Layering of FrameworkServices in a decorator-style is currently not supported
+
+ An implementation is searched in the following order:
+ 1. If conf is not null and the parameter TEZ_FRAMEWORK_MODE is set:
+ the value of TEZ_FRAMEWORK_MODE from the conf will be used
+ 2. If conf is null or the parameter TEZ_FRAMEWORK_MODE is not set
+ and the environment var TEZ_FRAMEWORK_MODE is not empty:
+ the value of the environment var will be used
+ 3. Otherwise: the default class will be instantiated and returned
+ */
+ public static <T extends FrameworkService> T get(Class<T> interfaze, @Nullable Configuration conf,
+ Class<?> defaultClazz) {
+ String modeInConf = conf != null ? conf.get(TezConfiguration.TEZ_FRAMEWORK_MODE) : null;
+ String modeInEnv = System.getenv(TezConstants.TEZ_FRAMEWORK_MODE);
+ try {
+ if (modeInConf != null) {
Review Comment:
ack, much cleaner and takes care of empty string too
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] TEZ-4007: Introduce AmExtensions and Zookeeper-based FrameworkServices [tez]
Posted by "tez-yetus (via GitHub)" <gi...@apache.org>.
tez-yetus commented on PR #427:
URL: https://github.com/apache/tez/pull/427#issuecomment-3729121113
:broken_heart: **-1 overall**
| Vote | Subsystem | Runtime | Logfile | Comment |
|:----:|----------:|--------:|:--------:|:-------:|
| +0 :ok: | reexec | 0m 30s | | Docker mode activated. |
|||| _ Prechecks _ |
| +1 :green_heart: | dupname | 0m 1s | | No case conflicting files found. |
| +0 :ok: | detsecrets | 0m 0s | | detect-secrets was not available. |
| +0 :ok: | xmllint | 0m 0s | | xmllint was not available. |
| +1 :green_heart: | @author | 0m 0s | | The patch does not contain any @author tags. |
| +1 :green_heart: | test4tests | 0m 0s | | The patch appears to include 14 new or modified test files. |
|||| _ master Compile Tests _ |
| +0 :ok: | mvndep | 2m 25s | | Maven dependency ordering for branch |
| +1 :green_heart: | mvninstall | 9m 31s | | master passed |
| +1 :green_heart: | compile | 5m 22s | | master passed |
| +1 :green_heart: | checkstyle | 3m 16s | | master passed |
| +1 :green_heart: | javadoc | 4m 5s | | master passed |
| -1 :x: | spotbugs | 1m 18s | [/branch-spotbugs-tez-api.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/38/artifact/out/branch-spotbugs-tez-api.txt) | tez-api in master failed. |
| -1 :x: | spotbugs | 0m 30s | [/branch-spotbugs-tez-common.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/38/artifact/out/branch-spotbugs-tez-common.txt) | tez-common in master failed. |
| -1 :x: | spotbugs | 0m 38s | [/branch-spotbugs-tez-runtime-library.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/38/artifact/out/branch-spotbugs-tez-runtime-library.txt) | tez-runtime-library in master failed. |
| -1 :x: | spotbugs | 0m 31s | [/branch-spotbugs-tez-examples.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/38/artifact/out/branch-spotbugs-tez-examples.txt) | tez-examples in master failed. |
| -1 :x: | spotbugs | 0m 44s | [/branch-spotbugs-tez-dag.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/38/artifact/out/branch-spotbugs-tez-dag.txt) | tez-dag in master failed. |
| -1 :x: | spotbugs | 0m 23s | [/branch-spotbugs-root.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/38/artifact/out/branch-spotbugs-root.txt) | root in master failed. |
| -0 :warning: | patch | 12m 46s | | Used diff version of patch file. Binary files and potentially other changes not applied. Please rebase and squash commits if necessary. |
|||| _ Patch Compile Tests _ |
| +0 :ok: | mvndep | 0m 16s | | Maven dependency ordering for patch |
| +1 :green_heart: | mvninstall | 6m 24s | | the patch passed |
| +1 :green_heart: | codespell | 0m 50s | | No new issues. |
| +1 :green_heart: | compile | 5m 28s | | the patch passed |
| +1 :green_heart: | javac | 5m 28s | | the patch passed |
| +1 :green_heart: | blanks | 0m 0s | | The patch has no blanks issues. |
| +1 :green_heart: | checkstyle | 0m 27s | | tez-api: The patch generated 0 new + 118 unchanged - 3 fixed = 118 total (was 121) |
| +1 :green_heart: | checkstyle | 0m 23s | | The patch passed checkstyle in tez-common |
| +1 :green_heart: | checkstyle | 0m 28s | | The patch passed checkstyle in tez-runtime-library |
| +1 :green_heart: | checkstyle | 0m 23s | | tez-examples: The patch generated 0 new + 10 unchanged - 1 fixed = 10 total (was 11) |
| -0 :warning: | checkstyle | 0m 34s | [/results-checkstyle-tez-dag.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/38/artifact/out/results-checkstyle-tez-dag.txt) | tez-dag: The patch generated 2 new + 610 unchanged - 2 fixed = 612 total (was 612) |
| -0 :warning: | checkstyle | 0m 46s | [/results-checkstyle-root.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/38/artifact/out/results-checkstyle-root.txt) | root: The patch generated 2 new + 740 unchanged - 6 fixed = 742 total (was 746) |
| +1 :green_heart: | javadoc | 3m 58s | | the patch passed |
| -1 :x: | spotbugs | 0m 37s | [/patch-spotbugs-tez-api.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/38/artifact/out/patch-spotbugs-tez-api.txt) | tez-api in the patch failed. |
| -1 :x: | spotbugs | 0m 30s | [/patch-spotbugs-tez-common.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/38/artifact/out/patch-spotbugs-tez-common.txt) | tez-common in the patch failed. |
| -1 :x: | spotbugs | 0m 38s | [/patch-spotbugs-tez-runtime-library.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/38/artifact/out/patch-spotbugs-tez-runtime-library.txt) | tez-runtime-library in the patch failed. |
| -1 :x: | spotbugs | 0m 31s | [/patch-spotbugs-tez-examples.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/38/artifact/out/patch-spotbugs-tez-examples.txt) | tez-examples in the patch failed. |
| -1 :x: | spotbugs | 0m 43s | [/patch-spotbugs-tez-dag.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/38/artifact/out/patch-spotbugs-tez-dag.txt) | tez-dag in the patch failed. |
| -1 :x: | spotbugs | 0m 24s | [/patch-spotbugs-root.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/38/artifact/out/patch-spotbugs-root.txt) | root in the patch failed. |
|||| _ Other Tests _ |
| +1 :green_heart: | unit | 2m 36s | | tez-api in the patch passed. |
| +1 :green_heart: | unit | 0m 43s | | tez-common in the patch passed. |
| +1 :green_heart: | unit | 6m 9s | | tez-runtime-library in the patch passed. |
| +1 :green_heart: | unit | 0m 29s | | tez-examples in the patch passed. |
| +1 :green_heart: | unit | 5m 54s | | tez-dag in the patch passed. |
| +1 :green_heart: | unit | 71m 24s | | root in the patch passed. |
| +1 :green_heart: | asflicense | 2m 48s | | The patch does not generate ASF License warnings. |
| | | 146m 5s | | |
| Subsystem | Report/Notes |
|----------:|:-------------|
| Docker | ClientAPI=1.52 ServerAPI=1.52 base: https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/38/artifact/out/Dockerfile |
| GITHUB PR | https://github.com/apache/tez/pull/427 |
| Optional Tests | dupname asflicense javac javadoc unit codespell detsecrets xmllint compile spotbugs checkstyle |
| uname | Linux a2c1249863f6 5.15.0-161-generic #171-Ubuntu SMP Sat Oct 11 08:17:01 UTC 2025 x86_64 x86_64 x86_64 GNU/Linux |
| Build tool | maven |
| Personality | /home/jenkins/jenkins-home/workspace/tez-multibranch_PR-427/src/.yetus/personality.sh |
| git revision | master / ad531a86ca6c8005e98413226cf4c6f0cd6884f0 |
| Default Java | Ubuntu-21.0.9+10-Ubuntu-124.04 |
| Test Results | https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/38/testReport/ |
| Max. process+thread count | 1428 (vs. ulimit of 5500) |
| modules | C: tez-api tez-common tez-runtime-library tez-examples tez-dag . U: . |
| Console output | https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/38/console |
| versions | git=2.43.0 maven=3.8.7 codespell=2.0.0 |
| Powered by | Apache Yetus 0.15.1 https://yetus.apache.org |
This message was automatically generated.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] TEZ-4007: Introduce AmExtensions and Zookeeper-based FrameworkServices [tez]
Posted by "abstractdog (via GitHub)" <gi...@apache.org>.
abstractdog commented on PR #427:
URL: https://github.com/apache/tez/pull/427#issuecomment-3738550613
hm, something happened to spotbugs, even the latest version didn't help:
https://github.com/apache/tez/pull/427/commits/0f9a6789796d2e66ed16363e9ed4c30f205a52b5
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] TEZ-4007: Introduce AmExtensions and Zookeeper-based FrameworkServices [tez]
Posted by "abstractdog (via GitHub)" <gi...@apache.org>.
abstractdog merged PR #427:
URL: https://github.com/apache/tez/pull/427
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] TEZ-4007: Introduce AmExtensions and Zookeeper-based FrameworkServices [tez]
Posted by "abstractdog (via GitHub)" <gi...@apache.org>.
abstractdog commented on code in PR #427:
URL: https://github.com/apache/tez/pull/427#discussion_r2681539466
##########
tez-dag/src/main/java/org/apache/tez/dag/app/DAGAppMaster.java:
##########
@@ -513,19 +514,21 @@ protected void serviceInit(final Configuration conf) throws Exception {
jobTokenSecretManager = new JobTokenSecretManager(amConf);
- sessionToken =
- TokenCache.getSessionToken(amCredentials);
+ sessionToken = frameworkService.getAMExtensions().getSessionToken(
+ appAttemptID, jobTokenSecretManager, amCredentials);
+
if (sessionToken == null) {
- throw new RuntimeException("Could not find session token in AM Credentials");
+ sessionToken = TokenCache.getSessionToken(amCredentials);
Review Comment:
right, so the extra effort doesn't make sense, let me change this to:
```
if (sessionToken == null) {
throw new RuntimeException("Could not find session token in AM Credentials");
}
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] TEZ-4007: Introduce AmExtensions and Zookeeper-based FrameworkServices [tez]
Posted by "abstractdog (via GitHub)" <gi...@apache.org>.
abstractdog commented on code in PR #427:
URL: https://github.com/apache/tez/pull/427#discussion_r2549826279
##########
tez-api/src/main/java/org/apache/tez/client/registry/zookeeper/ZkAMRegistryClient.java:
##########
@@ -0,0 +1,205 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.client.registry.zookeeper;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
+
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.imps.CuratorFrameworkState;
+import org.apache.curator.framework.recipes.cache.ChildData;
+import org.apache.curator.framework.recipes.cache.TreeCache;
+import org.apache.curator.framework.recipes.cache.TreeCacheEvent;
+import org.apache.curator.framework.recipes.cache.TreeCacheListener;
+import org.apache.curator.shaded.com.google.common.base.Charsets;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.registry.client.binding.RegistryUtils;
+import org.apache.hadoop.registry.client.types.ServiceRecord;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.tez.client.registry.AMRecord;
+import org.apache.tez.client.registry.AMRegistryClient;
+import org.apache.tez.client.registry.AMRegistryClientListener;
+import org.apache.tez.dag.api.TezConfiguration;
+
+import com.fasterxml.jackson.core.JsonParseException;
+import com.google.common.base.Preconditions;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Curator/Zookeeper implementation of {@link AMRegistryClient}.
+ */
+@InterfaceAudience.Public
+public final class ZkAMRegistryClient extends AMRegistryClient {
+ private static final Logger LOG = LoggerFactory.getLogger(ZkAMRegistryClient.class);
+ private static final Map<String, ZkAMRegistryClient> INSTANCES = new HashMap<>();
+
+ private final Configuration conf;
+ //Cache of known AMs
+ private final ConcurrentHashMap<String, AMRecord> amRecordCache = new ConcurrentHashMap<>();
+ private CuratorFramework client;
+
+ private ZkAMRegistryClient(final Configuration conf) {
+ this.conf = conf;
+ }
+
+ public static synchronized ZkAMRegistryClient getClient(final Configuration conf) {
+ String namespace = conf.get(TezConfiguration.TEZ_AM_REGISTRY_NAMESPACE);
+ ZkAMRegistryClient registry = INSTANCES.get(namespace);
+ if (registry == null) {
+ registry = new ZkAMRegistryClient(conf);
+ INSTANCES.put(namespace, registry);
+ }
+ LOG.info("Returning tez AM registry ({}) for namespace '{}'", System.identityHashCode(registry), namespace);
+ return registry;
+ }
+
+ /**
+ * Deserializes a {@link ServiceRecord} from ZooKeeper data and converts it into an {@link AMRecord}
+ * for caching.
+ *
+ * @param childData the ZooKeeper node data containing a serialized {@link ServiceRecord}
+ * @return an {@link AMRecord} constructed from the deserialized {@link ServiceRecord}, or {@code null}
+ * if no data is present
+ * @throws IOException if the data cannot be deserialized into a {@link ServiceRecord}
+ */
+ public static AMRecord getAMRecord(final ChildData childData) throws IOException {
+ // not a leaf path. Only leaf path contains AMRecord
+ if (!childData.getPath().contains(ApplicationId.appIdStrPrefix)) {
+ return null;
+ }
+ byte[] data = childData.getData();
+ // only the path appeared, there is no data yet
+ if (data.length == 0) {
+ return null;
+ }
+ String value = new String(data, Charsets.UTF_8);
+ try {
+ RegistryUtils.ServiceRecordMarshal marshal = new RegistryUtils.ServiceRecordMarshal();
+ ServiceRecord serviceRecord = marshal.fromJson(value);
+ return new AMRecord(serviceRecord);
+ } catch (JsonParseException e) {
+ //Not a json AMRecord (SRV), could be some other data.
+ LOG.warn("Non-json data received while de-serializing AMRecord: {}. Ignoring...", value);
+ return null;
+ }
+ }
+
+ public void start() throws Exception {
+ ZkConfig zkConf = new ZkConfig(this.conf);
+ client = zkConf.createCuratorFramework();
+ final TreeCache cache = new TreeCache(client, zkConf.getZkNamespace());
+ client.start();
+ cache.start();
+ cache.getListenable().addListener(new ZkRegistryListener());
+ }
+
+ @Override
+ public AMRecord getRecord(String appId) {
+ if (amRecordCache.get(appId) == null) {
+ return null;
+ }
+ //Return a copy
+ return new AMRecord(amRecordCache.get(appId));
+ }
+
+ @Override
+ public List<AMRecord> getAllRecords() {
+ return amRecordCache.values().stream().map(AMRecord::new).collect(Collectors.toList());
+ }
+
+ @Override
+ public synchronized void addListener(AMRegistryClientListener listener) {
+ getListeners().add(listener);
+ }
+
+ @Override
+ public void close() {
+ client.close();
+ }
+
+ /**
+ * Callback listener for ZooKeeper events that updates the local cache
+ * when child nodes under the monitored path change.
+ */
+ private class ZkRegistryListener implements TreeCacheListener {
+
+ @Override
+ public void childEvent(final CuratorFramework clientParam, final TreeCacheEvent event) throws Exception {
+ Preconditions.checkArgument(clientParam != null && clientParam.getState() == CuratorFrameworkState.STARTED,
+ "Curator client is not started");
+
+ ChildData childData = event.getData();
+ switch (event.getType()) {
+ case NODE_ADDED:
+ if (isEmpty(childData)) {
+ LOG.info("AppId allocated: {}", childData.getPath());
+ } else {
+ AMRecord amRecord = getAMRecord(childData);
+ if (amRecord != null) {
+ LOG.info("AM registered with data: {}. Notifying {} listeners.", amRecord, getListeners().size());
+ amRecordCache.put(amRecord.getApplicationId().toString(), amRecord);
+ notifyOnAdded(amRecord);
+ }
+ }
+ break;
+ case NODE_UPDATED:
+ if (isEmpty(childData)) {
+ throw new RuntimeException("AM updated with empty data");
+ } else {
+ AMRecord amRecord = getAMRecord(childData);
+ if (amRecord != null) {
+ LOG.info("AM updated data: {}. Notifying {} listeners.", amRecord, getListeners().size());
+ amRecordCache.put(amRecord.getApplicationId().toString(), amRecord);
+ notifyOnAdded(amRecord);
+ }
+ }
+ break;
+ case NODE_REMOVED:
+ if (isEmpty(childData)) {
+ LOG.info("Unused AppId unregistered: {}", childData.getPath());
+ } else {
+ AMRecord amRecord = getAMRecord(childData);
+ if (amRecord != null) {
+ LOG.info("AM removed: {}. Notifying {} listeners.", amRecord, getListeners().size());
+ amRecordCache.remove(amRecord.getApplicationId().toString(), amRecord);
Review Comment:
refactored amRecordCache collection key to ApplicationId according to other comments
I'm adding a unit test to double check correct AMRecord equals and hashCode behavior: if they both work properly, no matter the actual AMRecord instance, event removal should work properly
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] TEZ-4007: Introduce AmExtensions and Zookeeper-based FrameworkServices [tez]
Posted by "abstractdog (via GitHub)" <gi...@apache.org>.
abstractdog commented on code in PR #427:
URL: https://github.com/apache/tez/pull/427#discussion_r2543404391
##########
tez-api/src/main/java/org/apache/tez/client/registry/zookeeper/ZkFrameworkClient.java:
##########
@@ -0,0 +1,132 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.client.registry.zookeeper;
+
+import java.io.IOException;
+
+import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.client.api.YarnClientApplication;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.util.Records;
+import org.apache.tez.client.FrameworkClient;
+import org.apache.tez.client.registry.AMRecord;
+import org.apache.tez.dag.api.TezConfiguration;
+
+public class ZkFrameworkClient extends FrameworkClient {
+
+ private AMRecord amRecord;
+ private ZkAMRegistryClient amRegistryClient = null;
+ private volatile boolean isRunning = false;
+ private String amHost;
+ private int amPort;
+
+ @Override
+ public synchronized void init(TezConfiguration tezConf) {
+ if (this.amRegistryClient == null) {
+ try {
+ this.amRegistryClient = ZkAMRegistryClient.getClient(tezConf);
+ this.isRunning = true;
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ @Override
+ public void start() {
+ try {
+ this.amRegistryClient.start();
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ isRunning = true;
+ }
+
+ @Override
+ public void stop() {
+ isRunning = false;
+ amRegistryClient.close();
+ }
+
+ @Override
+ public void close() throws IOException {
+ amRegistryClient.close();
+ }
+
+ @Override
+ public YarnClientApplication createApplication() throws YarnException, IOException {
+ ApplicationSubmissionContext context = Records.newRecord(ApplicationSubmissionContext.class);
+ ApplicationId appId = amRecord.getApplicationId();
Review Comment:
right, confusing, added javadoc comment for clarity:
```
* <p>Hidden assumption here: this method assumes that
* {@code getApplicationReport()} has already been called before
* {@code createApplication()}, ensuring that {@code amRecord.getApplicationId()}
* is always available. This assumption holds in all supported usage patterns:
* the only code path where {@code createApplication()} might be called first is
* {@code TezClient.submitDAGApplication()}, but that path is never exercised in
* Zookeeper standalone mode because that mode assumes applications are already
* running. Therefore, the ordering guarantee is valid in practice.</p>
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] TEZ-4007: Introduce AmExtensions and Zookeeper-based FrameworkServices [tez]
Posted by "abstractdog (via GitHub)" <gi...@apache.org>.
abstractdog commented on code in PR #427:
URL: https://github.com/apache/tez/pull/427#discussion_r2549826279
##########
tez-api/src/main/java/org/apache/tez/client/registry/zookeeper/ZkAMRegistryClient.java:
##########
@@ -0,0 +1,205 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.client.registry.zookeeper;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
+
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.imps.CuratorFrameworkState;
+import org.apache.curator.framework.recipes.cache.ChildData;
+import org.apache.curator.framework.recipes.cache.TreeCache;
+import org.apache.curator.framework.recipes.cache.TreeCacheEvent;
+import org.apache.curator.framework.recipes.cache.TreeCacheListener;
+import org.apache.curator.shaded.com.google.common.base.Charsets;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.registry.client.binding.RegistryUtils;
+import org.apache.hadoop.registry.client.types.ServiceRecord;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.tez.client.registry.AMRecord;
+import org.apache.tez.client.registry.AMRegistryClient;
+import org.apache.tez.client.registry.AMRegistryClientListener;
+import org.apache.tez.dag.api.TezConfiguration;
+
+import com.fasterxml.jackson.core.JsonParseException;
+import com.google.common.base.Preconditions;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Curator/Zookeeper implementation of {@link AMRegistryClient}.
+ */
+@InterfaceAudience.Public
+public final class ZkAMRegistryClient extends AMRegistryClient {
+ private static final Logger LOG = LoggerFactory.getLogger(ZkAMRegistryClient.class);
+ private static final Map<String, ZkAMRegistryClient> INSTANCES = new HashMap<>();
+
+ private final Configuration conf;
+ //Cache of known AMs
+ private final ConcurrentHashMap<String, AMRecord> amRecordCache = new ConcurrentHashMap<>();
+ private CuratorFramework client;
+
+ private ZkAMRegistryClient(final Configuration conf) {
+ this.conf = conf;
+ }
+
+ public static synchronized ZkAMRegistryClient getClient(final Configuration conf) {
+ String namespace = conf.get(TezConfiguration.TEZ_AM_REGISTRY_NAMESPACE);
+ ZkAMRegistryClient registry = INSTANCES.get(namespace);
+ if (registry == null) {
+ registry = new ZkAMRegistryClient(conf);
+ INSTANCES.put(namespace, registry);
+ }
+ LOG.info("Returning tez AM registry ({}) for namespace '{}'", System.identityHashCode(registry), namespace);
+ return registry;
+ }
+
+ /**
+ * Deserializes a {@link ServiceRecord} from ZooKeeper data and converts it into an {@link AMRecord}
+ * for caching.
+ *
+ * @param childData the ZooKeeper node data containing a serialized {@link ServiceRecord}
+ * @return an {@link AMRecord} constructed from the deserialized {@link ServiceRecord}, or {@code null}
+ * if no data is present
+ * @throws IOException if the data cannot be deserialized into a {@link ServiceRecord}
+ */
+ public static AMRecord getAMRecord(final ChildData childData) throws IOException {
+ // not a leaf path. Only leaf path contains AMRecord
+ if (!childData.getPath().contains(ApplicationId.appIdStrPrefix)) {
+ return null;
+ }
+ byte[] data = childData.getData();
+ // only the path appeared, there is no data yet
+ if (data.length == 0) {
+ return null;
+ }
+ String value = new String(data, Charsets.UTF_8);
+ try {
+ RegistryUtils.ServiceRecordMarshal marshal = new RegistryUtils.ServiceRecordMarshal();
+ ServiceRecord serviceRecord = marshal.fromJson(value);
+ return new AMRecord(serviceRecord);
+ } catch (JsonParseException e) {
+ //Not a json AMRecord (SRV), could be some other data.
+ LOG.warn("Non-json data received while de-serializing AMRecord: {}. Ignoring...", value);
+ return null;
+ }
+ }
+
+ public void start() throws Exception {
+ ZkConfig zkConf = new ZkConfig(this.conf);
+ client = zkConf.createCuratorFramework();
+ final TreeCache cache = new TreeCache(client, zkConf.getZkNamespace());
+ client.start();
+ cache.start();
+ cache.getListenable().addListener(new ZkRegistryListener());
+ }
+
+ @Override
+ public AMRecord getRecord(String appId) {
+ if (amRecordCache.get(appId) == null) {
+ return null;
+ }
+ //Return a copy
+ return new AMRecord(amRecordCache.get(appId));
+ }
+
+ @Override
+ public List<AMRecord> getAllRecords() {
+ return amRecordCache.values().stream().map(AMRecord::new).collect(Collectors.toList());
+ }
+
+ @Override
+ public synchronized void addListener(AMRegistryClientListener listener) {
+ getListeners().add(listener);
+ }
+
+ @Override
+ public void close() {
+ client.close();
+ }
+
+ /**
+ * Callback listener for ZooKeeper events that updates the local cache
+ * when child nodes under the monitored path change.
+ */
+ private class ZkRegistryListener implements TreeCacheListener {
+
+ @Override
+ public void childEvent(final CuratorFramework clientParam, final TreeCacheEvent event) throws Exception {
+ Preconditions.checkArgument(clientParam != null && clientParam.getState() == CuratorFrameworkState.STARTED,
+ "Curator client is not started");
+
+ ChildData childData = event.getData();
+ switch (event.getType()) {
+ case NODE_ADDED:
+ if (isEmpty(childData)) {
+ LOG.info("AppId allocated: {}", childData.getPath());
+ } else {
+ AMRecord amRecord = getAMRecord(childData);
+ if (amRecord != null) {
+ LOG.info("AM registered with data: {}. Notifying {} listeners.", amRecord, getListeners().size());
+ amRecordCache.put(amRecord.getApplicationId().toString(), amRecord);
+ notifyOnAdded(amRecord);
+ }
+ }
+ break;
+ case NODE_UPDATED:
+ if (isEmpty(childData)) {
+ throw new RuntimeException("AM updated with empty data");
+ } else {
+ AMRecord amRecord = getAMRecord(childData);
+ if (amRecord != null) {
+ LOG.info("AM updated data: {}. Notifying {} listeners.", amRecord, getListeners().size());
+ amRecordCache.put(amRecord.getApplicationId().toString(), amRecord);
+ notifyOnAdded(amRecord);
+ }
+ }
+ break;
+ case NODE_REMOVED:
+ if (isEmpty(childData)) {
+ LOG.info("Unused AppId unregistered: {}", childData.getPath());
+ } else {
+ AMRecord amRecord = getAMRecord(childData);
+ if (amRecord != null) {
+ LOG.info("AM removed: {}. Notifying {} listeners.", amRecord, getListeners().size());
+ amRecordCache.remove(amRecord.getApplicationId().toString(), amRecord);
Review Comment:
refactored amRecordCache collection key to ApplicationId according to other comments
I'm adding a unit test to double check correct AMRecord equals and hashCode behavior: if they both work properly, no matter the actual AMRecord instance, event removal should work properly
UPDATE: this will be TestAMRecord.testRemoveFromCacheByDeserializedRecordAppId
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] TEZ-4007: Introduce AmExtensions and Zookeeper-based FrameworkServices [tez]
Posted by "abstractdog (via GitHub)" <gi...@apache.org>.
abstractdog commented on code in PR #427:
URL: https://github.com/apache/tez/pull/427#discussion_r2567589882
##########
tez-dag/src/main/java/org/apache/tez/frameworkplugins/zookeeper/ZkStandaloneAMExtensions.java:
##########
@@ -0,0 +1,90 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tez.frameworkplugins.zookeeper;
+
+import java.io.IOException;
+import java.util.Map;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.tez.client.registry.AMRegistry;
+import org.apache.tez.common.TezUtilsInternal;
+import org.apache.tez.common.security.JobTokenIdentifier;
+import org.apache.tez.common.security.JobTokenSecretManager;
+import org.apache.tez.common.security.TokenCache;
+import org.apache.tez.dag.api.records.DAGProtos;
+import org.apache.tez.dag.app.ClusterInfo;
+import org.apache.tez.dag.app.dag.Vertex;
+import org.apache.tez.frameworkplugins.AMExtensions;
+import org.apache.tez.frameworkplugins.ServerFrameworkService;
+
+public class ZkStandaloneAMExtensions implements AMExtensions {
+
+ private final ServerFrameworkService frameworkService;
+
+ public ZkStandaloneAMExtensions(ServerFrameworkService frameworkService) {
+ this.frameworkService = frameworkService;
+ }
+
+ @Override
+ public ContainerId allocateContainerId(Configuration conf) {
+ try {
+ AMRegistry amRegistry = frameworkService.getAMRegistry(conf);
+ if (amRegistry != null) {
+ ApplicationId appId = amRegistry.generateNewId();
+ // attemptId is set to 1 only then APP_LAUNCHED event gets triggered
+ ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.newInstance(appId, 1);
+ return ContainerId.newContainerId(applicationAttemptId, 0);
+ } else {
+ throw new RuntimeException("AMRegistry is required for ZkStandaloneAmExtensions");
+ }
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void checkTaskResources(Map<String, Vertex> vertices, ClusterInfo clusterInfo) {
+ // no-op: Zookeeper-based framework current enforce task resources at the moment
+ }
+
+ @Override
+ public DAGProtos.ConfigurationProto loadConfigurationProto() throws IOException {
+ return TezUtilsInternal.loadConfProtoFromText();
+ }
+
+ @Override
+ public Token<JobTokenIdentifier> getSessionToken(ApplicationAttemptId appAttemptID,
+ JobTokenSecretManager jobTokenSecretManager, Credentials amCredentials) {
+ JobTokenIdentifier identifier = new JobTokenIdentifier(new Text(appAttemptID.getApplicationId().toString()));
Review Comment:
the standalone mode doesn't account for different application attempt ids
while generating the app id, we rely on the zookeeper logic implemented in ZkAmRegistry, then deriving an artificial container ID and app attempt ID in this class (ZkStandaloneAMExtensions.allocateContainerId), see constant app attempt id:
```
ApplicationAttemptId applicationAttemptId = ApplicationAttemptId.newInstance(appId, 1);
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] [DRAFT] TEZ-4007: Zookeeper based FrameworkServices and AmExtensions (3/3) + refactor [tez]
Posted by "tez-yetus (via GitHub)" <gi...@apache.org>.
tez-yetus commented on PR #427:
URL: https://github.com/apache/tez/pull/427#issuecomment-3421004169
:broken_heart: **-1 overall**
| Vote | Subsystem | Runtime | Logfile | Comment |
|:----:|----------:|--------:|:--------:|:-------:|
| +0 :ok: | reexec | 0m 0s | | Docker mode activated. |
| -1 :x: | docker | 0m 2s | | Docker failed to build run-specific yetus/tez:tp-13498}. |
| Subsystem | Report/Notes |
|----------:|:-------------|
| GITHUB PR | https://github.com/apache/tez/pull/427 |
| Console output | https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/13/console |
| versions | git=2.34.1 |
| Powered by | Apache Yetus 0.15.1 https://yetus.apache.org |
This message was automatically generated.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] [DRAFT] TEZ-4007: Zookeeper based FrameworkServices and AmExtensions (3/3) - initial patch [tez]
Posted by "tez-yetus (via GitHub)" <gi...@apache.org>.
tez-yetus commented on PR #427:
URL: https://github.com/apache/tez/pull/427#issuecomment-3400937904
:broken_heart: **-1 overall**
| Vote | Subsystem | Runtime | Logfile | Comment |
|:----:|----------:|--------:|:--------:|:-------:|
| +0 :ok: | reexec | 0m 16s | | Docker mode activated. |
|||| _ Prechecks _ |
| +1 :green_heart: | dupname | 0m 1s | | No case conflicting files found. |
| +0 :ok: | detsecrets | 0m 0s | | detect-secrets was not available. |
| +0 :ok: | xmllint | 0m 0s | | xmllint was not available. |
| +1 :green_heart: | @author | 0m 0s | | The patch does not contain any @author tags. |
| +1 :green_heart: | test4tests | 0m 0s | | The patch appears to include 10 new or modified test files. |
|||| _ master Compile Tests _ |
| +0 :ok: | mvndep | 2m 43s | | Maven dependency ordering for branch |
| +1 :green_heart: | mvninstall | 10m 2s | | master passed |
| +1 :green_heart: | compile | 6m 19s | | master passed |
| +1 :green_heart: | checkstyle | 3m 44s | | master passed |
| +1 :green_heart: | javadoc | 4m 34s | | master passed |
| +0 :ok: | spotbugs | 2m 10s | | tez-api in master has 610 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 0m 23s | | branch/tez-build-tools no spotbugs output file (spotbugsXml.xml) |
| +0 :ok: | spotbugs | 0m 44s | | tez-common in master has 13 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 1m 14s | | tez-runtime-library in master has 241 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 0m 55s | | tez-mapreduce in master has 129 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 0m 42s | | tez-examples in master has 2 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 1m 48s | | tez-dag in master has 785 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 7m 31s | | root in master has 2066 extant spotbugs warnings. |
|||| _ Patch Compile Tests _ |
| +0 :ok: | mvndep | 0m 15s | | Maven dependency ordering for patch |
| -1 :x: | mvninstall | 0m 11s | [/patch-mvninstall-tez-api.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/5/artifact/out/patch-mvninstall-tez-api.txt) | tez-api in the patch failed. |
| -1 :x: | mvninstall | 0m 16s | [/patch-mvninstall-tez-examples.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/5/artifact/out/patch-mvninstall-tez-examples.txt) | tez-examples in the patch failed. |
| -1 :x: | mvninstall | 0m 13s | [/patch-mvninstall-tez-dag.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/5/artifact/out/patch-mvninstall-tez-dag.txt) | tez-dag in the patch failed. |
| -1 :x: | mvninstall | 1m 23s | [/patch-mvninstall-root.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/5/artifact/out/patch-mvninstall-root.txt) | root in the patch failed. |
| +1 :green_heart: | codespell | 0m 53s | | No new issues. |
| -1 :x: | compile | 0m 21s | [/patch-compile-tez-api.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/5/artifact/out/patch-compile-tez-api.txt) | tez-api in the patch failed. |
| -1 :x: | compile | 0m 26s | [/patch-compile-tez-examples.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/5/artifact/out/patch-compile-tez-examples.txt) | tez-examples in the patch failed. |
| -1 :x: | compile | 0m 23s | [/patch-compile-tez-dag.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/5/artifact/out/patch-compile-tez-dag.txt) | tez-dag in the patch failed. |
| -1 :x: | compile | 0m 26s | [/patch-compile-root.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/5/artifact/out/patch-compile-root.txt) | root in the patch failed. |
| -1 :x: | javac | 0m 21s | [/patch-compile-tez-api.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/5/artifact/out/patch-compile-tez-api.txt) | tez-api in the patch failed. |
| -1 :x: | javac | 0m 26s | [/patch-compile-tez-examples.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/5/artifact/out/patch-compile-tez-examples.txt) | tez-examples in the patch failed. |
| -1 :x: | javac | 0m 23s | [/patch-compile-tez-dag.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/5/artifact/out/patch-compile-tez-dag.txt) | tez-dag in the patch failed. |
| -1 :x: | javac | 0m 26s | [/patch-compile-root.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/5/artifact/out/patch-compile-root.txt) | root in the patch failed. |
| +1 :green_heart: | blanks | 0m 0s | | The patch has no blanks issues. |
| -0 :warning: | checkstyle | 0m 22s | [/results-checkstyle-tez-api.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/5/artifact/out/results-checkstyle-tez-api.txt) | tez-api: The patch generated 24 new + 112 unchanged - 11 fixed = 136 total (was 123) |
| +1 :green_heart: | checkstyle | 0m 17s | | The patch passed checkstyle in tez-build-tools |
| +1 :green_heart: | checkstyle | 0m 19s | | tez-common: The patch generated 0 new + 2 unchanged - 51 fixed = 2 total (was 53) |
| +1 :green_heart: | checkstyle | 0m 22s | | The patch passed checkstyle in tez-runtime-library |
| +1 :green_heart: | checkstyle | 0m 20s | | The patch passed checkstyle in tez-mapreduce |
| -0 :warning: | checkstyle | 0m 18s | [/results-checkstyle-tez-examples.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/5/artifact/out/results-checkstyle-tez-examples.txt) | tez-examples: The patch generated 6 new + 0 unchanged - 0 fixed = 6 total (was 0) |
| -0 :warning: | checkstyle | 0m 31s | [/results-checkstyle-tez-dag.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/5/artifact/out/results-checkstyle-tez-dag.txt) | tez-dag: The patch generated 244 new + 576 unchanged - 286 fixed = 820 total (was 862) |
| -0 :warning: | checkstyle | 0m 43s | [/results-checkstyle-root.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/5/artifact/out/results-checkstyle-root.txt) | root: The patch generated 270 new + 694 unchanged - 344 fixed = 964 total (was 1038) |
| -1 :x: | javadoc | 0m 22s | [/patch-javadoc-tez-api.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/5/artifact/out/patch-javadoc-tez-api.txt) | tez-api in the patch failed. |
| -1 :x: | javadoc | 0m 22s | [/patch-javadoc-tez-dag.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/5/artifact/out/patch-javadoc-tez-dag.txt) | tez-dag in the patch failed. |
| -1 :x: | javadoc | 0m 21s | [/patch-javadoc-root.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/5/artifact/out/patch-javadoc-root.txt) | root in the patch failed. |
| -1 :x: | spotbugs | 0m 21s | [/patch-spotbugs-tez-api.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/5/artifact/out/patch-spotbugs-tez-api.txt) | tez-api in the patch failed. |
| +0 :ok: | spotbugs | 0m 22s | | tez-build-tools has no data from spotbugs |
| -1 :x: | spotbugs | 0m 28s | [/patch-spotbugs-tez-examples.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/5/artifact/out/patch-spotbugs-tez-examples.txt) | tez-examples in the patch failed. |
| -1 :x: | spotbugs | 0m 22s | [/patch-spotbugs-tez-dag.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/5/artifact/out/patch-spotbugs-tez-dag.txt) | tez-dag in the patch failed. |
| -1 :x: | spotbugs | 0m 35s | [/patch-spotbugs-root.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/5/artifact/out/patch-spotbugs-root.txt) | root in the patch failed. |
|||| _ Other Tests _ |
| -1 :x: | unit | 0m 21s | [/patch-unit-tez-api.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/5/artifact/out/patch-unit-tez-api.txt) | tez-api in the patch failed. |
| +1 :green_heart: | unit | 0m 20s | | tez-build-tools in the patch passed. |
| +1 :green_heart: | unit | 0m 43s | | tez-common in the patch passed. |
| +1 :green_heart: | unit | 6m 2s | | tez-runtime-library in the patch passed. |
| +1 :green_heart: | unit | 1m 33s | | tez-mapreduce in the patch passed. |
| -1 :x: | unit | 0m 28s | [/patch-unit-tez-examples.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/5/artifact/out/patch-unit-tez-examples.txt) | tez-examples in the patch failed. |
| -1 :x: | unit | 0m 22s | [/patch-unit-tez-dag.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/5/artifact/out/patch-unit-tez-dag.txt) | tez-dag in the patch failed. |
| -1 :x: | unit | 1m 58s | [/patch-unit-root.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/5/artifact/out/patch-unit-root.txt) | root in the patch failed. |
| -1 :x: | asflicense | 2m 28s | [/results-asflicense.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/5/artifact/out/results-asflicense.txt) | The patch generated 2 ASF License warnings. |
| | | 81m 50s | | |
| Subsystem | Report/Notes |
|----------:|:-------------|
| Docker | ClientAPI=1.51 ServerAPI=1.51 base: https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/5/artifact/out/Dockerfile |
| GITHUB PR | https://github.com/apache/tez/pull/427 |
| Optional Tests | dupname asflicense javac javadoc unit codespell detsecrets xmllint compile spotbugs checkstyle |
| uname | Linux 809091b40aa4 5.15.0-152-generic #162-Ubuntu SMP Wed Jul 23 09:48:42 UTC 2025 x86_64 x86_64 x86_64 GNU/Linux |
| Build tool | maven |
| Personality | /home/jenkins/jenkins-home/workspace/tez-multibranch_PR-427/src/.yetus/personality.sh |
| git revision | master / d3c969aa5b87ae00998e91e147b911a1fbe21ac5 |
| Default Java | Ubuntu-21.0.8+9-Ubuntu-0ubuntu124.04.1 |
| Test Results | https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/5/testReport/ |
| Max. process+thread count | 1109 (vs. ulimit of 5500) |
| modules | C: tez-api tez-build-tools tez-common tez-runtime-library tez-mapreduce tez-examples tez-dag . U: . |
| Console output | https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/5/console |
| versions | git=2.43.0 maven=3.8.7 spotbugs=4.9.3 codespell=2.0.0 |
| Powered by | Apache Yetus 0.15.1 https://yetus.apache.org |
This message was automatically generated.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] [DRAFT] TEZ-4007: Zookeeper based FrameworkServices and AmExtensions (3/3) + refactor [tez]
Posted by "tez-yetus (via GitHub)" <gi...@apache.org>.
tez-yetus commented on PR #427:
URL: https://github.com/apache/tez/pull/427#issuecomment-3415950043
:broken_heart: **-1 overall**
| Vote | Subsystem | Runtime | Logfile | Comment |
|:----:|----------:|--------:|:--------:|:-------:|
| +0 :ok: | reexec | 0m 16s | | Docker mode activated. |
|||| _ Prechecks _ |
| +1 :green_heart: | dupname | 0m 0s | | No case conflicting files found. |
| +0 :ok: | detsecrets | 0m 0s | | detect-secrets was not available. |
| +0 :ok: | xmllint | 0m 0s | | xmllint was not available. |
| +1 :green_heart: | @author | 0m 1s | | The patch does not contain any @author tags. |
| +1 :green_heart: | test4tests | 0m 0s | | The patch appears to include 10 new or modified test files. |
|||| _ master Compile Tests _ |
| +0 :ok: | mvndep | 2m 43s | | Maven dependency ordering for branch |
| +1 :green_heart: | mvninstall | 10m 26s | | master passed |
| +1 :green_heart: | compile | 5m 59s | | master passed |
| +1 :green_heart: | checkstyle | 3m 29s | | master passed |
| +1 :green_heart: | javadoc | 4m 16s | | master passed |
| +0 :ok: | spotbugs | 2m 9s | | tez-api in master has 610 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 0m 45s | | tez-common in master has 13 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 1m 12s | | tez-runtime-library in master has 241 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 0m 57s | | tez-mapreduce in master has 129 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 0m 43s | | tez-examples in master has 2 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 1m 43s | | tez-dag in master has 785 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 6m 57s | | root in master has 2066 extant spotbugs warnings. |
|||| _ Patch Compile Tests _ |
| +0 :ok: | mvndep | 0m 18s | | Maven dependency ordering for patch |
| +1 :green_heart: | mvninstall | 6m 42s | | the patch passed |
| +1 :green_heart: | codespell | 0m 50s | | No new issues. |
| +1 :green_heart: | compile | 5m 57s | | the patch passed |
| +1 :green_heart: | javac | 5m 57s | | the patch passed |
| +1 :green_heart: | blanks | 0m 0s | | The patch has no blanks issues. |
| -0 :warning: | checkstyle | 0m 26s | [/results-checkstyle-tez-api.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/12/artifact/out/results-checkstyle-tez-api.txt) | tez-api: The patch generated 3 new + 134 unchanged - 11 fixed = 137 total (was 145) |
| -0 :warning: | checkstyle | 0m 22s | [/results-checkstyle-tez-examples.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/12/artifact/out/results-checkstyle-tez-examples.txt) | tez-examples: The patch generated 6 new + 0 unchanged - 0 fixed = 6 total (was 0) |
| -0 :warning: | checkstyle | 0m 35s | [/results-checkstyle-tez-dag.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/12/artifact/out/results-checkstyle-tez-dag.txt) | tez-dag: The patch generated 2 new + 818 unchanged - 2 fixed = 820 total (was 820) |
| -0 :warning: | checkstyle | 0m 46s | [/results-checkstyle-root.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/12/artifact/out/results-checkstyle-root.txt) | root: The patch generated 11 new + 954 unchanged - 13 fixed = 965 total (was 967) |
| -1 :x: | javadoc | 0m 41s | [/results-javadoc-javadoc-tez-api.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/12/artifact/out/results-javadoc-javadoc-tez-api.txt) | tez-api generated 1 new + 10 unchanged - 0 fixed = 11 total (was 10) |
| -1 :x: | javadoc | 1m 21s | [/results-javadoc-javadoc-root.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/12/artifact/out/results-javadoc-javadoc-root.txt) | root generated 1 new + 19 unchanged - 0 fixed = 20 total (was 19) |
| -1 :x: | spotbugs | 1m 42s | [/new-spotbugs-tez-api.html](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/12/artifact/out/new-spotbugs-tez-api.html) | tez-api generated 1 new + 610 unchanged - 0 fixed = 611 total (was 610) |
| -1 :x: | spotbugs | 7m 20s | [/new-spotbugs-root.html](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/12/artifact/out/new-spotbugs-root.html) | root generated 1 new + 2066 unchanged - 0 fixed = 2067 total (was 2066) |
|||| _ Other Tests _ |
| +1 :green_heart: | unit | 0m 53s | | tez-api in the patch passed. |
| +1 :green_heart: | unit | 0m 48s | | tez-common in the patch passed. |
| +1 :green_heart: | unit | 6m 8s | | tez-runtime-library in the patch passed. |
| +1 :green_heart: | unit | 1m 36s | | tez-mapreduce in the patch passed. |
| +1 :green_heart: | unit | 0m 32s | | tez-examples in the patch passed. |
| +1 :green_heart: | unit | 0m 47s | | tez-dag in the patch passed. |
| -1 :x: | unit | 34m 17s | [/patch-unit-root.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/12/artifact/out/patch-unit-root.txt) | root in the patch passed. |
| -1 :x: | asflicense | 2m 57s | [/results-asflicense.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/12/artifact/out/results-asflicense.txt) | The patch generated 2 ASF License warnings. |
| | | 131m 49s | | |
| Reason | Tests |
|-------:|:------|
| SpotBugs | module:tez-api |
| | new org.apache.tez.client.TezYarnClient(YarnClient) may expose internal representation by storing an externally mutable object into TezYarnClient.yarnClient At TezYarnClient.java:representation by storing an externally mutable object into TezYarnClient.yarnClient At TezYarnClient.java:[line 47] |
| SpotBugs | module:root |
| | new org.apache.tez.client.TezYarnClient(YarnClient) may expose internal representation by storing an externally mutable object into TezYarnClient.yarnClient At TezYarnClient.java:representation by storing an externally mutable object into TezYarnClient.yarnClient At TezYarnClient.java:[line 47] |
| Failed junit tests | tez.dag.history.ats.acls.TestATSHistoryWithACLs |
| | tez.dag.history.ats.acls.TestATSHistoryV15 |
| | tez.auxservices.TestShuffleHandlerJobs |
| | tez.dag.history.logging.ats.TestATSHistoryWithMiniCluster |
| | tez.history.TestHistoryParser |
| | tez.mapreduce.TestMRRJobsDAGApi |
| | tez.test.TestAMRecovery |
| | tez.test.TestPipelinedShuffle |
| | tez.test.TestTaskErrorsUsingLocalMode |
| | tez.test.TestAMRecoveryAggregationBroadcast |
| | tez.test.TestDAGRecovery2 |
| | tez.test.TestRecovery |
| | tez.test.TestLocalMode |
| | tez.test.TestDAGRecovery |
| | tez.test.TestSecureShuffle |
| | tez.test.TestTezJobs |
| | tez.test.TestExceptionPropagation |
| | tez.test.TestFaultTolerance |
| | tez.mapreduce.TestMRRJobs |
| | tez.test.TestAM |
| | tez.analyzer.TestAnalyzer |
| | tez.tests.TestExtServicesWithLocalMode |
| | tez.tests.TestExternalTezServices |
| | tez.tests.TestExternalTezServicesErrors |
| Subsystem | Report/Notes |
|----------:|:-------------|
| Docker | ClientAPI=1.51 ServerAPI=1.51 base: https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/12/artifact/out/Dockerfile |
| GITHUB PR | https://github.com/apache/tez/pull/427 |
| Optional Tests | dupname asflicense javac javadoc unit codespell detsecrets xmllint compile spotbugs checkstyle |
| uname | Linux d6a3dd7bfd63 5.15.0-152-generic #162-Ubuntu SMP Wed Jul 23 09:48:42 UTC 2025 x86_64 x86_64 x86_64 GNU/Linux |
| Build tool | maven |
| Personality | /home/jenkins/jenkins-home/workspace/tez-multibranch_PR-427/src/.yetus/personality.sh |
| git revision | master / db11fb6c047e1b128e50b7c922a989c22bc84427 |
| Default Java | Ubuntu-21.0.8+9-Ubuntu-0ubuntu124.04.1 |
| Test Results | https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/12/testReport/ |
| Max. process+thread count | 1109 (vs. ulimit of 5500) |
| modules | C: tez-api tez-common tez-runtime-library tez-mapreduce tez-examples tez-dag . U: . |
| Console output | https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/12/console |
| versions | git=2.43.0 maven=3.8.7 spotbugs=4.9.3 codespell=2.0.0 |
| Powered by | Apache Yetus 0.15.1 https://yetus.apache.org |
This message was automatically generated.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] TEZ-4007: Introduce AmExtensions and Zookeeper-based FrameworkServices [tez]
Posted by "tez-yetus (via GitHub)" <gi...@apache.org>.
tez-yetus commented on PR #427:
URL: https://github.com/apache/tez/pull/427#issuecomment-3585167290
:broken_heart: **-1 overall**
| Vote | Subsystem | Runtime | Logfile | Comment |
|:----:|----------:|--------:|:--------:|:-------:|
| +0 :ok: | reexec | 20m 25s | | Docker mode activated. |
|||| _ Prechecks _ |
| +1 :green_heart: | dupname | 0m 1s | | No case conflicting files found. |
| +0 :ok: | detsecrets | 0m 0s | | detect-secrets was not available. |
| +0 :ok: | xmllint | 0m 0s | | xmllint was not available. |
| +1 :green_heart: | @author | 0m 0s | | The patch does not contain any @author tags. |
| +1 :green_heart: | test4tests | 0m 0s | | The patch appears to include 14 new or modified test files. |
|||| _ master Compile Tests _ |
| +0 :ok: | mvndep | 2m 39s | | Maven dependency ordering for branch |
| +1 :green_heart: | mvninstall | 10m 3s | | master passed |
| +1 :green_heart: | compile | 5m 19s | | master passed |
| +1 :green_heart: | checkstyle | 3m 1s | | master passed |
| +1 :green_heart: | javadoc | 3m 42s | | master passed |
| +0 :ok: | spotbugs | 2m 11s | | tez-api in master has 610 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 0m 43s | | tez-common in master has 13 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 1m 10s | | tez-runtime-library in master has 241 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 0m 40s | | tez-examples in master has 2 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 1m 49s | | tez-dag in master has 785 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 7m 34s | | root in master has 2066 extant spotbugs warnings. |
| -0 :warning: | patch | 8m 16s | | Used diff version of patch file. Binary files and potentially other changes not applied. Please rebase and squash commits if necessary. |
|||| _ Patch Compile Tests _ |
| +0 :ok: | mvndep | 0m 16s | | Maven dependency ordering for patch |
| -1 :x: | mvninstall | 0m 24s | [/patch-mvninstall-tez-api.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/27/artifact/out/patch-mvninstall-tez-api.txt) | tez-api in the patch failed. |
| -1 :x: | mvninstall | 0m 26s | [/patch-mvninstall-tez-dag.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/27/artifact/out/patch-mvninstall-tez-dag.txt) | tez-dag in the patch failed. |
| -1 :x: | mvninstall | 1m 42s | [/patch-mvninstall-root.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/27/artifact/out/patch-mvninstall-root.txt) | root in the patch failed. |
| +1 :green_heart: | codespell | 0m 55s | | No new issues. |
| -1 :x: | compile | 0m 36s | [/patch-compile-tez-api.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/27/artifact/out/patch-compile-tez-api.txt) | tez-api in the patch failed. |
| -1 :x: | compile | 0m 37s | [/patch-compile-tez-dag.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/27/artifact/out/patch-compile-tez-dag.txt) | tez-dag in the patch failed. |
| -1 :x: | compile | 0m 37s | [/patch-compile-root.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/27/artifact/out/patch-compile-root.txt) | root in the patch failed. |
| -1 :x: | javac | 0m 36s | [/patch-compile-tez-api.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/27/artifact/out/patch-compile-tez-api.txt) | tez-api in the patch failed. |
| -1 :x: | javac | 0m 37s | [/patch-compile-tez-dag.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/27/artifact/out/patch-compile-tez-dag.txt) | tez-dag in the patch failed. |
| -1 :x: | javac | 0m 37s | [/patch-compile-root.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/27/artifact/out/patch-compile-root.txt) | root in the patch failed. |
| +1 :green_heart: | blanks | 0m 0s | | The patch has no blanks issues. |
| -0 :warning: | checkstyle | 0m 22s | [/results-checkstyle-tez-api.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/27/artifact/out/results-checkstyle-tez-api.txt) | tez-api: The patch generated 1 new + 118 unchanged - 3 fixed = 119 total (was 121) |
| +1 :green_heart: | checkstyle | 0m 18s | | The patch passed checkstyle in tez-common |
| +1 :green_heart: | checkstyle | 0m 24s | | The patch passed checkstyle in tez-runtime-library |
| +1 :green_heart: | checkstyle | 0m 19s | | tez-examples: The patch generated 0 new + 10 unchanged - 1 fixed = 10 total (was 11) |
| -0 :warning: | checkstyle | 0m 31s | [/results-checkstyle-tez-dag.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/27/artifact/out/results-checkstyle-tez-dag.txt) | tez-dag: The patch generated 1 new + 610 unchanged - 2 fixed = 611 total (was 612) |
| -0 :warning: | checkstyle | 0m 43s | [/results-checkstyle-root.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/27/artifact/out/results-checkstyle-root.txt) | root: The patch generated 2 new + 740 unchanged - 6 fixed = 742 total (was 746) |
| -1 :x: | javadoc | 0m 37s | [/results-javadoc-javadoc-tez-api.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/27/artifact/out/results-javadoc-javadoc-tez-api.txt) | tez-api generated 1 new + 10 unchanged - 0 fixed = 11 total (was 10) |
| -1 :x: | javadoc | 1m 16s | [/results-javadoc-javadoc-root.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/27/artifact/out/results-javadoc-javadoc-root.txt) | root generated 1 new + 19 unchanged - 0 fixed = 20 total (was 19) |
| -1 :x: | spotbugs | 0m 34s | [/patch-spotbugs-tez-api.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/27/artifact/out/patch-spotbugs-tez-api.txt) | tez-api in the patch failed. |
| -1 :x: | spotbugs | 0m 32s | [/patch-spotbugs-tez-dag.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/27/artifact/out/patch-spotbugs-tez-dag.txt) | tez-dag in the patch failed. |
| -1 :x: | spotbugs | 0m 45s | [/patch-spotbugs-root.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/27/artifact/out/patch-spotbugs-root.txt) | root in the patch failed. |
|||| _ Other Tests _ |
| -1 :x: | unit | 0m 35s | [/patch-unit-tez-api.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/27/artifact/out/patch-unit-tez-api.txt) | tez-api in the patch failed. |
| +1 :green_heart: | unit | 0m 45s | | tez-common in the patch passed. |
| +1 :green_heart: | unit | 5m 56s | | tez-runtime-library in the patch passed. |
| +1 :green_heart: | unit | 0m 26s | | tez-examples in the patch passed. |
| -1 :x: | unit | 0m 36s | [/patch-unit-tez-dag.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/27/artifact/out/patch-unit-tez-dag.txt) | tez-dag in the patch failed. |
| -1 :x: | unit | 2m 1s | [/patch-unit-root.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/27/artifact/out/patch-unit-root.txt) | root in the patch failed. |
| +1 :green_heart: | asflicense | 1m 50s | | The patch does not generate ASF License warnings. |
| | | 95m 10s | | |
| Subsystem | Report/Notes |
|----------:|:-------------|
| Docker | ClientAPI=1.52 ServerAPI=1.52 base: https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/27/artifact/out/Dockerfile |
| GITHUB PR | https://github.com/apache/tez/pull/427 |
| Optional Tests | dupname asflicense javac javadoc unit codespell detsecrets xmllint compile spotbugs checkstyle |
| uname | Linux 0f9248c05bc0 5.15.0-161-generic #171-Ubuntu SMP Sat Oct 11 08:17:01 UTC 2025 x86_64 x86_64 x86_64 GNU/Linux |
| Build tool | maven |
| Personality | /home/jenkins/jenkins-home/workspace/tez-multibranch_PR-427/src/.yetus/personality.sh |
| git revision | master / 1b0138328c7a3647cb34bc9061f4e847ab8cb096 |
| Default Java | Ubuntu-21.0.8+9-Ubuntu-0ubuntu124.04.1 |
| Test Results | https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/27/testReport/ |
| Max. process+thread count | 1103 (vs. ulimit of 5500) |
| modules | C: tez-api tez-common tez-runtime-library tez-examples tez-dag . U: . |
| Console output | https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/27/console |
| versions | git=2.43.0 maven=3.8.7 spotbugs=4.9.3 codespell=2.0.0 |
| Powered by | Apache Yetus 0.15.1 https://yetus.apache.org |
This message was automatically generated.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] TEZ-4007: Introduce AmExtensions and Zookeeper-based FrameworkServices [tez]
Posted by "tez-yetus (via GitHub)" <gi...@apache.org>.
tez-yetus commented on PR #427:
URL: https://github.com/apache/tez/pull/427#issuecomment-3598374712
:confetti_ball: **+1 overall**
| Vote | Subsystem | Runtime | Logfile | Comment |
|:----:|----------:|--------:|:--------:|:-------:|
| +0 :ok: | reexec | 0m 29s | | Docker mode activated. |
|||| _ Prechecks _ |
| +1 :green_heart: | dupname | 0m 0s | | No case conflicting files found. |
| +0 :ok: | detsecrets | 0m 1s | | detect-secrets was not available. |
| +0 :ok: | xmllint | 0m 1s | | xmllint was not available. |
| +1 :green_heart: | @author | 0m 0s | | The patch does not contain any @author tags. |
| +1 :green_heart: | test4tests | 0m 0s | | The patch appears to include 14 new or modified test files. |
|||| _ master Compile Tests _ |
| +0 :ok: | mvndep | 2m 40s | | Maven dependency ordering for branch |
| +1 :green_heart: | mvninstall | 9m 59s | | master passed |
| +1 :green_heart: | compile | 5m 15s | | master passed |
| +1 :green_heart: | checkstyle | 2m 54s | | master passed |
| +1 :green_heart: | javadoc | 4m 26s | | master passed |
| +0 :ok: | spotbugs | 2m 11s | | tez-api in master has 610 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 0m 47s | | tez-common in master has 13 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 1m 11s | | tez-runtime-library in master has 241 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 0m 41s | | tez-examples in master has 2 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 1m 45s | | tez-dag in master has 785 extant spotbugs warnings. |
| +0 :ok: | spotbugs | 7m 21s | | root in master has 2066 extant spotbugs warnings. |
| -0 :warning: | patch | 8m 6s | | Used diff version of patch file. Binary files and potentially other changes not applied. Please rebase and squash commits if necessary. |
|||| _ Patch Compile Tests _ |
| +0 :ok: | mvndep | 0m 16s | | Maven dependency ordering for patch |
| +1 :green_heart: | mvninstall | 6m 53s | | the patch passed |
| +1 :green_heart: | codespell | 0m 51s | | No new issues. |
| +1 :green_heart: | compile | 5m 20s | | the patch passed |
| +1 :green_heart: | javac | 5m 20s | | the patch passed |
| +1 :green_heart: | blanks | 0m 0s | | The patch has no blanks issues. |
| -0 :warning: | checkstyle | 0m 22s | [/results-checkstyle-tez-api.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/34/artifact/out/results-checkstyle-tez-api.txt) | tez-api: The patch generated 1 new + 118 unchanged - 3 fixed = 119 total (was 121) |
| +1 :green_heart: | checkstyle | 0m 21s | | The patch passed checkstyle in tez-common |
| +1 :green_heart: | checkstyle | 0m 21s | | The patch passed checkstyle in tez-runtime-library |
| +1 :green_heart: | checkstyle | 0m 19s | | tez-examples: The patch generated 0 new + 10 unchanged - 1 fixed = 10 total (was 11) |
| -0 :warning: | checkstyle | 0m 30s | [/results-checkstyle-tez-dag.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/34/artifact/out/results-checkstyle-tez-dag.txt) | tez-dag: The patch generated 2 new + 610 unchanged - 2 fixed = 612 total (was 612) |
| -0 :warning: | checkstyle | 0m 43s | [/results-checkstyle-root.txt](https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/34/artifact/out/results-checkstyle-root.txt) | root: The patch generated 3 new + 740 unchanged - 6 fixed = 743 total (was 746) |
| +1 :green_heart: | javadoc | 3m 38s | | the patch passed |
| +1 :green_heart: | spotbugs | 14m 43s | | the patch passed |
|||| _ Other Tests _ |
| +1 :green_heart: | unit | 2m 35s | | tez-api in the patch passed. |
| +1 :green_heart: | unit | 0m 40s | | tez-common in the patch passed. |
| +1 :green_heart: | unit | 5m 58s | | tez-runtime-library in the patch passed. |
| +1 :green_heart: | unit | 0m 26s | | tez-examples in the patch passed. |
| +1 :green_heart: | unit | 6m 1s | | tez-dag in the patch passed. |
| +1 :green_heart: | unit | 73m 2s | | root in the patch passed. |
| +1 :green_heart: | asflicense | 2m 32s | | The patch does not generate ASF License warnings. |
| | | 170m 23s | | |
| Subsystem | Report/Notes |
|----------:|:-------------|
| Docker | ClientAPI=1.52 ServerAPI=1.52 base: https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/34/artifact/out/Dockerfile |
| GITHUB PR | https://github.com/apache/tez/pull/427 |
| Optional Tests | dupname asflicense javac javadoc unit codespell detsecrets xmllint compile spotbugs checkstyle |
| uname | Linux dee85b6ffdbd 5.15.0-161-generic #171-Ubuntu SMP Sat Oct 11 08:17:01 UTC 2025 x86_64 x86_64 x86_64 GNU/Linux |
| Build tool | maven |
| Personality | /home/jenkins/jenkins-home/workspace/tez-multibranch_PR-427/src/.yetus/personality.sh |
| git revision | master / 2e8a6fb3844a7a85bcdd5e36d055d404cee79a31 |
| Default Java | Ubuntu-21.0.8+9-Ubuntu-0ubuntu124.04.1 |
| Test Results | https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/34/testReport/ |
| Max. process+thread count | 2107 (vs. ulimit of 5500) |
| modules | C: tez-api tez-common tez-runtime-library tez-examples tez-dag . U: . |
| Console output | https://ci-hadoop.apache.org/job/tez-multibranch/job/PR-427/34/console |
| versions | git=2.43.0 maven=3.8.7 spotbugs=4.9.3 codespell=2.0.0 |
| Powered by | Apache Yetus 0.15.1 https://yetus.apache.org |
This message was automatically generated.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] TEZ-4007: Introduce AmExtensions and Zookeeper-based FrameworkServices [tez]
Posted by "abstractdog (via GitHub)" <gi...@apache.org>.
abstractdog commented on code in PR #427:
URL: https://github.com/apache/tez/pull/427#discussion_r2561667762
##########
tez-api/src/main/java/org/apache/tez/client/registry/zookeeper/ZkFrameworkClient.java:
##########
@@ -0,0 +1,164 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.client.registry.zookeeper;
+
+import java.io.IOException;
+
+import org.apache.hadoop.yarn.api.protocolrecords.GetNewApplicationResponse;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ApplicationReport;
+import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
+import org.apache.hadoop.yarn.api.records.FinalApplicationStatus;
+import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.client.api.YarnClientApplication;
+import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.util.Records;
+import org.apache.tez.client.FrameworkClient;
+import org.apache.tez.client.registry.AMRecord;
+import org.apache.tez.dag.api.TezConfiguration;
+
+public class ZkFrameworkClient extends FrameworkClient {
+
+ private AMRecord amRecord;
+ private ZkAMRegistryClient amRegistryClient = null;
+ private volatile boolean isRunning = false;
+ private String amHost;
+ private int amPort;
+
+ @Override
+ public synchronized void init(TezConfiguration tezConf) {
+ if (this.amRegistryClient == null) {
+ try {
+ this.amRegistryClient = ZkAMRegistryClient.getClient(tezConf);
+ this.isRunning = true;
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+ }
+
+ @Override
+ public void start() {
+ try {
+ this.amRegistryClient.start();
+ isRunning = true;
+ } catch (Exception e) {
+ throw new RuntimeException(e);
+ }
+ }
+
+ @Override
+ public void stop() {
+ isRunning = false;
+ close();
+ }
+
+ @Override
+ public void close() {
+ if (amRegistryClient != null) {
+ amRegistryClient.close();
+ }
+ }
+
+ /**
+ * Creates a dummy {@link YarnClientApplication} using a pre-existing {@link ApplicationId}
+ * rather than requesting a new one from the ResourceManager.
+ *
+ * <p><strong>Note:</strong> This is a <em>dummy, backward-compatibility implementation</em>.
+ * Instead of allocating a fresh application ID from the ResourceManager, this method
+ * reuses the {@code applicationId} already obtained via {@code getApplicationReport()}.
+ * This allows legacy code paths to continue operating without requiring actual
+ * creation of a new application.</p>
+ *
+ * <p>Hidden assumption here: this method assumes that
+ * {@code getApplicationReport()} has already been called before
+ * {@code createApplication()}, ensuring that {@code amRecord.getApplicationId()}
+ * is always available. This assumption holds in all supported usage patterns:
+ * the only code path where {@code createApplication()} might be called first is
+ * {@code TezClient.submitDAGApplication()}, but that path is never exercised in
+ * Zookeeper standalone mode because that mode assumes applications are already
+ * running. Therefore, the ordering guarantee is valid in practice.</p>
+ *
+ * <p>
+ * The method constructs a minimal {@link ApplicationSubmissionContext} and a
+ * synthetic {@link GetNewApplicationResponse}, both populated with the already
+ * known application ID. These objects are then wrapped into a
+ * {@link YarnClientApplication} instance and returned.
+ * </p>
+ *
+ * @return a {@link YarnClientApplication} backed by a submission context and
+ * a mocked {@link GetNewApplicationResponse}, both tied to the pre-existing
+ * application ID.
+ */
+ @Override
+ public YarnClientApplication createApplication() {
+ ApplicationSubmissionContext context = Records.newRecord(ApplicationSubmissionContext.class);
+ ApplicationId appId = amRecord.getApplicationId();
Review Comment:
I know it looks confusing, and already added a javadoc explain this:
```
* <p><strong>Note:</strong> This is a <em>dummy, backward-compatibility implementation</em>.
* Instead of allocating a fresh application ID from the ResourceManager, this method
* reuses the {@code applicationId} already obtained via {@code getApplicationReport()}.
* This allows legacy code paths to continue operating without requiring actual
* creation of a new application.</p>
*
* <p>Hidden assumption here: this method assumes that
* {@code getApplicationReport()} has already been called before
* {@code createApplication()}, ensuring that {@code amRecord.getApplicationId()}
* is always available. This assumption holds in all supported usage patterns:
* the only code path where {@code createApplication()} might be called first is
* {@code TezClient.submitDAGApplication()}, but that path is never exercised in
* Zookeeper standalone mode because that mode assumes applications are already
* running. Therefore, the ordering guarantee is valid in practice.</p>
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] TEZ-4007: Introduce AmExtensions and Zookeeper-based FrameworkServices [tez]
Posted by "abstractdog (via GitHub)" <gi...@apache.org>.
abstractdog commented on code in PR #427:
URL: https://github.com/apache/tez/pull/427#discussion_r2563630356
##########
tez-api/src/main/java/org/apache/tez/frameworkplugins/FrameworkUtils.java:
##########
@@ -0,0 +1,96 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tez.frameworkplugins;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.ServiceLoader;
+
+import javax.annotation.Nullable;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.tez.common.ReflectionUtils;
+import org.apache.tez.dag.api.TezConfiguration;
+import org.apache.tez.dag.api.TezConstants;
+import org.apache.tez.dag.api.TezReflectionException;
+
+public final class FrameworkUtils {
+
+ private FrameworkUtils() {}
+
+ /*
+ Searches for a FrameworkService provider which implements a target interface.
+ The interface should be either ClientFrameworkService or ServerFrameworkService.
+ Depending on which interface is used, either the client or server class of a
+ matching FrameworkMode will be used as the implementation.
+
+ NOTE: Layering of FrameworkServices in a decorator-style is currently not supported
+
+ An implementation is searched in the following order:
+ 1. If conf is not null and the parameter TEZ_FRAMEWORK_MODE is set:
+ the value of TEZ_FRAMEWORK_MODE from the conf will be used
+ 2. If conf is null or the parameter TEZ_FRAMEWORK_MODE is not set
+ and the environment var TEZ_FRAMEWORK_MODE is not empty:
+ the value of the environment var will be used
+ 3. Otherwise:
+ the default java.util.ServiceLoader behavior will be used,
+ i.e. the implementation classname should appear in a file on the classpath at the location
+ META-INF/services/org.apache.tez.frameworkplugins.ClientFrameworkService
+ or META-INF/services/org.apache.tez.frameworkplugins.ServerFrameworkService
+ */
+ public static <T extends FrameworkService> T get(Class<T> interfaze, @Nullable Configuration conf) {
+ try {
+ if ((conf != null) && (conf.get(TezConfiguration.TEZ_FRAMEWORK_MODE) != null)) {
+ return getByMode(interfaze, conf.get(TezConfiguration.TEZ_FRAMEWORK_MODE));
+ } else if (System.getenv(TezConstants.TEZ_FRAMEWORK_MODE) != null) {
+ return getByMode(interfaze, System.getenv(TezConstants.TEZ_FRAMEWORK_MODE));
+ } else {
+ return getByServiceLoader(interfaze);
+ }
+ } catch (TezReflectionException e) {
+ throw new RuntimeException("Failed to load framework service for interface: " + interfaze.getName(), e);
+ }
+ }
+
+ private static <T extends FrameworkService> T getByServiceLoader(Class<T> interfaze) {
+ List<T> services = new ArrayList<>();
+ ServiceLoader<T> frameworkService = ServiceLoader.load(interfaze);
+ for (T service : frameworkService) {
+ services.add(service);
+ }
+ if (services.isEmpty()) {
+ return null;
Review Comment:
yeah, I was thinking about the same, which might look hacky, but maybe it's worth it
the problem is that this method is in `tez-api`, whereas the server-side stuff is in `tez-dag`, and `tez-api` by design cannot depend on `tez-dag`, so instead of the null-checks, this might need at least two runtime workarounds:
1. check if `interfaze` is `ServerFrameworkService` or `ClientFrameworkService`, and return Yarn one accordingly
2. instantiate the `YarnServerFrameworkService` by reflection (reflection is needed here, because even if moving the `ServerFrameworkService` to `tez-api` fine, doing the same with `YarnServerFrameworkService` looks strange, as it's Tez AM specific implementation
UPDATE: I just realized this is already handled in the `getByMode` before the `getByServiceLoader`, so this is another case, sorry for the noise, let me check this
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] TEZ-4007: Introduce AmExtensions and Zookeeper-based FrameworkServices [tez]
Posted by "abstractdog (via GitHub)" <gi...@apache.org>.
abstractdog commented on code in PR #427:
URL: https://github.com/apache/tez/pull/427#discussion_r2563630356
##########
tez-api/src/main/java/org/apache/tez/frameworkplugins/FrameworkUtils.java:
##########
@@ -0,0 +1,96 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.tez.frameworkplugins;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.ServiceLoader;
+
+import javax.annotation.Nullable;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.tez.common.ReflectionUtils;
+import org.apache.tez.dag.api.TezConfiguration;
+import org.apache.tez.dag.api.TezConstants;
+import org.apache.tez.dag.api.TezReflectionException;
+
+public final class FrameworkUtils {
+
+ private FrameworkUtils() {}
+
+ /*
+ Searches for a FrameworkService provider which implements a target interface.
+ The interface should be either ClientFrameworkService or ServerFrameworkService.
+ Depending on which interface is used, either the client or server class of a
+ matching FrameworkMode will be used as the implementation.
+
+ NOTE: Layering of FrameworkServices in a decorator-style is currently not supported
+
+ An implementation is searched in the following order:
+ 1. If conf is not null and the parameter TEZ_FRAMEWORK_MODE is set:
+ the value of TEZ_FRAMEWORK_MODE from the conf will be used
+ 2. If conf is null or the parameter TEZ_FRAMEWORK_MODE is not set
+ and the environment var TEZ_FRAMEWORK_MODE is not empty:
+ the value of the environment var will be used
+ 3. Otherwise:
+ the default java.util.ServiceLoader behavior will be used,
+ i.e. the implementation classname should appear in a file on the classpath at the location
+ META-INF/services/org.apache.tez.frameworkplugins.ClientFrameworkService
+ or META-INF/services/org.apache.tez.frameworkplugins.ServerFrameworkService
+ */
+ public static <T extends FrameworkService> T get(Class<T> interfaze, @Nullable Configuration conf) {
+ try {
+ if ((conf != null) && (conf.get(TezConfiguration.TEZ_FRAMEWORK_MODE) != null)) {
+ return getByMode(interfaze, conf.get(TezConfiguration.TEZ_FRAMEWORK_MODE));
+ } else if (System.getenv(TezConstants.TEZ_FRAMEWORK_MODE) != null) {
+ return getByMode(interfaze, System.getenv(TezConstants.TEZ_FRAMEWORK_MODE));
+ } else {
+ return getByServiceLoader(interfaze);
+ }
+ } catch (TezReflectionException e) {
+ throw new RuntimeException("Failed to load framework service for interface: " + interfaze.getName(), e);
+ }
+ }
+
+ private static <T extends FrameworkService> T getByServiceLoader(Class<T> interfaze) {
+ List<T> services = new ArrayList<>();
+ ServiceLoader<T> frameworkService = ServiceLoader.load(interfaze);
+ for (T service : frameworkService) {
+ services.add(service);
+ }
+ if (services.isEmpty()) {
+ return null;
Review Comment:
yeah, I was thinking about the same, which might look hacky, but maybe it's worth it
the problem is that this method is in `tez-api`, whereas the server-side stuff is in `tez-dag`, and `tez-api` by design cannot depend on `tez-dag`, so instead of the null-checks, this might need at least two runtime workarounds:
1. check if `interfaze` is `ServerFrameworkService` or `ClientFrameworkService`, and return Yarn one accordingly
2. instantiate the `YarnServerFrameworkService` by reflection (reflection is needed here, because even if moving the `ServerFrameworkService` to `tez-api` fine, doing the same with `YarnServerFrameworkService` looks strange, as it's Tez AM specific implementation
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] TEZ-4007: Introduce AmExtensions and Zookeeper-based FrameworkServices [tez]
Posted by "abstractdog (via GitHub)" <gi...@apache.org>.
abstractdog commented on code in PR #427:
URL: https://github.com/apache/tez/pull/427#discussion_r2563595946
##########
tez-api/src/main/java/org/apache/tez/client/registry/zookeeper/ZkConfig.java:
##########
@@ -0,0 +1,128 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.client.registry.zookeeper;
+
+import org.apache.curator.RetryPolicy;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.CuratorFrameworkFactory;
+import org.apache.curator.retry.ExponentialBackoffRetry;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.tez.dag.api.TezConfiguration;
+
+import com.google.common.base.Preconditions;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class ZkConfig {
+ private static final Logger LOG = LoggerFactory.getLogger(ZkConfig.class);
+
+ // if namespace defined in config is 'foo' and COMPUTE_GROUP_NAME env is 'bar' then the zkpaths will be of format
+ // /tez-external-sessions/foo/bar
+ private final static String ZK_NAMESPACE_PREFIX = "/tez-external-sessions";
+ public final static String COMPUTE_GROUP_NAME_ENV = "COMPUTE_GROUP_NAME";
+ public final static String DEFAULT_COMPUTE_GROUP_NAME = "default-compute";
+
+ private final String zkQuorum;
+ private final String zkNamespace;
+ private final int curatorBackoffSleep;
+ private final int curatorMaxRetries;
+ private final int sessionTimeoutMs;
+ private final int connectionTimeoutMs;
+
+ public ZkConfig(Configuration conf) {
+ zkQuorum = conf.get(TezConfiguration.TEZ_AM_ZOOKEEPER_QUORUM);
+ Preconditions.checkNotNull(zkQuorum);
+
+ String fullZkNamespace = ZK_NAMESPACE_PREFIX;
+
+ String namespace = conf.get(TezConfiguration.TEZ_AM_REGISTRY_NAMESPACE,
Review Comment:
this regex replaces leading slashes with an empty string, which leads to this exception in ZK:
`java.lang.IllegalArgumentException: Path must start with / character`
in ZK, a node should start with a slash
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
Re: [PR] TEZ-4007: Introduce AmExtensions and Zookeeper-based FrameworkServices [tez]
Posted by "abstractdog (via GitHub)" <gi...@apache.org>.
abstractdog commented on code in PR #427:
URL: https://github.com/apache/tez/pull/427#discussion_r2671865869
##########
tez-api/src/main/java/org/apache/tez/client/registry/zookeeper/ZkAMRegistryClient.java:
##########
@@ -0,0 +1,209 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.tez.client.registry.zookeeper;
+
+import java.io.IOException;
+import java.nio.charset.StandardCharsets;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.stream.Collectors;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.curator.framework.CuratorFramework;
+import org.apache.curator.framework.imps.CuratorFrameworkState;
+import org.apache.curator.framework.recipes.cache.ChildData;
+import org.apache.curator.framework.recipes.cache.TreeCache;
+import org.apache.curator.framework.recipes.cache.TreeCacheEvent;
+import org.apache.curator.framework.recipes.cache.TreeCacheListener;
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.registry.client.types.ServiceRecord;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.tez.client.registry.AMRecord;
+import org.apache.tez.client.registry.AMRegistryClient;
+import org.apache.tez.client.registry.AMRegistryUtils;
+import org.apache.tez.dag.api.TezConfiguration;
+
+import com.fasterxml.jackson.core.JsonParseException;
+import com.google.common.base.Preconditions;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Curator/Zookeeper implementation of {@link AMRegistryClient}.
+ */
+@InterfaceAudience.Public
+public final class ZkAMRegistryClient extends AMRegistryClient {
+ private static final Logger LOG = LoggerFactory.getLogger(ZkAMRegistryClient.class);
+ private static final Map<String, ZkAMRegistryClient> INSTANCES = new HashMap<>();
+
+ private final Configuration conf;
+ // Cache of known AMs
+ private final ConcurrentHashMap<ApplicationId, AMRecord> amRecordCache = new ConcurrentHashMap<>();
+
+ private CuratorFramework client;
+ private TreeCache cache;
+ private ZkRegistryListener listener;
+
+ private ZkAMRegistryClient(final Configuration conf) {
+ this.conf = conf;
+ }
+
+ public static synchronized ZkAMRegistryClient getClient(final Configuration conf) {
+ String namespace = conf.get(TezConfiguration.TEZ_AM_REGISTRY_NAMESPACE);
+ ZkAMRegistryClient registry = INSTANCES.get(namespace);
+ if (registry == null) {
+ registry = new ZkAMRegistryClient(conf);
+ INSTANCES.put(namespace, registry);
+ }
+ LOG.info("Returning tez AM registry ({}) for namespace '{}'", System.identityHashCode(registry), namespace);
+ return registry;
+ }
+
+ /**
+ * Deserializes a {@link ServiceRecord} from ZooKeeper data and converts it into an {@link AMRecord}
+ * for caching.
+ *
+ * @param childData the ZooKeeper node data containing a serialized {@link ServiceRecord}
+ * @return an {@link AMRecord} constructed from the deserialized {@link ServiceRecord}, or {@code null}
+ * if no data is present
+ * @throws IOException if the data cannot be deserialized into a {@link ServiceRecord}
+ */
+ public static AMRecord getAMRecord(final ChildData childData) throws IOException {
+ // Not a leaf path. Only leaf path contains AMRecord.
+ if (!childData.getPath().contains(ApplicationId.appIdStrPrefix)) {
+ return null;
+ }
+ byte[] data = childData.getData();
+ // only the path appeared, there is no data yet
+ if (data.length == 0) {
+ return null;
+ }
+ String value = new String(data, StandardCharsets.UTF_8);
+ try {
+ return AMRegistryUtils.jsonStringToRecord(value);
+ } catch (JsonParseException e) {
+ //Not a json AMRecord (SRV), could be some other data.
+ LOG.warn("Non-json data received while de-serializing AMRecord: {}. Ignoring...", value);
+ return null;
+ }
+ }
+
+ public void start() throws Exception {
+ ZkConfig zkConf = new ZkConfig(this.conf);
+ client = zkConf.createCuratorFramework();
+ cache = new TreeCache(client, zkConf.getZkNamespace());
+ client.start();
+ cache.start();
+ listener = new ZkRegistryListener();
+ cache.getListenable().addListener(listener);
+ }
+
+ @Override
+ public AMRecord getRecord(ApplicationId appId) {
+ AMRecord rec = amRecordCache.get(appId);
+ // Return a copy.
+ return rec == null ? null : new AMRecord(rec);
+ }
+
+ @Override
+ public List<AMRecord> getAllRecords() {
+ return amRecordCache.values().stream().map(AMRecord::new).collect(Collectors.toList());
+ }
+
+ @Override
+ public void close() {
+ IOUtils.closeQuietly(cache);
+ IOUtils.closeQuietly(client);
+ }
+
+ public boolean isInitialized() {
+ return listener.initialized;
+ }
+
+ /**
+ * Callback listener for ZooKeeper events that updates the local cache
+ * when child nodes under the monitored path change.
+ */
+ private class ZkRegistryListener implements TreeCacheListener {
+
+ private boolean initialized = false;
+
+ @Override
+ public void childEvent(final CuratorFramework clientParam, final TreeCacheEvent event) throws Exception {
+ Preconditions.checkArgument(clientParam != null && clientParam.getState() == CuratorFrameworkState.STARTED,
+ "Curator client is not started");
+
+ ChildData childData = event.getData();
+ switch (event.getType()) {
+ case NODE_ADDED:
+ if (isEmpty(childData)) {
+ LOG.info("AppId allocated: {}", childData.getPath());
+ } else {
+ AMRecord amRecord = getAMRecord(childData);
+ if (amRecord != null) {
+ LOG.info("AM registered with data: {}. Notifying listeners.", amRecord);
+ amRecordCache.put(amRecord.getApplicationId(), amRecord);
+ notifyOnAdded(amRecord);
+ }
+ }
+ break;
+ case NODE_UPDATED:
+ if (isEmpty(childData)) {
+ throw new RuntimeException("AM updated with empty data");
+ } else {
+ AMRecord amRecord = getAMRecord(childData);
+ if (amRecord != null) {
+ LOG.info("AM updated data: {}. Notifying listeners.", amRecord);
+ amRecordCache.put(amRecord.getApplicationId(), amRecord);
+ notifyOnAdded(amRecord);
Review Comment:
good catch, yes!
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@tez.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org