You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by su...@apache.org on 2019/10/05 03:46:20 UTC
[hadoop] branch branch-3.2 updated: Revert "YARN-9873. Mutation API
Config Change updates Version Number. Contributed by Prabhu Joseph"
This is an automated email from the ASF dual-hosted git repository.
sunilg pushed a commit to branch branch-3.2
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-3.2 by this push:
new 5704f15 Revert "YARN-9873. Mutation API Config Change updates Version Number. Contributed by Prabhu Joseph"
5704f15 is described below
commit 5704f1558974e97ace213c75fe99d5f327d33db9
Author: Sunil G <su...@apache.org>
AuthorDate: Sat Oct 5 09:16:04 2019 +0530
Revert "YARN-9873. Mutation API Config Change updates Version Number. Contributed by Prabhu Joseph"
This reverts commit 3a0afcfb7f196bee8e534a3e80211bfa077b2aee.
---
.../scheduler/MutableConfigurationProvider.java | 6 ----
.../conf/FSSchedulerConfigurationStore.java | 7 -----
.../capacity/conf/InMemoryConfigurationStore.java | 8 -----
.../capacity/conf/LeveldbConfigurationStore.java | 23 +--------------
.../conf/MutableCSConfigurationProvider.java | 5 ----
.../capacity/conf/YarnConfigurationStore.java | 6 ----
.../capacity/conf/ZKConfigurationStore.java | 19 ------------
.../server/resourcemanager/webapp/RMWSConsts.java | 3 --
.../resourcemanager/webapp/RMWebServices.java | 34 +---------------------
.../capacity/conf/TestZKConfigurationStore.java | 15 ----------
10 files changed, 2 insertions(+), 124 deletions(-)
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
index eff8aa8..9e843df 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/MutableConfigurationProvider.java
@@ -65,12 +65,6 @@ public interface MutableConfigurationProvider {
*/
Configuration getConfiguration();
- /**
- * Get the last updated scheduler config version.
- * @return Last updated scheduler config version.
- */
- long getConfigVersion() throws Exception;
-
void formatConfigurationInStore(Configuration conf) throws Exception;
/**
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/FSSchedulerConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/FSSchedulerConfigurationStore.java
index 14cd0a4..ddc5c8a 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/FSSchedulerConfigurationStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/FSSchedulerConfigurationStore.java
@@ -148,13 +148,6 @@ public class FSSchedulerConfigurationStore extends YarnConfigurationStore {
tempConfigPath = null;
}
- @Override
- public long getConfigVersion() throws Exception {
- String version = getLatestConfigPath().getName().
- substring(YarnConfiguration.CS_CONFIGURATION_FILE.length() + 1);
- return Long.parseLong(version);
- }
-
private void finalizeFileSystemFile() throws IOException {
// call confirmMutation() make sure tempConfigPath is not null
Path finalConfigPath = getFinalConfigPath(tempConfigPath);
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java
index 59d140e..4871443 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/InMemoryConfigurationStore.java
@@ -33,13 +33,11 @@ public class InMemoryConfigurationStore extends YarnConfigurationStore {
private Configuration schedConf;
private LogMutation pendingMutation;
- private long configVersion;
@Override
public void initialize(Configuration conf, Configuration schedConf,
RMContext rmContext) {
this.schedConf = schedConf;
- this.configVersion = System.currentTimeMillis();
}
@Override
@@ -59,7 +57,6 @@ public class InMemoryConfigurationStore extends YarnConfigurationStore {
}
}
}
- this.configVersion = System.currentTimeMillis();
pendingMutation = null;
}
@@ -74,11 +71,6 @@ public class InMemoryConfigurationStore extends YarnConfigurationStore {
}
@Override
- public long getConfigVersion() throws Exception {
- return configVersion;
- }
-
- @Override
public List<LogMutation> getConfirmedConfHistory(long fromId) {
// Unimplemented.
return null;
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/LeveldbConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/LeveldbConfigurationStore.java
index 19614c2..0792d7f 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/LeveldbConfigurationStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/LeveldbConfigurationStore.java
@@ -68,7 +68,6 @@ public class LeveldbConfigurationStore extends YarnConfigurationStore {
private static final String DB_NAME = "yarn-conf-store";
private static final String LOG_KEY = "log";
private static final String VERSION_KEY = "version";
- private static final String CONF_VERSION_KEY = "conf-version";
private DB db;
private long maxLogs;
@@ -125,10 +124,6 @@ public class LeveldbConfigurationStore extends YarnConfigurationStore {
return 1;
} else if (key2Str.equals(LOG_KEY)) {
return -1;
- } else if (key1Str.equals(CONF_VERSION_KEY)) {
- return 1;
- } else if (key2Str.equals(CONF_VERSION_KEY)) {
- return -1;
}
return key1Str.compareTo(key2Str);
}
@@ -151,10 +146,6 @@ public class LeveldbConfigurationStore extends YarnConfigurationStore {
File dbfile = new File(storeRoot.toString());
try {
db = JniDBFactory.factory.open(dbfile, options);
- if (db.get(bytes(CONF_VERSION_KEY)) == null) {
- db.put(bytes(CONF_VERSION_KEY),
- bytes(String.valueOf(System.currentTimeMillis())));
- }
} catch (NativeDB.DBException e) {
if (e.isNotFound() || e.getMessage().contains(" does not exist ")) {
LOG.info("Creating conf database at " + dbfile);
@@ -167,8 +158,6 @@ public class LeveldbConfigurationStore extends YarnConfigurationStore {
initBatch.put(bytes(kv.getKey()), bytes(kv.getValue()));
}
db.write(initBatch);
- db.put(bytes(CONF_VERSION_KEY),
- bytes(String.valueOf(System.currentTimeMillis())));
} catch (DBException dbErr) {
throw new IOException(dbErr.getMessage(), dbErr);
}
@@ -226,8 +215,6 @@ public class LeveldbConfigurationStore extends YarnConfigurationStore {
}
}
db.write(updateBatch);
- db.put(bytes(CONF_VERSION_KEY),
- bytes(String.valueOf(System.currentTimeMillis())));
pendingMutation = null;
}
@@ -263,8 +250,7 @@ public class LeveldbConfigurationStore extends YarnConfigurationStore {
Map.Entry<byte[], byte[]> entry = itr.next();
String key = new String(entry.getKey(), StandardCharsets.UTF_8);
String value = new String(entry.getValue(), StandardCharsets.UTF_8);
- if (key.equals(LOG_KEY) || key.equals(VERSION_KEY) ||
- key.equals(CONF_VERSION_KEY)) {
+ if (key.equals(LOG_KEY) || key.equals(VERSION_KEY)) {
break;
}
config.set(key, value);
@@ -273,13 +259,6 @@ public class LeveldbConfigurationStore extends YarnConfigurationStore {
}
@Override
- public long getConfigVersion() throws Exception {
- String version = new String(db.get(bytes(CONF_VERSION_KEY)),
- StandardCharsets.UTF_8);
- return Long.parseLong(version);
- }
-
- @Override
public List<LogMutation> getConfirmedConfHistory(long fromId) {
return null; // unimplemented
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
index ba325b3..6d153dc 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/MutableCSConfigurationProvider.java
@@ -135,11 +135,6 @@ public class MutableCSConfigurationProvider implements CSConfigurationProvider,
}
@Override
- public long getConfigVersion() throws Exception {
- return confStore.getConfigVersion();
- }
-
- @Override
public ConfigurationMutationACLPolicy getAclMutationPolicy() {
return aclMutationPolicy;
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java
index bbeacc0..493f38e 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/YarnConfigurationStore.java
@@ -133,12 +133,6 @@ public abstract class YarnConfigurationStore {
public abstract void format() throws Exception;
/**
- * Get the last updated config version.
- * @return Last updated config version.
- */
- public abstract long getConfigVersion() throws Exception;
-
- /**
* Get a list of confirmed configuration mutations starting from a given id.
* @param fromId id from which to start getting mutations, inclusive
* @return list of configuration mutations
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java
index 063910d..766029b 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/ZKConfigurationStore.java
@@ -62,13 +62,11 @@ public class ZKConfigurationStore extends YarnConfigurationStore {
private static final String LOGS_PATH = "LOGS";
private static final String CONF_STORE_PATH = "CONF_STORE";
private static final String FENCING_PATH = "FENCING";
- private static final String CONF_VERSION_PATH = "CONF_VERSION";
private String zkVersionPath;
private String logsPath;
private String confStorePath;
private String fencingNodePath;
- private String confVersionPath;
@VisibleForTesting
protected ZKCuratorManager zkManager;
@@ -91,7 +89,6 @@ public class ZKConfigurationStore extends YarnConfigurationStore {
this.logsPath = getNodePath(znodeParentPath, LOGS_PATH);
this.confStorePath = getNodePath(znodeParentPath, CONF_STORE_PATH);
this.fencingNodePath = getNodePath(znodeParentPath, FENCING_PATH);
- this.confVersionPath = getNodePath(znodeParentPath, CONF_VERSION_PATH);
zkManager.createRootDirRecursively(znodeParentPath, zkAcl);
zkManager.delete(fencingNodePath);
@@ -102,12 +99,6 @@ public class ZKConfigurationStore extends YarnConfigurationStore {
serializeObject(new LinkedList<LogMutation>()), -1);
}
- if (!zkManager.exists(confVersionPath)) {
- zkManager.create(confVersionPath);
- zkManager.setData(confVersionPath,
- String.valueOf(System.currentTimeMillis()), -1);
- }
-
if (!zkManager.exists(confStorePath)) {
zkManager.create(confStorePath);
HashMap<String, String> mapSchedConf = new HashMap<>();
@@ -115,8 +106,6 @@ public class ZKConfigurationStore extends YarnConfigurationStore {
mapSchedConf.put(entry.getKey(), entry.getValue());
}
zkManager.setData(confStorePath, serializeObject(mapSchedConf), -1);
- zkManager.setData(confVersionPath,
- String.valueOf(System.currentTimeMillis()), -1);
}
}
@@ -196,9 +185,6 @@ public class ZKConfigurationStore extends YarnConfigurationStore {
}
zkManager.safeSetData(confStorePath, serializeObject(mapConf), -1,
zkAcl, fencingNodePath);
- zkManager.setData(confVersionPath,
- String.valueOf(System.currentTimeMillis()), -1);
-
}
pendingMutation = null;
}
@@ -228,11 +214,6 @@ public class ZKConfigurationStore extends YarnConfigurationStore {
}
@Override
- public long getConfigVersion() throws Exception {
- return Long.parseLong(zkManager.getStringData(confVersionPath));
- }
-
- @Override
public List<LogMutation> getConfirmedConfHistory(long fromId) {
return null; // unimplemented
}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWSConsts.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWSConsts.java
index c02b3f4..4479c94 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWSConsts.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWSConsts.java
@@ -51,9 +51,6 @@ public final class RMWSConsts {
/** Path for {@code RMWebServices#formatSchedulerConfiguration}. */
public static final String FORMAT_SCHEDULER_CONF = "/scheduler-conf/format";
- /** Path for {@code RMWebServices#getSchedulerConfigurationVersion}. */
- public static final String SCHEDULER_CONF_VERSION = "/scheduler-conf/version";
-
/** Path for {@code RMWebServiceProtocol#dumpSchedulerLogs}. */
public static final String SCHEDULER_LOGS = "/scheduler/logs";
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
index facd83f..56af548 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/RMWebServices.java
@@ -2345,7 +2345,7 @@ public class RMWebServices extends WebServices implements RMWebServiceProtocol {
}
} else {
return Response.status(Status.BAD_REQUEST)
- .entity("Scheduler Configuration format only supported by " +
+ .entity("Configuration change only supported by " +
"MutableConfScheduler.").build();
}
}
@@ -2436,38 +2436,6 @@ public class RMWebServices extends WebServices implements RMWebServiceProtocol {
}
@GET
- @Path(RMWSConsts.SCHEDULER_CONF_VERSION)
- @Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
- MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 })
- public Response getSchedulerConfigurationVersion(@Context
- HttpServletRequest hsr) throws AuthorizationException {
- // Only admin user is allowed to get scheduler conf version
- UserGroupInformation callerUGI = getCallerUserGroupInformation(hsr, true);
- initForWritableEndpoints(callerUGI, true);
-
- ResourceScheduler scheduler = rm.getResourceScheduler();
- if (scheduler instanceof MutableConfScheduler
- && ((MutableConfScheduler) scheduler).isConfigurationMutable()) {
- MutableConfigurationProvider mutableConfigurationProvider =
- ((MutableConfScheduler) scheduler).getMutableConfProvider();
-
- try {
- long configVersion = mutableConfigurationProvider
- .getConfigVersion();
- return Response.status(Status.OK).entity(configVersion).build();
- } catch (Exception e) {
- LOG.error("Exception thrown when fetching configuration version.", e);
- return Response.status(Status.BAD_REQUEST).entity(e.getMessage())
- .build();
- }
- } else {
- return Response.status(Status.BAD_REQUEST)
- .entity("Configuration Version only supported by "
- + "MutableConfScheduler.").build();
- }
- }
-
- @GET
@Path(RMWSConsts.CHECK_USER_ACCESS_TO_QUEUE)
@Produces({ MediaType.APPLICATION_JSON + "; " + JettyUtils.UTF_8,
MediaType.APPLICATION_XML + "; " + JettyUtils.UTF_8 })
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java
index 591d978..6a930e8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/conf/TestZKConfigurationStore.java
@@ -138,21 +138,6 @@ public class TestZKConfigurationStore extends ConfigurationStoreBaseTest {
}
@Test
- public void testGetConfigurationVersion() throws Exception {
- confStore.initialize(conf, schedConf, rmContext);
- long v1 = confStore.getConfigVersion();
- Thread.sleep(2000);
- Map<String, String> update = new HashMap<>();
- update.put("keyver", "valver");
- YarnConfigurationStore.LogMutation mutation =
- new YarnConfigurationStore.LogMutation(update, TEST_USER);
- confStore.logMutation(mutation);
- confStore.confirmMutation(true);
- long v2 = confStore.getConfigVersion();
- assertTrue(v2 > v1);
- }
-
- @Test
public void testPersistUpdatedConfiguration() throws Exception {
confStore.initialize(conf, schedConf, rmContext);
assertNull(confStore.retrieve().get("key"));
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org