You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sn...@apache.org on 2020/08/28 15:10:27 UTC
[hadoop] branch trunk updated: YARN-10386. Create new JSON schema
for Placement Rules. Contributed by Peter Bacsko and Benjamin Teke
This is an automated email from the ASF dual-hosted git repository.
snemeth pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 6e3d705 YARN-10386. Create new JSON schema for Placement Rules. Contributed by Peter Bacsko and Benjamin Teke
6e3d705 is described below
commit 6e3d7050a9c3a42f089b3b18c64ce37f99308f16
Author: Szilard Nemeth <sn...@apache.org>
AuthorDate: Fri Aug 28 17:09:31 2020 +0200
YARN-10386. Create new JSON schema for Placement Rules. Contributed by Peter Bacsko and Benjamin Teke
---
.gitignore | 1 +
hadoop-project/pom.xml | 49 ++++++++++++++
.../hadoop-yarn/dev-support/findbugs-exclude.xml | 5 ++
.../hadoop-yarn-server-resourcemanager/pom.xml | 31 +++++++++
.../capacity/placement/GeneratePojos.java | 76 ++++++++++++++++++++++
.../scheduler/capacity/placement/package-info.java | 23 +++++++
.../main/json_schema/MappingRulesDescription.json | 36 ++++++++++
7 files changed, 221 insertions(+)
diff --git a/.gitignore b/.gitignore
index 9b2d400..7c422a4 100644
--- a/.gitignore
+++ b/.gitignore
@@ -51,6 +51,7 @@ patchprocess/
.history/
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/package-lock.json
hadoop-yarn-project/hadoop-yarn/hadoop-yarn-ui/src/main/webapp/yarn-error.log
+hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/placement/schema
#robotframework outputs
log.html
diff --git a/hadoop-project/pom.xml b/hadoop-project/pom.xml
index 12ee139..4b7585e 100644
--- a/hadoop-project/pom.xml
+++ b/hadoop-project/pom.xml
@@ -207,6 +207,7 @@
<powermock.version>1.5.6</powermock.version>
<solr.version>7.7.0</solr.version>
<openssl-wildfly.version>1.0.7.Final</openssl-wildfly.version>
+ <jsonschema2pojo.version>1.0.2</jsonschema2pojo.version>
</properties>
<dependencyManagement>
@@ -1166,6 +1167,11 @@
<version>${jackson2.version}</version>
</dependency>
<dependency>
+ <groupId>com.fasterxml.jackson.dataformat</groupId>
+ <artifactId>jackson-dataformat-yaml</artifactId>
+ <version>${jackson2.version}</version>
+ </dependency>
+ <dependency>
<groupId>org.mockito</groupId>
<artifactId>mockito-core</artifactId>
<version>2.28.2</version>
@@ -1728,6 +1734,49 @@
<artifactId>jna</artifactId>
<version>${jna.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.jsonschema2pojo</groupId>
+ <artifactId>jsonschema2pojo-core</artifactId>
+ <version>${jsonschema2pojo.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.dataformat</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.codehaus.jackson</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.code.gson</groupId>
+ <artifactId>gson</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.code.findbugs</groupId>
+ <artifactId>annotations</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.scala-lang</groupId>
+ <artifactId>scala-library</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>org.jsonschema2pojo</groupId>
+ <artifactId>jsonschema2pojo-scalagen</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.google.code.javaparser</groupId>
+ <artifactId>javaparser</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>javax.validation</groupId>
+ <artifactId>validation-api</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
</dependencies>
</dependencyManagement>
diff --git a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
index 95706f9..81e8884 100644
--- a/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
+++ b/hadoop-yarn-project/hadoop-yarn/dev-support/findbugs-exclude.xml
@@ -711,4 +711,9 @@
<Class name="~org\.apache\.hadoop\.yarn\.server\.timelineservice\.reader\.TestTimelineReaderWebServicesHBaseStorage.*" />
<Bug pattern="UMAC_UNCALLABLE_METHOD_OF_ANONYMOUS_CLASS" />
</Match>
+
+ <!-- Generated files from JSON schema -->
+ <Match>
+ <Package name="org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.placement.schema" />
+ </Match>
</FindBugsFilter>
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml
index 3a1fa06..7283bf8 100644
--- a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/pom.xml
@@ -250,6 +250,21 @@
<scope>test</scope>
</dependency>
+ <!-- Necessary to include jackson separately to avoid issues with
+ maven enforcer plugin in "hadoop-client-check-test-invariants"
+ -->
+ <dependency>
+ <groupId>org.jsonschema2pojo</groupId>
+ <artifactId>jsonschema2pojo-core</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>com.fasterxml.jackson.dataformat</groupId>
+ <artifactId>jackson-dataformat-yaml</artifactId>
+ </dependency>
</dependencies>
<build>
@@ -380,6 +395,22 @@
</excludes>
</configuration>
</plugin>
+ <plugin>
+ <groupId>org.jsonschema2pojo</groupId>
+ <artifactId>jsonschema2pojo-maven-plugin</artifactId>
+ <configuration>
+ <sourceDirectory>${basedir}/src/main/json_schema</sourceDirectory>
+ <targetPackage>org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.placement.schema</targetPackage>
+ <outputDirectory>${basedir}/src/main/java</outputDirectory>
+ </configuration>
+ <executions>
+ <execution>
+ <goals>
+ <goal>generate</goal>
+ </goals>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
</build>
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/placement/GeneratePojos.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/placement/GeneratePojos.java
new file mode 100644
index 0000000..a769ac6
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/placement/GeneratePojos.java
@@ -0,0 +1,76 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.placement;
+
+import java.io.File;
+import java.io.IOException;
+import java.net.URL;
+import java.nio.file.Paths;
+
+import org.jsonschema2pojo.DefaultGenerationConfig;
+import org.jsonschema2pojo.GenerationConfig;
+import org.jsonschema2pojo.Jackson2Annotator;
+import org.jsonschema2pojo.SchemaGenerator;
+import org.jsonschema2pojo.SchemaMapper;
+import org.jsonschema2pojo.SchemaStore;
+import org.jsonschema2pojo.rules.RuleFactory;
+
+import com.sun.codemodel.JCodeModel;
+
+/**
+ * Helper class to re-generate java POJOs based on the JSON schema.
+ */
+public final class GeneratePojos {
+ @SuppressWarnings("checkstyle:linelength")
+ private static final String TARGET_PACKAGE =
+ "org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.placement.schema";
+
+ private GeneratePojos() {
+ // no instances
+ }
+
+ public static void main(String[] args) throws IOException {
+ JCodeModel codeModel = new JCodeModel();
+ URL schemaURL = Paths.get(
+ "src/main/json_schema/MappingRulesDescription.json").toUri().toURL();
+
+ GenerationConfig config = new DefaultGenerationConfig() {
+ @Override
+ public boolean isGenerateBuilders() {
+ return false;
+ }
+
+ @Override
+ public boolean isUsePrimitives() {
+ return true;
+ }
+ };
+
+ SchemaMapper mapper =
+ new SchemaMapper(
+ new RuleFactory(config,
+ new Jackson2Annotator(config),
+ new SchemaStore()),
+ new SchemaGenerator());
+
+ mapper.generate(codeModel, "ignore", TARGET_PACKAGE, schemaURL);
+
+ codeModel.build(new File("src/main/java"));
+ }
+}
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/placement/package-info.java b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/placement/package-info.java
new file mode 100644
index 0000000..766d1fd
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/placement/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Contains classes that are related to the newer
+ * version of CS placement engine.
+ */
+package org.apache.hadoop.yarn.server.resourcemanager.scheduler.capacity.placement;
\ No newline at end of file
diff --git a/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/json_schema/MappingRulesDescription.json b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/json_schema/MappingRulesDescription.json
new file mode 100644
index 0000000..152e64c
--- /dev/null
+++ b/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/json_schema/MappingRulesDescription.json
@@ -0,0 +1,36 @@
+{
+ "$schema": "http://json-schema.org/draft/2019-09/schema#",
+
+ "type" : "object",
+ "properties" : {
+ "rules" : {
+ "type" : "array",
+ "items": {
+ "type": "object",
+ "properties": {
+ "type": { "type": "string",
+ "enum": ["user", "group", "application"] },
+ "matches": { "type": "string" },
+ "policy": { "type": "string",
+ "enum": [
+ "specified",
+ "reject",
+ "defaultQueue",
+ "user",
+ "primaryGroup",
+ "secondaryGroup",
+ "primaryGroupUser",
+ "secondaryGroupUser",
+ "setDefaultQueue",
+ "custom" ] },
+ "parentQueue": { "type": "string" },
+ "fallbackResult": { "type": "string",
+ "enum": ["skip", "reject", "placeDefault"] },
+ "create": { "type": "boolean" },
+ "value": { "type": "string" },
+ "customPlacement": { "type" : "string" }
+ }
+ }
+ }
+ }
+}
\ No newline at end of file
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org