You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ozone.apache.org by pi...@apache.org on 2022/03/17 23:56:08 UTC
[ozone] branch master updated: HDDS-6213. Pluggable OzoneManager request handling hooks (#3104)
This is an automated email from the ASF dual-hosted git repository.
pifta pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/ozone.git
The following commit(s) were added to refs/heads/master by this push:
new 74d92c8 HDDS-6213. Pluggable OzoneManager request handling hooks (#3104)
74d92c8 is described below
commit 74d92c86be579e6d90535a13a276e1970ac644fc
Author: Istvan Fajth <pi...@cloudera.com>
AuthorDate: Fri Mar 18 00:55:35 2022 +0100
HDDS-6213. Pluggable OzoneManager request handling hooks (#3104)
---
dev-support/annotations/pom.xml | 114 +++++
.../RequestFeatureValidatorProcessor.java | 289 ++++++++++++
.../org/apache/ozone/annotations/package-info.java | 5 +
.../services/javax.annotation.processing.Processor | 24 +-
hadoop-hdds/interface-client/pom.xml | 5 +
hadoop-ozone/dev-support/checks/rat.sh | 4 +-
hadoop-ozone/dist/src/main/license/jar-report.txt | 1 +
hadoop-ozone/ozone-manager/pom.xml | 5 +
.../validation/RequestFeatureValidator.java | 99 ++++
.../request/validation/RequestProcessingPhase.java | 28 ++
.../om/request/validation/RequestValidations.java | 107 +++++
.../om/request/validation/ValidationCondition.java | 55 +++
.../om/request/validation/ValidationContext.java | 52 ++
.../om/request/validation/ValidatorRegistry.java | 201 ++++++++
.../ozone/om/request/validation/package-info.java | 62 +++
...OzoneManagerProtocolServerSideTranslatorPB.java | 23 +-
.../TestRequestFeatureValidatorProcessor.java | 524 +++++++++++++++++++++
.../request/validation/TestRequestValidations.java | 349 ++++++++++++++
.../request/validation/TestValidatorRegistry.java | 215 +++++++++
.../GeneralValidatorsForTesting.java | 190 ++++++++
.../ValidatorsForOnlyOldClientValidations.java | 43 ++
pom.xml | 26 +
22 files changed, 2393 insertions(+), 28 deletions(-)
diff --git a/dev-support/annotations/pom.xml b/dev-support/annotations/pom.xml
new file mode 100644
index 0000000..e654002
--- /dev/null
+++ b/dev-support/annotations/pom.xml
@@ -0,0 +1,114 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License. See accompanying LICENSE file.
+-->
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0
+https://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <modelVersion>4.0.0</modelVersion>
+
+ <groupId>org.apache.ozone</groupId>
+ <artifactId>ozone-annotation-processing</artifactId>
+ <version>1.3.0-SNAPSHOT</version>
+ <description>Apache Ozone annotation processing tools for validating custom
+ annotations at compile time.
+ </description>
+ <name>Apache Ozone Annotation Processing</name>
+ <packaging>jar</packaging>
+
+ <properties>
+ <project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ <project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
+ <spotbugs.version>3.1.12</spotbugs.version>
+ <maven-checkstyle-plugin.version>3.1.0</maven-checkstyle-plugin.version>
+ <checkstyle.version>8.29</checkstyle.version>
+ </properties>
+
+ <dependencies>
+ <dependency>
+ <groupId>com.github.spotbugs</groupId>
+ <artifactId>spotbugs</artifactId>
+ <version>${spotbugs.version}</version>
+ <scope>provided</scope>
+ </dependency>
+ </dependencies>
+
+ <build>
+ <plugins>
+ <!--
+ | As this pom is not a child of the main ozone pom.xml, if we want
+ | findbugs and checkstyle to be running for the module, the plugins
+ | have to be defined here.
+ -->
+ <plugin>
+ <groupId>com.github.spotbugs</groupId>
+ <artifactId>spotbugs-maven-plugin</artifactId>
+ <version>${spotbugs.version}</version>
+ <configuration>
+ <maxHeap>1024</maxHeap>
+ <xmlOutput>true</xmlOutput>
+ <includeTests>true</includeTests>
+ </configuration>
+ </plugin>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-checkstyle-plugin</artifactId>
+ <version>${maven-checkstyle-plugin.version}</version>
+ <dependencies>
+ <dependency>
+ <groupId>com.puppycrawl.tools</groupId>
+ <artifactId>checkstyle</artifactId>
+ <version>${checkstyle.version}</version>
+ </dependency>
+ </dependencies>
+ <configuration>
+ <configLocation>../../hadoop-hdds/dev-support/checkstyle/checkstyle.xml</configLocation>
+ <suppressionsLocation>../../hadoop-hdds/dev-support/checkstyle/suppressions.xml</suppressionsLocation>
+ <includeTestSourceDirectory>true</includeTestSourceDirectory>
+ <failOnViolation>false</failOnViolation>
+ <outputFile>${project.build.directory}/test/checkstyle-errors.xml</outputFile>
+ </configuration>
+ </plugin>
+ <!--
+ | Compiler plugin configuration have to be modified to compile the
+ | annotation processor, without having any annotation processor applied
+ | as due to the presence of
+ | META-INF/services/javac.annotation.processing.Processor the compilation
+ | would fail as the linked processor class can not be find while we are
+ | compiling it, hence the compiler argument -proc:none is specified here.
+ -->
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>3.9.0</version>
+ <configuration>
+ <source>1.8</source>
+ <target>1.8</target>
+ </configuration>
+ <executions>
+ <execution>
+ <id>default-compile</id>
+ <configuration>
+ <compilerArgument>-proc:none</compilerArgument>
+ <includes>
+ <include>org/apache/ozone/annotations/**</include>
+ </includes>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
+ </plugins>
+ </build>
+
+</project>
\ No newline at end of file
diff --git a/dev-support/annotations/src/main/java/org/apache/ozone/annotations/RequestFeatureValidatorProcessor.java b/dev-support/annotations/src/main/java/org/apache/ozone/annotations/RequestFeatureValidatorProcessor.java
new file mode 100644
index 0000000..830706e
--- /dev/null
+++ b/dev-support/annotations/src/main/java/org/apache/ozone/annotations/RequestFeatureValidatorProcessor.java
@@ -0,0 +1,289 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.ozone.annotations;
+
+import javax.annotation.processing.AbstractProcessor;
+import javax.annotation.processing.RoundEnvironment;
+import javax.annotation.processing.SupportedAnnotationTypes;
+import javax.annotation.processing.SupportedSourceVersion;
+import javax.lang.model.SourceVersion;
+import javax.lang.model.element.AnnotationMirror;
+import javax.lang.model.element.AnnotationValue;
+import javax.lang.model.element.AnnotationValueVisitor;
+import javax.lang.model.element.Element;
+import javax.lang.model.element.ElementKind;
+import javax.lang.model.element.ExecutableElement;
+import javax.lang.model.element.Modifier;
+import javax.lang.model.element.TypeElement;
+import javax.lang.model.element.VariableElement;
+import javax.lang.model.type.ExecutableType;
+import javax.lang.model.type.TypeMirror;
+import javax.lang.model.util.SimpleAnnotationValueVisitor8;
+import javax.tools.Diagnostic;
+import java.util.List;
+import java.util.Map.Entry;
+import java.util.Set;
+
+/**
+ * This class is an annotation processor that is hooked into the java compiler
+ * and is used to validate the RequestFeatureValidator annotations in the
+ * codebase, to ensure that the annotated methods have the proper signature and
+ * return type.
+ *
+ * The module is compiled in a different execution via Maven before anything
+ * else is compiled, and then javac picks this class up as an annotation
+ * processor from the classpath via a ServiceLoader, based on the
+ * META-INF/services/javax.annotation.processing.Processor file in the module's
+ * resources folder.
+ */
+@SupportedAnnotationTypes(
+ "org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator")
+@SupportedSourceVersion(SourceVersion.RELEASE_8)
+public class RequestFeatureValidatorProcessor extends AbstractProcessor {
+
+ public static final String ERROR_CONDITION_IS_EMPTY =
+ "RequestFeatureValidator has an empty condition list. Please define the"
+ + " ValidationCondition in which the validator has to be applied.";
+ public static final String ERROR_ANNOTATED_ELEMENT_IS_NOT_A_METHOD =
+ "RequestFeatureValidator annotation is not applied to a method.";
+ public static final String ERROR_VALIDATOR_METHOD_HAS_TO_BE_STATIC =
+ "Only static methods can be annotated with the RequestFeatureValidator"
+ + " annotation.";
+ public static final String ERROR_UNEXPECTED_PARAMETER_COUNT =
+ "Unexpected parameter count. Expected: %d; found: %d.";
+ public static final String ERROR_VALIDATOR_METHOD_HAS_TO_RETURN_OMREQUEST =
+ "Pre-processing validator methods annotated with RequestFeatureValidator"
+ + " annotation has to return an OMRequest object.";
+ public static final String ERROR_VALIDATOR_METHOD_HAS_TO_RETURN_OMRESPONSE =
+ "Post-processing validator methods annotated with RequestFeatureValidator"
+ + " annotation has to return an OMResponse object.";
+ public static final String ERROR_FIRST_PARAM_HAS_TO_BE_OMREQUEST =
+ "First parameter of a RequestFeatureValidator method has to be an"
+ + " OMRequest object.";
+ public static final String ERROR_LAST_PARAM_HAS_TO_BE_VALIDATION_CONTEXT =
+ "Last parameter of a RequestFeatureValidator method has to be"
+ + " ValidationContext object.";
+ public static final String ERROR_SECOND_PARAM_HAS_TO_BE_OMRESPONSE =
+ "Second parameter of a RequestFeatureValidator method has to be an"
+ + " OMResponse object.";
+
+ public static final String OM_REQUEST_CLASS_NAME =
+ "org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos"
+ + ".OMRequest";
+ public static final String OM_RESPONSE_CLASS_NAME =
+ "org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos"
+ + ".OMResponse";
+ public static final String VALIDATION_CONTEXT_CLASS_NAME =
+ "org.apache.hadoop.ozone.om.request.validation.ValidationContext";
+
+ public static final String ANNOTATION_SIMPLE_NAME = "RequestFeatureValidator";
+ public static final String ANNOTATION_CONDITIONS_PROPERTY_NAME = "conditions";
+ public static final String ANNOTATION_PROCESSING_PHASE_PROPERTY_NAME =
+ "processingPhase";
+
+ public static final String PROCESSING_PHASE_PRE_PROCESS = "PRE_PROCESS";
+ public static final String PROCESSING_PHASE_POST_PROCESS = "POST_PROCESS";
+ public static final String ERROR_NO_PROCESSING_PHASE_DEFINED =
+ "RequestFeatureValidator has an invalid ProcessingPhase defined.";
+
+ @Override
+ public boolean process(Set<? extends TypeElement> annotations,
+ RoundEnvironment roundEnv) {
+ for (TypeElement annotation : annotations) {
+ if (!annotation.getSimpleName().contentEquals(ANNOTATION_SIMPLE_NAME)) {
+ continue;
+ }
+ processElements(roundEnv.getElementsAnnotatedWith(annotation));
+ }
+ return false;
+ }
+
+ private void processElements(Set<? extends Element> annotatedElements) {
+ for (Element elem : annotatedElements) {
+ for (AnnotationMirror methodAnnotation : elem.getAnnotationMirrors()) {
+ validateAnnotatedMethod(elem, methodAnnotation);
+ }
+ }
+ }
+
+ private void validateAnnotatedMethod(
+ Element elem, AnnotationMirror methodAnnotation) {
+ boolean isPreprocessor = checkAndEvaluateAnnotation(methodAnnotation);
+
+ checkMethodIsAnnotated(elem);
+ ensureAnnotatedMethodIsStatic(elem);
+ ensurePreProcessorReturnsOMReqest((ExecutableElement) elem, isPreprocessor);
+ ensurePostProcessorReturnsOMResponse(
+ (ExecutableElement) elem, isPreprocessor);
+ ensureMethodParameters(elem, isPreprocessor);
+ }
+
+ private void ensureMethodParameters(Element elem, boolean isPreprocessor) {
+ List<? extends TypeMirror> paramTypes =
+ ((ExecutableType) elem.asType()).getParameterTypes();
+ ensureParameterCount(isPreprocessor, paramTypes);
+ ensureParameterRequirements(paramTypes, 0, OM_REQUEST_CLASS_NAME,
+ ERROR_FIRST_PARAM_HAS_TO_BE_OMREQUEST);
+ if (!isPreprocessor) {
+ ensureParameterRequirements(paramTypes, 1, OM_RESPONSE_CLASS_NAME,
+ ERROR_SECOND_PARAM_HAS_TO_BE_OMRESPONSE);
+ }
+ int contextOrder = isPreprocessor ? 1 : 2;
+ ensureParameterRequirements(paramTypes, contextOrder,
+ VALIDATION_CONTEXT_CLASS_NAME,
+ ERROR_LAST_PARAM_HAS_TO_BE_VALIDATION_CONTEXT);
+ }
+
+ private void ensureParameterCount(boolean isPreprocessor,
+ List<? extends TypeMirror> paramTypes) {
+ int realParamCount = paramTypes.size();
+ int expectedParamCount = isPreprocessor ? 2 : 3;
+ if (realParamCount != expectedParamCount) {
+ emitErrorMsg(String.format(ERROR_UNEXPECTED_PARAMETER_COUNT,
+ expectedParamCount, realParamCount));
+ }
+ }
+
+ private void ensureParameterRequirements(
+ List<? extends TypeMirror> paramTypes,
+ int index, String validationContextClassName,
+ String errorLastParamHasToBeValidationContext) {
+ if (paramTypes.size() >= index + 1 &&
+ !paramTypes.get(index).toString().equals(validationContextClassName)) {
+ emitErrorMsg(errorLastParamHasToBeValidationContext);
+ }
+ }
+
+ private void ensurePostProcessorReturnsOMResponse(
+ ExecutableElement elem, boolean isPreprocessor) {
+ if (!isPreprocessor && !elem.getReturnType().toString()
+ .equals(OM_RESPONSE_CLASS_NAME)) {
+ emitErrorMsg(ERROR_VALIDATOR_METHOD_HAS_TO_RETURN_OMRESPONSE);
+ }
+ }
+
+ private void ensurePreProcessorReturnsOMReqest(
+ ExecutableElement elem, boolean isPreprocessor) {
+ if (isPreprocessor && !elem.getReturnType().toString()
+ .equals(OM_REQUEST_CLASS_NAME)) {
+ emitErrorMsg(ERROR_VALIDATOR_METHOD_HAS_TO_RETURN_OMREQUEST);
+ }
+ }
+
+ private void ensureAnnotatedMethodIsStatic(Element elem) {
+ if (!elem.getModifiers().contains(Modifier.STATIC)) {
+ emitErrorMsg(ERROR_VALIDATOR_METHOD_HAS_TO_BE_STATIC);
+ }
+ }
+
+ private void checkMethodIsAnnotated(Element elem) {
+ if (elem.getKind() != ElementKind.METHOD) {
+ emitErrorMsg(ERROR_ANNOTATED_ELEMENT_IS_NOT_A_METHOD);
+ }
+ }
+
+ private boolean checkAndEvaluateAnnotation(
+ AnnotationMirror methodAnnotation) {
+ boolean isPreprocessor = false;
+ for (Entry<? extends ExecutableElement, ? extends AnnotationValue>
+ entry : methodAnnotation.getElementValues().entrySet()) {
+
+ if (hasInvalidValidationCondition(entry)) {
+ emitErrorMsg(ERROR_CONDITION_IS_EMPTY);
+ }
+ if (isProcessingPhaseValue(entry)) {
+ isPreprocessor = evaluateProcessingPhase(entry);
+ }
+ }
+ return isPreprocessor;
+ }
+
+ private boolean evaluateProcessingPhase(
+ Entry<? extends ExecutableElement, ? extends AnnotationValue> entry) {
+ String procPhase = visit(entry, new ProcessingPhaseVisitor());
+ if (procPhase.equals(PROCESSING_PHASE_PRE_PROCESS)) {
+ return true;
+ } else if (procPhase.equals(PROCESSING_PHASE_POST_PROCESS)) {
+ return false;
+ }
+ return false;
+ }
+
+ private boolean isProcessingPhaseValue(
+ Entry<? extends ExecutableElement, ? extends AnnotationValue> entry) {
+ return isPropertyNamedAs(entry, ANNOTATION_PROCESSING_PHASE_PROPERTY_NAME);
+ }
+
+ private boolean hasInvalidValidationCondition(
+ Entry<? extends ExecutableElement, ? extends AnnotationValue> entry) {
+ return isPropertyNamedAs(entry, ANNOTATION_CONDITIONS_PROPERTY_NAME)
+ && !visit(entry, new ConditionValidator());
+ }
+
+ private boolean isPropertyNamedAs(
+ Entry<? extends ExecutableElement, ? extends AnnotationValue> entry,
+ String simpleName) {
+ return entry.getKey().getSimpleName().contentEquals(simpleName);
+ }
+
+ private <T> T visit(
+ Entry<? extends ExecutableElement, ? extends AnnotationValue> entry,
+ AnnotationValueVisitor<T, Void> visitor) {
+ return entry.getValue().accept(visitor, null);
+ }
+
+ private static class ConditionValidator
+ extends SimpleAnnotationValueVisitor8<Boolean, Void> {
+
+ ConditionValidator() {
+ super(Boolean.TRUE);
+ }
+
+ @Override
+ public Boolean visitArray(List<? extends AnnotationValue> vals,
+ Void unused) {
+ if (vals.isEmpty()) {
+ return Boolean.FALSE;
+ }
+ return Boolean.TRUE;
+ }
+
+ }
+
+ private static class ProcessingPhaseVisitor
+ extends SimpleAnnotationValueVisitor8<String, Void> {
+
+ ProcessingPhaseVisitor() {
+ super("UNKNOWN");
+ }
+
+ @Override
+ public String visitEnumConstant(VariableElement c, Void unused) {
+ if (c.getSimpleName().contentEquals(PROCESSING_PHASE_PRE_PROCESS)) {
+ return PROCESSING_PHASE_PRE_PROCESS;
+ }
+ if (c.getSimpleName().contentEquals(PROCESSING_PHASE_POST_PROCESS)) {
+ return PROCESSING_PHASE_POST_PROCESS;
+ }
+ throw new IllegalStateException(ERROR_NO_PROCESSING_PHASE_DEFINED);
+ }
+ }
+
+ private void emitErrorMsg(String s) {
+ processingEnv.getMessager().printMessage(Diagnostic.Kind.ERROR, s);
+ }
+}
diff --git a/dev-support/annotations/src/main/java/org/apache/ozone/annotations/package-info.java b/dev-support/annotations/src/main/java/org/apache/ozone/annotations/package-info.java
new file mode 100644
index 0000000..b13720d
--- /dev/null
+++ b/dev-support/annotations/src/main/java/org/apache/ozone/annotations/package-info.java
@@ -0,0 +1,5 @@
+/**
+ * Annotation processors used at compile time by the Ozone project to validate
+ * internal annotations and related code as needed, if needed.
+ */
+package org.apache.ozone.annotations;
\ No newline at end of file
diff --git a/hadoop-ozone/dev-support/checks/rat.sh b/dev-support/annotations/src/main/resources/META-INF/services/javax.annotation.processing.Processor
old mode 100755
new mode 100644
similarity index 55%
copy from hadoop-ozone/dev-support/checks/rat.sh
copy to dev-support/annotations/src/main/resources/META-INF/services/javax.annotation.processing.Processor
index 464d636..35b0c1e
--- a/hadoop-ozone/dev-support/checks/rat.sh
+++ b/dev-support/annotations/src/main/resources/META-INF/services/javax.annotation.processing.Processor
@@ -1,4 +1,3 @@
-#!/usr/bin/env bash
# Licensed to the Apache Software Foundation (ASF) under one or more
# contributor license agreements. See the NOTICE file distributed with
# this work for additional information regarding copyright ownership.
@@ -13,26 +12,5 @@
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
-DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
-cd "$DIR/../../.." || exit 1
-
-REPORT_DIR=${OUTPUT_DIR:-"$DIR/../../../target/rat"}
-mkdir -p "$REPORT_DIR"
-
-REPORT_FILE="$REPORT_DIR/summary.txt"
-
-cd hadoop-hdds || exit 1
-mvn -B -fn org.apache.rat:apache-rat-plugin:0.13:check
-cd ../hadoop-ozone || exit 1
-mvn -B -fn org.apache.rat:apache-rat-plugin:0.13:check
-
-cd "$DIR/../../.." || exit 1
-
-grep -r --include=rat.txt "!????" hadoop-hdds hadoop-ozone | tee "$REPORT_FILE"
-
-wc -l "$REPORT_FILE" | awk '{print $1}'> "$REPORT_DIR/failures"
-
-if [[ -s "${REPORT_FILE}" ]]; then
- exit 1
-fi
+org.apache.ozone.annotations.RequestFeatureValidatorProcessor
\ No newline at end of file
diff --git a/hadoop-hdds/interface-client/pom.xml b/hadoop-hdds/interface-client/pom.xml
index 4b97b45..eed1807 100644
--- a/hadoop-hdds/interface-client/pom.xml
+++ b/hadoop-hdds/interface-client/pom.xml
@@ -47,6 +47,11 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
<groupId>javax.annotation</groupId>
<artifactId>javax.annotation-api</artifactId>
</dependency>
+ <dependency>
+ <groupId>com.google.code.findbugs</groupId>
+ <artifactId>jsr305</artifactId>
+ <scope>compile</scope>
+ </dependency>
</dependencies>
<build>
<plugins>
diff --git a/hadoop-ozone/dev-support/checks/rat.sh b/hadoop-ozone/dev-support/checks/rat.sh
index 464d636..b67c594 100755
--- a/hadoop-ozone/dev-support/checks/rat.sh
+++ b/hadoop-ozone/dev-support/checks/rat.sh
@@ -21,7 +21,9 @@ mkdir -p "$REPORT_DIR"
REPORT_FILE="$REPORT_DIR/summary.txt"
-cd hadoop-hdds || exit 1
+cd dev-support/annotations || exit 1
+mvn -B -fn org.apache.rat:apache-rat-plugin:0.13:check
+cd ../../hadoop-hdds || exit 1
mvn -B -fn org.apache.rat:apache-rat-plugin:0.13:check
cd ../hadoop-ozone || exit 1
mvn -B -fn org.apache.rat:apache-rat-plugin:0.13:check
diff --git a/hadoop-ozone/dist/src/main/license/jar-report.txt b/hadoop-ozone/dist/src/main/license/jar-report.txt
index ec949fd..da8350d 100644
--- a/hadoop-ozone/dist/src/main/license/jar-report.txt
+++ b/hadoop-ozone/dist/src/main/license/jar-report.txt
@@ -183,6 +183,7 @@ share/ozone/lib/opentracing-noop.jar
share/ozone/lib/opentracing-tracerresolver.jar
share/ozone/lib/opentracing-util.jar
share/ozone/lib/osgi-resource-locator.jar
+share/ozone/lib/ozone-annotation-processing.jar
share/ozone/lib/ozone-client.jar
share/ozone/lib/ozone-common.jar
share/ozone/lib/ozone-csi.jar
diff --git a/hadoop-ozone/ozone-manager/pom.xml b/hadoop-ozone/ozone-manager/pom.xml
index cea140b..5e42414 100644
--- a/hadoop-ozone/ozone-manager/pom.xml
+++ b/hadoop-ozone/ozone-manager/pom.xml
@@ -134,6 +134,11 @@ https://maven.apache.org/xsd/maven-4.0.0.xsd">
</dependency>
<dependency>
+ <groupId>com.google.testing.compile</groupId>
+ <artifactId>compile-testing</artifactId>
+ </dependency>
+
+ <dependency>
<groupId>org.reflections</groupId>
<artifactId>reflections</artifactId>
<version>0.9.11</version>
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/RequestFeatureValidator.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/RequestFeatureValidator.java
new file mode 100644
index 0000000..5f1f082
--- /dev/null
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/RequestFeatureValidator.java
@@ -0,0 +1,99 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.ozone.om.request.validation;
+
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type;
+
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Retention;
+import java.lang.annotation.RetentionPolicy;
+import java.lang.annotation.Target;
+
+/**
+ * An annotation to mark methods that do certain request validations.
+ *
+ * The methods annotated with this annotation are collected by the
+ * {@link ValidatorRegistry} class during the initialization of the server.
+ *
+ * The conditions specify the specific use case in which the validator should be
+ * applied to the request. See {@link ValidationCondition} for more details
+ * on the specific conditions.
+ * The validator method should be applied to just one specific request type
+ * to help keep these methods simple and straightforward. If you want to use
+ * the same validation for different request types, use inheritance, and
+ * annotate the override method that just calls super.
+ * Note that the aim is to have these validators together with the request
+ * processing code, so the handling of these specific situations are easy to
+ * find.
+ *
+ * The annotated methods have to have a fixed signature.
+ * A {@link RequestProcessingPhase#PRE_PROCESS} phase method is running before
+ * the request is processed by the regular code.
+ * Its signature has to be the following:
+ * - it has to be static and idempotent
+ * - it has to have two parameters
+ * - the first parameter it is an
+ * {@link
+ * org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest}
+ * - the second parameter of type {@link ValidationContext}
+ * - the method has to return the modified request, or throw a ServiceException
+ * in case the request is considered to be invalid
+ * - the method does not need to care about preserving the request it gets,
+ * the original request is captured and saved by the calling environment.
+ *
+ * A {@link RequestProcessingPhase#POST_PROCESS} phase method is running once
+ * the
+ * {@link
+ * org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse}
+ * is calculated for a given request.
+ * Its signature has to be the following:
+ * - it has to be static and idempotent
+ * - it has three parameters
+ * - similalry to the pre-processing validators, first parameter is the
+ * OMRequest, the second parameter is the OMResponse, and the third
+ * parameter is a ValidationContext.
+ * - the method has to return the modified OMResponse or throw a
+ * ServiceException if the request is considered invalid based on response.
+ * - the method gets the request object that was supplied for the general
+ * request processing code, not the original request, while it gets a copy
+ * of the original response object provided by the general request processing
+ * code.
+ */
+@Retention(RetentionPolicy.RUNTIME)
+@Target(ElementType.METHOD)
+public @interface RequestFeatureValidator {
+
+ /**
+ * Runtime conditions in which a validator should run.
+ * @return a list of conditions when the validator should be applied
+ */
+ ValidationCondition[] conditions();
+
+ /**
+ * Defines if the validation has to run before or after the general request
+ * processing.
+ * @return if this is a pre or post processing validator
+ */
+ RequestProcessingPhase processingPhase();
+
+ /**
+ * The type of the request handled by this validator method.
+ * @return the requestType to whihc the validator shoudl be applied
+ */
+ Type requestType();
+
+}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/RequestProcessingPhase.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/RequestProcessingPhase.java
new file mode 100644
index 0000000..6721568
--- /dev/null
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/RequestProcessingPhase.java
@@ -0,0 +1,28 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.ozone.om.request.validation;
+
+/**
+ * Processing phase defines when a request validator should run.
+ *
+ * There are two hooking point at the moment, before and after the generic
+ * request processing code.
+ */
+public enum RequestProcessingPhase {
+ PRE_PROCESS,
+ POST_PROCESS
+}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/RequestValidations.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/RequestValidations.java
new file mode 100644
index 0000000..fe34b74
--- /dev/null
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/RequestValidations.java
@@ -0,0 +1,107 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.ozone.om.request.validation;
+
+import com.google.protobuf.ServiceException;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+import java.util.Arrays;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import static org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase.POST_PROCESS;
+import static org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase.PRE_PROCESS;
+
+/**
+ * Main class to configure and set up and access the request/response
+ * validation framework.
+ */
+public class RequestValidations {
+
+ static final Logger LOG = LoggerFactory.getLogger(RequestValidations.class);
+ private static final String DEFAULT_PACKAGE = "org.apache.hadoop.ozone";
+
+ private String validationsPackageName = DEFAULT_PACKAGE;
+ private ValidationContext context = null;
+ private ValidatorRegistry registry = null;
+
+ public synchronized RequestValidations fromPackage(String packageName) {
+ validationsPackageName = packageName;
+ return this;
+ }
+
+ public RequestValidations withinContext(ValidationContext validationContext) {
+ this.context = validationContext;
+ return this;
+ }
+
+ public synchronized RequestValidations load() {
+ registry = new ValidatorRegistry(validationsPackageName);
+ return this;
+ }
+
+ public OMRequest validateRequest(OMRequest request) throws ServiceException {
+ List<Method> validations = registry.validationsFor(
+ conditions(request), request.getCmdType(), PRE_PROCESS);
+
+ OMRequest validatedRequest = request.toBuilder().build();
+ try {
+ for (Method m : validations) {
+ validatedRequest =
+ (OMRequest) m.invoke(null, validatedRequest, context);
+ LOG.debug("Running the {} request pre-process validation from {}.{}",
+ m.getName(), m.getDeclaringClass().getPackage().getName(),
+ m.getDeclaringClass().getSimpleName());
+ }
+ } catch (IllegalAccessException | InvocationTargetException e) {
+ throw new ServiceException(e);
+ }
+ return validatedRequest;
+ }
+
+ public OMResponse validateResponse(OMRequest request, OMResponse response)
+ throws ServiceException {
+ List<Method> validations = registry.validationsFor(
+ conditions(request), request.getCmdType(), POST_PROCESS);
+
+ OMResponse validatedResponse = response.toBuilder().build();
+ try {
+ for (Method m : validations) {
+ validatedResponse =
+ (OMResponse) m.invoke(null, request, response, context);
+ LOG.debug("Running the {} request post-process validation from {}.{}",
+ m.getName(), m.getDeclaringClass().getPackage().getName(),
+ m.getDeclaringClass().getSimpleName());
+ }
+ } catch (InvocationTargetException | IllegalAccessException e) {
+ throw new ServiceException(e);
+ }
+ return validatedResponse;
+ }
+
+ private List<ValidationCondition> conditions(OMRequest request) {
+ return Arrays.stream(ValidationCondition.values())
+ .filter(c -> c.shouldApply(request, context))
+ .collect(Collectors.toList());
+ }
+
+}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/ValidationCondition.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/ValidationCondition.java
new file mode 100644
index 0000000..9630500
--- /dev/null
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/ValidationCondition.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.ozone.om.request.validation;
+
+import org.apache.hadoop.ozone.ClientVersion;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest;
+
+/**
+ * Defines conditions for which validators can be assigned to.
+ *
+ * These conditions describe a situation where special request handling might
+ * be necessary. In these cases we do not override the actual request handling
+ * code, but based on certain request properties we might reject a request
+ * early, or we might modify the request, or the response received/sent from/to
+ * the client.
+ */
+public enum ValidationCondition {
+ /**
+ * Classifies validations that has to run after an upgrade until the cluster
+ * is in a pre-finalized state.
+ */
+ CLUSTER_NEEDS_FINALIZATION {
+ @Override
+ public boolean shouldApply(OMRequest req, ValidationContext ctx) {
+ return ctx.versionManager().needsFinalization();
+ }
+ },
+
+ /**
+ * Classifies validations that has to run, when the client uses an older
+ * protocol version than the server.
+ */
+ OLDER_CLIENT_REQUESTS {
+ @Override
+ public boolean shouldApply(OMRequest req, ValidationContext ctx) {
+ return req.getVersion() < ClientVersion.CURRENT_VERSION;
+ }
+ };
+
+ public abstract boolean shouldApply(OMRequest req, ValidationContext ctx);
+}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/ValidationContext.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/ValidationContext.java
new file mode 100644
index 0000000..510d433
--- /dev/null
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/ValidationContext.java
@@ -0,0 +1,52 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.ozone.om.request.validation;
+
+import org.apache.hadoop.hdds.annotation.InterfaceStability;
+import org.apache.hadoop.ozone.upgrade.LayoutVersionManager;
+
+/**
+ * A context that contains useful information for request validator instances.
+ */
+@InterfaceStability.Evolving
+public interface ValidationContext {
+
+ /**
+ * Gets the {@link LayoutVersionManager} of the service, so that a pre
+ * finalization validation can check if the layout version it belongs to
+ * is finalized already or not.
+ *
+ * @return the {@link LayoutVersionManager} of the service
+ */
+ LayoutVersionManager versionManager();
+
+ /**
+ * Creates a context object based on the given parameters.
+ *
+ * @param versionManager the {@link LayoutVersionManager} of the service
+ * @return the {@link ValidationContext} specified by the parameters.
+ */
+ static ValidationContext of(LayoutVersionManager versionManager) {
+
+ return new ValidationContext() {
+ @Override
+ public LayoutVersionManager versionManager() {
+ return versionManager;
+ }
+ };
+ }
+}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/ValidatorRegistry.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/ValidatorRegistry.java
new file mode 100644
index 0000000..72bd0bb
--- /dev/null
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/ValidatorRegistry.java
@@ -0,0 +1,201 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.ozone.om.request.validation;
+
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type;
+import org.reflections.Reflections;
+import org.reflections.scanners.MethodAnnotationsScanner;
+import org.reflections.util.ClasspathHelper;
+import org.reflections.util.ConfigurationBuilder;
+
+import java.lang.reflect.Method;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.EnumMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+
+import static org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase.POST_PROCESS;
+import static org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase.PRE_PROCESS;
+
+/**
+ * Registry that loads and stores the request validators to be applied by
+ * a service.
+ */
+public class ValidatorRegistry {
+
+ private final EnumMap<ValidationCondition,
+ EnumMap<Type, EnumMap<RequestProcessingPhase, List<Method>>>>
+ validators = new EnumMap<>(ValidationCondition.class);
+
+ /**
+ * Creates a {@link ValidatorRegistry} instance that discovers validation
+ * methods in the provided package and the packages in the same resource.
+ * A validation method is recognized by the {@link RequestFeatureValidator}
+ * annotation that contains important information about how and when to use
+ * the validator.
+ * @param validatorPackage the main package inside which validatiors should
+ * be discovered.
+ */
+ ValidatorRegistry(String validatorPackage) {
+ this(ClasspathHelper.forPackage(validatorPackage));
+ }
+
+ /**
+ * Creates a {@link ValidatorRegistry} instance that discovers validation
+ * methods under the provided URL.
+ * A validation method is recognized by the {@link RequestFeatureValidator}
+ * annotation that contains important information about how and when to use
+ * the validator.
+ * @param searchUrls the path in which the annotated methods are searched.
+ */
+ ValidatorRegistry(Collection<URL> searchUrls) {
+ Reflections reflections = new Reflections(new ConfigurationBuilder()
+ .setUrls(searchUrls)
+ .setScanners(new MethodAnnotationsScanner())
+ .useParallelExecutor()
+ );
+
+ Set<Method> describedValidators =
+ reflections.getMethodsAnnotatedWith(RequestFeatureValidator.class);
+ initMaps(describedValidators);
+ }
+
+ /**
+ * Get the validators that has to be run in the given list of
+ * {@link ValidationCondition}s, for the given requestType and
+ * {@link RequestProcessingPhase}.
+ *
+ * @param conditions conditions that are present for the request
+ * @param requestType the type of the protocol message
+ * @param phase the request processing phase
+ * @return the list of validation methods that has to run.
+ */
+ List<Method> validationsFor(
+ List<ValidationCondition> conditions,
+ Type requestType,
+ RequestProcessingPhase phase) {
+
+ if (conditions.isEmpty() || validators.isEmpty()) {
+ return Collections.emptyList();
+ }
+
+ Set<Method> returnValue =
+ new HashSet<>(validationsFor(conditions.get(0), requestType, phase));
+
+ for (int i = 1; i < conditions.size(); i++) {
+ returnValue.addAll(validationsFor(conditions.get(i), requestType, phase));
+ }
+ return new ArrayList<>(returnValue);
+ }
+
+ /**
+ * Grabs validations for one particular condition.
+ *
+ * @param condition conditions that are present for the request
+ * @param requestType the type of the protocol message
+ * @param phase the request processing phase
+ * @return the list of validation methods that has to run.
+ */
+ private List<Method> validationsFor(
+ ValidationCondition condition,
+ Type requestType,
+ RequestProcessingPhase phase) {
+
+ EnumMap<Type, EnumMap<RequestProcessingPhase, List<Method>>>
+ requestTypeMap = validators.get(condition);
+ if (requestTypeMap == null || requestTypeMap.isEmpty()) {
+ return Collections.emptyList();
+ }
+
+ EnumMap<RequestProcessingPhase, List<Method>> phases =
+ requestTypeMap.get(requestType);
+ if (phases == null) {
+ return Collections.emptyList();
+ }
+
+ List<Method> validatorsForPhase = phases.get(phase);
+ if (validatorsForPhase == null) {
+ return Collections.emptyList();
+ }
+ return validatorsForPhase;
+ }
+
+ /**
+ * Initializes the internal request validator store.
+ * The requests are stored in the following structure:
+ * - An EnumMap with the {@link ValidationCondition} as the key, and in which
+ * - values are an EnumMap with the request type as the key, and in which
+ * - values are Pair of lists, in which
+ * - left side is the pre-processing validations list
+ * - right side is the post-processing validations list
+ * @param describedValidators collection of the annotated methods to process.
+ */
+ void initMaps(Collection<Method> describedValidators) {
+ for (Method m : describedValidators) {
+ RequestFeatureValidator descriptor =
+ m.getAnnotation(RequestFeatureValidator.class);
+ m.setAccessible(true);
+
+ for (ValidationCondition condition : descriptor.conditions()) {
+ EnumMap<Type, EnumMap<RequestProcessingPhase, List<Method>>>
+ requestTypeMap = getAndInitialize(
+ condition, newTypeMap(), validators);
+ EnumMap<RequestProcessingPhase, List<Method>> phases = getAndInitialize(
+ descriptor.requestType(), newPhaseMap(), requestTypeMap);
+ if (isPreProcessValidator(descriptor)) {
+ getAndInitialize(PRE_PROCESS, new ArrayList<>(), phases).add(m);
+ } else if (isPostProcessValidator(descriptor)) {
+ getAndInitialize(POST_PROCESS, new ArrayList<>(), phases).add(m);
+ }
+ }
+ }
+ }
+
+ private EnumMap<Type,
+ EnumMap<RequestProcessingPhase, List<Method>>> newTypeMap() {
+ return new EnumMap<>(Type.class);
+ }
+
+ private EnumMap<RequestProcessingPhase, List<Method>> newPhaseMap() {
+ return new EnumMap<>(RequestProcessingPhase.class);
+ }
+
+ private <K, V> V getAndInitialize(K key, V defaultValue, Map<K, V> from) {
+ V inMapValue = from.get(key);
+ if (inMapValue == null || !from.containsKey(key)) {
+ from.put(key, defaultValue);
+ return defaultValue;
+ }
+ return inMapValue;
+ }
+
+ private boolean isPreProcessValidator(RequestFeatureValidator descriptor) {
+ return descriptor.processingPhase()
+ .equals(PRE_PROCESS);
+ }
+
+ private boolean isPostProcessValidator(RequestFeatureValidator descriptor) {
+ return descriptor.processingPhase()
+ .equals(POST_PROCESS);
+ }
+
+}
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/package-info.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/package-info.java
new file mode 100644
index 0000000..e82bab7
--- /dev/null
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/om/request/validation/package-info.java
@@ -0,0 +1,62 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+
+/**
+ * Request's feature validation handling.
+ *
+ * This package holds facilities to add new situation specific behaviour to
+ * request handling without cluttering the basic logic of the request handler
+ * code.
+ *
+ * Typical use case scenarios, that we had in mind during the design:
+ * - during an upgrade, in the pre-finalized state certain request types are
+ * to be rejected based on provided properties of the request not based on the
+ * request type
+ * - a client connects to the server but uses an older version of the protocol
+ * - a client connects to the server but uses a newer version of the protocol
+ * - the code can handle certain checks that have to run all the time, but at
+ * first we do not see a general use case that we would pull in immediately.
+ * These are the current
+ * {@link org.apache.hadoop.ozone.om.request.validation.ValidationCondition}s
+ * but this list might be extended later on if we see other use cases.
+ *
+ * The system uses a reflection based discovery to find methods that are
+ * annotated with the
+ * {@link org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator}
+ * annotation.
+ * This annotation is used to specify the condition in which a certain validator
+ * has to be used, the request type to which the validation should be applied,
+ * and the request processing phase in which we apply the validation.
+ *
+ * One validator can be applied in multiple
+ * {@link org.apache.hadoop.ozone.om.request.validation.ValidationCondition}
+ * but a validator has to handle strictly just one
+ * {@link org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type
+ * }.
+ * The main reason to avoid validating multiple request types with the same
+ * validator, is that these validators have to be simple methods without state
+ * any complex validation has to happen in the reql request handling.
+ * In these validators we need to ensure that in the given condition the request
+ * is rejected with a proper message, or rewritten to the proper format if for
+ * example we want to handle an old request with a new server, but we need some
+ * additional values set to something default, while in the meantime we want to
+ * add meaning to a null value from newer clients.
+ *
+ * In general, it is a good practice to have the request handling code, and the
+ * validations tied together in one class.
+ */
+package org.apache.hadoop.ozone.om.request.validation;
\ No newline at end of file
diff --git a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerProtocolServerSideTranslatorPB.java b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerProtocolServerSideTranslatorPB.java
index ef82069..06aa6c5 100644
--- a/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerProtocolServerSideTranslatorPB.java
+++ b/hadoop-ozone/ozone-manager/src/main/java/org/apache/hadoop/ozone/protocolPB/OzoneManagerProtocolServerSideTranslatorPB.java
@@ -38,6 +38,8 @@ import org.apache.hadoop.ozone.om.ratis.OzoneManagerRatisServer;
import org.apache.hadoop.ozone.om.ratis.OzoneManagerRatisServer.RaftServerStatus;
import org.apache.hadoop.ozone.om.ratis.utils.OzoneManagerRatisUtils;
import org.apache.hadoop.ozone.om.request.OMClientRequest;
+import org.apache.hadoop.ozone.om.request.validation.RequestValidations;
+import org.apache.hadoop.ozone.om.request.validation.ValidationContext;
import org.apache.hadoop.ozone.om.response.OMClientResponse;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest;
import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse;
@@ -60,6 +62,9 @@ public class OzoneManagerProtocolServerSideTranslatorPB implements
OzoneManagerProtocolPB {
private static final Logger LOG = LoggerFactory
.getLogger(OzoneManagerProtocolServerSideTranslatorPB.class);
+ private static final String OM_REQUESTS_PACKAGE =
+ "org.apache.hadoop.ozone";
+
private final OzoneManagerRatisServer omRatisServer;
private final RequestHandler handler;
private final boolean isRatisEnabled;
@@ -68,6 +73,7 @@ public class OzoneManagerProtocolServerSideTranslatorPB implements
private final AtomicLong transactionIndex;
private final OzoneProtocolMessageDispatcher<OMRequest, OMResponse,
ProtocolMessageEnum> dispatcher;
+ private final RequestValidations requestValidations;
/**
* Constructs an instance of the server handler.
@@ -110,19 +116,28 @@ public class OzoneManagerProtocolServerSideTranslatorPB implements
this.omRatisServer = ratisServer;
dispatcher = new OzoneProtocolMessageDispatcher<>("OzoneProtocol",
metrics, LOG, OMPBHelper::processForDebug, OMPBHelper::processForDebug);
+ // TODO: make this injectable for testing...
+ requestValidations =
+ new RequestValidations()
+ .fromPackage(OM_REQUESTS_PACKAGE)
+ .withinContext(
+ ValidationContext.of(ozoneManager.getVersionManager()))
+ .load();
}
/**
- * Submit requests to Ratis server for OM HA implementation.
- * TODO: Once HA is implemented fully, we should have only one server side
- * translator for OM protocol.
+ * Submit mutating requests to Ratis server in OM, and process read requests.
*/
@Override
public OMResponse submitRequest(RpcController controller,
OMRequest request) throws ServiceException {
+ OMRequest validatedRequest = requestValidations.validateRequest(request);
- return dispatcher.processRequest(request, this::processRequest,
+ OMResponse response =
+ dispatcher.processRequest(validatedRequest, this::processRequest,
request.getCmdType(), request.getTraceID());
+
+ return requestValidations.validateResponse(request, response);
}
private OMResponse processRequest(OMRequest request) throws
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestRequestFeatureValidatorProcessor.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestRequestFeatureValidatorProcessor.java
new file mode 100644
index 0000000..57b369d
--- /dev/null
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestRequestFeatureValidatorProcessor.java
@@ -0,0 +1,524 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.ozone.om.request.validation;
+
+import com.google.testing.compile.Compilation;
+import com.google.testing.compile.JavaFileObjects;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type;
+import org.apache.ozone.annotations.RequestFeatureValidatorProcessor;
+import org.junit.Test;
+
+import java.lang.annotation.Annotation;
+import java.lang.annotation.ElementType;
+import java.lang.annotation.Target;
+import java.util.ArrayList;
+import java.util.List;
+
+import static com.google.testing.compile.CompilationSubject.assertThat;
+import static com.google.testing.compile.Compiler.javac;
+import static org.apache.ozone.annotations.RequestFeatureValidatorProcessor.ERROR_CONDITION_IS_EMPTY;
+import static org.apache.ozone.annotations.RequestFeatureValidatorProcessor.ERROR_FIRST_PARAM_HAS_TO_BE_OMREQUEST;
+import static org.apache.ozone.annotations.RequestFeatureValidatorProcessor.ERROR_LAST_PARAM_HAS_TO_BE_VALIDATION_CONTEXT;
+import static org.apache.ozone.annotations.RequestFeatureValidatorProcessor.ERROR_SECOND_PARAM_HAS_TO_BE_OMRESPONSE;
+import static org.apache.ozone.annotations.RequestFeatureValidatorProcessor.ERROR_UNEXPECTED_PARAMETER_COUNT;
+import static org.apache.ozone.annotations.RequestFeatureValidatorProcessor.ERROR_VALIDATOR_METHOD_HAS_TO_BE_STATIC;
+import static org.apache.ozone.annotations.RequestFeatureValidatorProcessor.ERROR_VALIDATOR_METHOD_HAS_TO_RETURN_OMREQUEST;
+import static org.apache.ozone.annotations.RequestFeatureValidatorProcessor.ERROR_VALIDATOR_METHOD_HAS_TO_RETURN_OMRESPONSE;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertSame;
+
+/**
+ * Compile tests against the annotation processor for the
+ * {@link RequestFeatureValidator} annotation.
+ *
+ * The processor should ensure the method signatures and return values, based
+ * on annotation arguments provided.
+ */
+public class TestRequestFeatureValidatorProcessor {
+
+ private static final String CLASSNAME = "Validation";
+
+ @Test
+ public void testAnnotationCanOnlyBeAppliedOnMethods() {
+ Class<RequestFeatureValidator> c = RequestFeatureValidator.class;
+ for (Annotation a : c.getAnnotations()) {
+ if (a instanceof Target) {
+ assertEquals(1, ((Target) a).value().length);
+ assertSame(((Target) a).value()[0], ElementType.METHOD);
+ }
+ }
+ }
+
+ @Test
+ public void testACorrectAnnotationSetupForPreProcessCompiles() {
+ List<String> source = generateSourceOfValidatorMethodWith(
+ annotationOf(someConditions(), preProcess(), aReqType()),
+ modifiers("public", "static"),
+ returnValue("OMRequest"),
+ parameters("OMRequest rq", "ValidationContext ctx"),
+ exceptions("ServiceException"));
+
+ assertThat(compile(source)).succeeded();
+ }
+
+ @Test
+ public void testACorrectAnnotationSetupForPostProcessCompiles() {
+ List<String> source = generateSourceOfValidatorMethodWith(
+ annotationOf(someConditions(), postProcess(), aReqType()),
+ modifiers("public", "static"),
+ returnValue("OMResponse"),
+ parameters("OMRequest rq", "OMResponse rp", "ValidationContext ctx"),
+ exceptions("ServiceException"));
+
+ assertThat(compile(source)).succeeded();
+ }
+
+ @Test
+ public void testValidatorDoesNotNecessarilyThrowsExceptions() {
+ List<String> source = generateSourceOfValidatorMethodWith(
+ annotationOf(someConditions(), preProcess(), aReqType()),
+ modifiers("public", "static"),
+ returnValue("OMRequest"),
+ parameters("OMRequest rq", "ValidationContext ctx"),
+ exceptions());
+
+ assertThat(compile(source)).succeeded();
+ }
+
+ @Test
+ public void testNonStaticValidatorDoesNotCompile() {
+ List<String> source = generateSourceOfValidatorMethodWith(
+ annotationOf(someConditions(), preProcess(), aReqType()),
+ modifiers("public"),
+ returnValue("OMRequest"),
+ parameters("OMRequest rq", "ValidationContext ctx"),
+ exceptions());
+
+ assertThat(compile(source))
+ .hadErrorContaining(ERROR_VALIDATOR_METHOD_HAS_TO_BE_STATIC);
+ }
+
+ @Test
+ public void testValidatorMethodCanBeFinal() {
+ List<String> source = generateSourceOfValidatorMethodWith(
+ annotationOf(someConditions(), preProcess(), aReqType()),
+ modifiers("public", "static", "final"),
+ returnValue("OMRequest"),
+ parameters("OMRequest rq", "ValidationContext ctx"),
+ exceptions());
+
+ assertThat(compile(source)).succeeded();
+ }
+
+ @Test
+ public void testValidatorMethodCanBePrivate() {
+ List<String> source = generateSourceOfValidatorMethodWith(
+ annotationOf(someConditions(), preProcess(), aReqType()),
+ modifiers("private", "static"),
+ returnValue("OMRequest"),
+ parameters("OMRequest rq", "ValidationContext ctx"),
+ exceptions());
+
+ assertThat(compile(source)).succeeded();
+ }
+
+ @Test
+ public void testValidatorMethodCanBeDefaultVisible() {
+ List<String> source = generateSourceOfValidatorMethodWith(
+ annotationOf(someConditions(), preProcess(), aReqType()),
+ modifiers("static"),
+ returnValue("OMRequest"),
+ parameters("OMRequest rq", "ValidationContext ctx"),
+ exceptions());
+
+ assertThat(compile(source)).succeeded();
+ }
+
+ @Test
+ public void testValidatorMethodCanBeProtected() {
+ List<String> source = generateSourceOfValidatorMethodWith(
+ annotationOf(someConditions(), preProcess(), aReqType()),
+ modifiers("protected", "static"),
+ returnValue("OMRequest"),
+ parameters("OMRequest rq", "ValidationContext ctx"),
+ exceptions());
+
+ assertThat(compile(source)).succeeded();
+ }
+
+ @Test
+ public void testEmptyValidationConditionListDoesNotCompile() {
+ List<String> source = generateSourceOfValidatorMethodWith(
+ annotationOf(emptyConditions(), preProcess(), aReqType()),
+ modifiers("public", "static"),
+ returnValue("OMRequest"),
+ parameters("OMRequest rq", "ValidationContext ctx"),
+ exceptions());
+
+ assertThat(compile(source)).hadErrorContaining(ERROR_CONDITION_IS_EMPTY);
+ }
+
+ @Test
+ public void testNotEnoughParametersForPreProcess() {
+ List<String> source = generateSourceOfValidatorMethodWith(
+ annotationOf(someConditions(), preProcess(), aReqType()),
+ modifiers("public", "static"),
+ returnValue("OMRequest"),
+ parameters("OMRequest rq"),
+ exceptions());
+
+ assertThat(compile(source))
+ .hadErrorContaining(
+ String.format(ERROR_UNEXPECTED_PARAMETER_COUNT, 2, 1));
+ }
+
+ @Test
+ public void testTooManyParametersForPreProcess() {
+ List<String> source = generateSourceOfValidatorMethodWith(
+ annotationOf(someConditions(), preProcess(), aReqType()),
+ modifiers("public", "static"),
+ returnValue("OMRequest"),
+ parameters("OMRequest rq", "OMResponse rp", "ValidationContext ctx"),
+ exceptions());
+
+ assertThat(compile(source))
+ .hadErrorContaining(
+ String.format(ERROR_UNEXPECTED_PARAMETER_COUNT, 2, 3));
+ }
+
+ @Test
+ public void testNotEnoughParametersForPostProcess() {
+ List<String> source = generateSourceOfValidatorMethodWith(
+ annotationOf(someConditions(), postProcess(), aReqType()),
+ modifiers("public", "static"),
+ returnValue("OMResponse"),
+ parameters("OMRequest rq", "OMResponse rp"),
+ exceptions());
+
+ assertThat(compile(source))
+ .hadErrorContaining(
+ String.format(ERROR_UNEXPECTED_PARAMETER_COUNT, 3, 2));
+ }
+
+ @Test
+ public void testTooManyParametersForPostProcess() {
+ List<String> source = generateSourceOfValidatorMethodWith(
+ annotationOf(someConditions(), postProcess(), aReqType()),
+ modifiers("public", "static"),
+ returnValue("OMResponse"),
+ parameters("OMRequest rq", "OMResponse rp", "ValidationContext ctx",
+ "String name"),
+ exceptions());
+
+ assertThat(compile(source))
+ .hadErrorContaining(
+ String.format(ERROR_UNEXPECTED_PARAMETER_COUNT, 3, 4));
+ }
+
+ @Test
+ public void testWrongReturnValueForPreProcess() {
+ List<String> source = generateSourceOfValidatorMethodWith(
+ annotationOf(someConditions(), preProcess(), aReqType()),
+ modifiers("public", "static"),
+ returnValue("String"),
+ parameters("OMRequest rq", "ValidationContext ctx"),
+ exceptions());
+
+ assertThat(compile(source))
+ .hadErrorContaining(ERROR_VALIDATOR_METHOD_HAS_TO_RETURN_OMREQUEST);
+ }
+
+ @Test
+ public void testWrongReturnValueForPostProcess() {
+ List<String> source = generateSourceOfValidatorMethodWith(
+ annotationOf(someConditions(), postProcess(), aReqType()),
+ modifiers("public", "static"),
+ returnValue("String"),
+ parameters("OMRequest rq", "OMResponse rp", "ValidationContext ctx"),
+ exceptions());
+
+ assertThat(compile(source))
+ .hadErrorContaining(ERROR_VALIDATOR_METHOD_HAS_TO_RETURN_OMRESPONSE);
+ }
+
+ @Test
+ public void testWrongFirstArgumentForPreProcess() {
+ List<String> source = generateSourceOfValidatorMethodWith(
+ annotationOf(someConditions(), preProcess(), aReqType()),
+ modifiers("public", "static"),
+ returnValue("OMRequest"),
+ parameters("String rq", "ValidationContext ctx"),
+ exceptions());
+
+ assertThat(compile(source))
+ .hadErrorContaining(ERROR_FIRST_PARAM_HAS_TO_BE_OMREQUEST);
+ }
+
+ @Test
+ public void testWrongFirstArgumentForPostProcess() {
+ List<String> source = generateSourceOfValidatorMethodWith(
+ annotationOf(someConditions(), postProcess(), aReqType()),
+ modifiers("public", "static"),
+ returnValue("OMResponse"),
+ parameters("String rq", "OMResponse rp", "ValidationContext ctx"),
+ exceptions());
+
+ assertThat(compile(source))
+ .hadErrorContaining(ERROR_FIRST_PARAM_HAS_TO_BE_OMREQUEST);
+ }
+
+ @Test
+ public void testWrongSecondArgumentForPreProcess() {
+ List<String> source = generateSourceOfValidatorMethodWith(
+ annotationOf(someConditions(), preProcess(), aReqType()),
+ modifiers("public", "static"),
+ returnValue("OMRequest"),
+ parameters("OMRequest rq", "String ctx"),
+ exceptions());
+
+ assertThat(compile(source))
+ .hadErrorContaining(ERROR_LAST_PARAM_HAS_TO_BE_VALIDATION_CONTEXT);
+ }
+
+ @Test
+ public void testWrongSecondArgumentForPostProcess() {
+ List<String> source = generateSourceOfValidatorMethodWith(
+ annotationOf(someConditions(), postProcess(), aReqType()),
+ modifiers("public", "static"),
+ returnValue("OMResponse"),
+ parameters("OMRequest rq", "String rp", "ValidationContext ctx"),
+ exceptions());
+
+ assertThat(compile(source))
+ .hadErrorContaining(ERROR_SECOND_PARAM_HAS_TO_BE_OMRESPONSE);
+ }
+
+ @Test
+ public void testWrongThirdArgumentForPostProcess() {
+ List<String> source = generateSourceOfValidatorMethodWith(
+ annotationOf(someConditions(), postProcess(), aReqType()),
+ modifiers("public", "static"),
+ returnValue("OMResponse"),
+ parameters("OMRequest rq", "OMResponse rp", "String ctx"),
+ exceptions());
+
+ assertThat(compile(source))
+ .hadErrorContaining(ERROR_LAST_PARAM_HAS_TO_BE_VALIDATION_CONTEXT);
+ }
+
+ @Test
+ public void testInvalidProcessingPhase() {
+ List<String> source = generateSourceOfValidatorMethodWith(
+ annotationOf(someConditions(), "INVALID", aReqType()),
+ modifiers("public", "static"),
+ returnValue("OMResponse"),
+ parameters("OMRequest rq", "OMResponse rp", "ValidationContext ctx"),
+ exceptions("ServiceException"));
+
+ assertThat(compile(source)).failed();
+ }
+
+ @Test
+ public void testMultipleErrorMessages() {
+ List<String> source = generateSourceOfValidatorMethodWith(
+ annotationOf(emptyConditions(), postProcess(), aReqType()),
+ modifiers(),
+ returnValue("String"),
+ parameters("String rq", "int rp", "String ctx"),
+ exceptions());
+
+ Compilation compilation = compile(source);
+ assertThat(compilation).hadErrorContaining(ERROR_CONDITION_IS_EMPTY);
+ assertThat(compilation)
+ .hadErrorContaining(ERROR_VALIDATOR_METHOD_HAS_TO_BE_STATIC);
+ assertThat(compilation)
+ .hadErrorContaining(ERROR_VALIDATOR_METHOD_HAS_TO_RETURN_OMRESPONSE);
+ assertThat(compilation)
+ .hadErrorContaining(ERROR_FIRST_PARAM_HAS_TO_BE_OMREQUEST);
+ assertThat(compilation)
+ .hadErrorContaining(ERROR_SECOND_PARAM_HAS_TO_BE_OMRESPONSE);
+ assertThat(compilation)
+ .hadErrorContaining(ERROR_LAST_PARAM_HAS_TO_BE_VALIDATION_CONTEXT);
+ }
+
+ private Compilation compile(List<String> source) {
+ Compilation c = javac()
+ .withProcessors(new RequestFeatureValidatorProcessor())
+ .compile(JavaFileObjects.forSourceLines(CLASSNAME, source));
+ c.diagnostics().forEach(System.out::println);
+ return c;
+ }
+
+ private ValidationCondition[] someConditions() {
+ return
+ new ValidationCondition[] {ValidationCondition.OLDER_CLIENT_REQUESTS};
+ }
+
+ private ValidationCondition[] emptyConditions() {
+ return new ValidationCondition[] {};
+ }
+
+ private RequestProcessingPhase preProcess() {
+ return RequestProcessingPhase.PRE_PROCESS;
+ }
+
+ private RequestProcessingPhase postProcess() {
+ return RequestProcessingPhase.POST_PROCESS;
+ }
+
+ private Type aReqType() {
+ return Type.CreateVolume;
+ }
+
+ private String returnValue(String retVal) {
+ return retVal;
+ }
+
+ private String[] parameters(String... params) {
+ return params;
+ }
+
+ private String[] modifiers(String... modifiers) {
+ return modifiers;
+ }
+
+ private String[] exceptions(String... exceptions) {
+ return exceptions;
+ }
+
+ private List<String> generateSourceOfValidatorMethodWith(
+ String annotation,
+ String[] modifiers,
+ String returnType,
+ String[] paramspecs,
+ String[] exceptions) {
+ List<String> lines = new ArrayList<>(allImports());
+ lines.add("");
+ lines.add("public class " + CLASSNAME + " {");
+ lines.add("");
+ lines.add(" " + annotation);
+ StringBuilder signature =
+ buildMethodSignature(modifiers, returnType, paramspecs, exceptions);
+ lines.add(signature.toString());
+ lines.add(" return null;");
+ lines.add(" }");
+ lines.add("}");
+ lines.add("");
+ lines.stream()
+ .filter(s -> !s.startsWith("import"))
+ .forEach(System.out::println);
+ return lines;
+ }
+
+ private String annotationOf(
+ ValidationCondition[] conditions,
+ RequestProcessingPhase phase,
+ Type reqType) {
+ return annotationOf(conditions, phase.name(), reqType);
+ }
+
+ private String annotationOf(
+ ValidationCondition[] conditions,
+ String phase,
+ Type reqType) {
+ StringBuilder annotation = new StringBuilder();
+ annotation.append("@RequestFeatureValidator(");
+ StringBuilder conditionsArray = new StringBuilder();
+ conditionsArray.append("conditions = { ");
+ if (conditions.length > 0) {
+ for (ValidationCondition condition : conditions) {
+ conditionsArray.append(condition.name()).append(", ");
+ }
+ annotation
+ .append(conditionsArray.substring(0, conditionsArray.length() - 2));
+ } else {
+ annotation.append(conditionsArray);
+ }
+ annotation.append(" }");
+ annotation.append(", processingPhase = ").append(phase);
+ annotation.append(", requestType = ").append(reqType.name());
+ annotation.append(" )");
+ return annotation.toString();
+ }
+
+ private List<String> allImports() {
+ List<String> imports = new ArrayList<>();
+ imports.add("import org.apache.hadoop.ozone.om.request.validation"
+ + ".RequestFeatureValidator;");
+ imports.add("import org.apache.hadoop.ozone.protocol.proto"
+ + ".OzoneManagerProtocolProtos.OMRequest;");
+ imports.add("import org.apache.hadoop.ozone.protocol.proto"
+ + ".OzoneManagerProtocolProtos.OMResponse;");
+ imports.add("import org.apache.hadoop.ozone.om.request.validation"
+ + ".ValidationContext;");
+ imports.add("import com.google.protobuf.ServiceException;");
+ for (ValidationCondition condition : ValidationCondition.values()) {
+ imports.add("import static org.apache.hadoop.ozone.om.request.validation"
+ + ".ValidationCondition." + condition.name() + ";");
+ }
+ for (RequestProcessingPhase phase : RequestProcessingPhase.values()) {
+ imports.add("import static org.apache.hadoop.ozone.om.request.validation"
+ + ".RequestProcessingPhase." + phase.name() + ";");
+ }
+ for (Type reqType : Type.values()) {
+ imports.add("import static org.apache.hadoop.ozone.protocol.proto"
+ + ".OzoneManagerProtocolProtos.Type." + reqType.name() + ";");
+ }
+ return imports;
+ }
+
+ private StringBuilder buildMethodSignature(
+ String[] modifiers, String returnType,
+ String[] paramspecs, String[] exceptions) {
+ StringBuilder signature = new StringBuilder();
+ signature.append(" ");
+ for (String modifier : modifiers) {
+ signature.append(modifier).append(" ");
+ }
+ signature.append(returnType).append(" ");
+ signature.append("validatorMethod(");
+ signature.append(createParameterList(paramspecs));
+ signature.append(") ");
+ signature.append(createThrowsClause(exceptions));
+ return signature.append(" {");
+ }
+
+ private String createParameterList(String[] paramSpecs) {
+ if (paramSpecs == null || paramSpecs.length == 0) {
+ return "";
+ }
+ StringBuilder parameters = new StringBuilder();
+ for (String paramSpec : paramSpecs) {
+ parameters.append(paramSpec).append(", ");
+ }
+ return parameters.substring(0, parameters.length() - 2);
+ }
+
+ private String createThrowsClause(String[] exceptions) {
+ StringBuilder throwsClause = new StringBuilder();
+ if (exceptions != null && exceptions.length > 0) {
+ throwsClause.append(" throws ");
+ StringBuilder exceptionList = new StringBuilder();
+ for (String exception : exceptions) {
+ exceptionList.append(exception).append(", ");
+ }
+ throwsClause
+ .append(exceptionList.substring(0, exceptionList.length() - 2));
+ }
+ return throwsClause.toString();
+ }
+}
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestRequestValidations.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestRequestValidations.java
new file mode 100644
index 0000000..4508eaf
--- /dev/null
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestRequestValidations.java
@@ -0,0 +1,349 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.ozone.om.request.validation;
+
+import com.google.protobuf.ServiceException;
+import org.apache.hadoop.ozone.ClientVersion;
+import org.apache.hadoop.ozone.om.request.validation.testvalidatorset1.GeneralValidatorsForTesting;
+import org.apache.hadoop.ozone.om.request.validation.testvalidatorset1.GeneralValidatorsForTesting.ValidationListener;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type;
+import org.apache.hadoop.ozone.upgrade.LayoutVersionManager;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.apache.hadoop.ozone.om.request.validation.ValidationContext.of;
+import static org.apache.hadoop.ozone.om.request.validation.testvalidatorset1.GeneralValidatorsForTesting.startValidatorTest;
+import static org.apache.hadoop.ozone.om.request.validation.testvalidatorset1.GeneralValidatorsForTesting.finishValidatorTest;
+import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Status.OK;
+import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type.CreateKey;
+import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type.DeleteKeys;
+import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type.RenameKey;
+import static org.junit.Assert.fail;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+/**
+ * Testing the RequestValidations class that is used to run the validation for
+ * any given request that arrives to OzoneManager.
+ */
+public class TestRequestValidations {
+ private static final String PACKAGE =
+ "org.apache.hadoop.ozone.om.request.validation.testvalidatorset1";
+
+ private static final String PACKAGE_WO_VALIDATORS =
+ "org.apache.hadoop.hdds.annotation";
+
+ private final ValidationListenerImpl validationListener =
+ new ValidationListenerImpl();
+
+ @Before
+ public void setup() {
+ startValidatorTest();
+ validationListener.attach();
+ }
+
+ @After
+ public void tearDown() {
+ validationListener.detach();
+ finishValidatorTest();
+ }
+
+ @Test(expected = NullPointerException.class)
+ public void testUsingRegistryWithoutLoading() throws ServiceException {
+ new RequestValidations()
+ .fromPackage(PACKAGE)
+ .withinContext(of(aFinalizedVersionManager()))
+ .validateRequest(aCreateKeyRequest(currentClientVersion()));
+ }
+
+ @Test(expected = NullPointerException.class)
+ public void testUsingRegistryWithoutContext() throws ServiceException {
+ new RequestValidations()
+ .fromPackage(PACKAGE)
+ .load()
+ .validateRequest(aCreateKeyRequest(currentClientVersion()));
+ }
+
+ @Test
+ public void testUsingRegistryWithoutPackage() throws ServiceException {
+ new RequestValidations()
+ .withinContext(of(aFinalizedVersionManager()))
+ .load()
+ .validateRequest(aCreateKeyRequest(currentClientVersion()));
+
+ validationListener.assertNumOfEvents(0);
+ }
+
+ @Test
+ public void testNoPreValidationsWithoutValidationMethods()
+ throws ServiceException {
+ int omVersion = 0;
+ ValidationContext ctx = of(aFinalizedVersionManager());
+ RequestValidations validations = loadEmptyValidations(ctx);
+
+ validations.validateRequest(aCreateKeyRequest(omVersion));
+
+ validationListener.assertNumOfEvents(0);
+ }
+
+ @Test
+ public void testNoPostValidationsWithoutValidationMethods()
+ throws ServiceException {
+ ValidationContext ctx = of(aFinalizedVersionManager());
+ RequestValidations validations = loadEmptyValidations(ctx);
+
+ validations.validateResponse(
+ aCreateKeyRequest(currentClientVersion()), aCreateKeyResponse());
+
+ validationListener.assertNumOfEvents(0);
+ }
+
+ @Test
+ public void testNoPreValidationsRunningForRequestTypeWithoutValidators()
+ throws ServiceException {
+ ValidationContext ctx = of(aFinalizedVersionManager());
+ RequestValidations validations = loadValidations(ctx);
+
+ validations.validateRequest(aRenameKeyRequest(currentClientVersion()));
+
+ validationListener.assertNumOfEvents(0);
+ }
+
+ @Test
+ public void testNoPostValidationsAreRunningForRequestTypeWithoutValidators()
+ throws ServiceException {
+ ValidationContext ctx = of(aFinalizedVersionManager());
+ RequestValidations validations = loadValidations(ctx);
+
+ validations.validateResponse(
+ aRenameKeyRequest(currentClientVersion()), aRenameKeyResponse());
+
+ validationListener.assertNumOfEvents(0);
+ }
+
+ @Test
+ public void testPreProcessorExceptionHandling() {
+ ValidationContext ctx = of(aFinalizedVersionManager());
+ RequestValidations validations = loadValidations(ctx);
+
+ try {
+ validations.validateRequest(aDeleteKeysRequest(olderClientVersion()));
+ fail("ServiceException was expected but was not thrown.");
+ } catch (ServiceException ignored) { }
+
+ validationListener.assertNumOfEvents(1);
+ validationListener.assertExactListOfValidatorsCalled(
+ "throwingPreProcessValidator");
+ }
+
+ @Test
+ public void testPostProcessorExceptionHandling() {
+ ValidationContext ctx = of(aFinalizedVersionManager());
+ RequestValidations validations = loadValidations(ctx);
+
+ try {
+ validations.validateResponse(
+ aDeleteKeysRequest(olderClientVersion()), aDeleteKeysResponse());
+ fail("ServiceException was expected but was not thrown.");
+ } catch (ServiceException ignored) { }
+
+ validationListener.assertNumOfEvents(1);
+ validationListener.assertExactListOfValidatorsCalled(
+ "throwingPostProcessValidator");
+ }
+
+ @Test
+ public void testOldClientConditionIsRecognizedAndPreValidatorsApplied()
+ throws ServiceException {
+ ValidationContext ctx = of(aFinalizedVersionManager());
+ RequestValidations validations = loadValidations(ctx);
+
+ validations.validateRequest(aCreateKeyRequest(olderClientVersion()));
+
+ validationListener.assertNumOfEvents(1);
+ validationListener.assertExactListOfValidatorsCalled(
+ "oldClientPreProcessCreateKeyValidator");
+ }
+
+ @Test
+ public void testOldClientConditionIsRecognizedAndPostValidatorsApplied()
+ throws ServiceException {
+ ValidationContext ctx = of(aFinalizedVersionManager());
+ RequestValidations validations = loadValidations(ctx);
+
+ validations.validateResponse(
+ aCreateKeyRequest(olderClientVersion()), aCreateKeyResponse());
+
+ validationListener.assertNumOfEvents(2);
+ validationListener.assertExactListOfValidatorsCalled(
+ "oldClientPostProcessCreateKeyValidator",
+ "oldClientPostProcessCreateKeyValidator2");
+ }
+
+ @Test
+ public void testPreFinalizedWithOldClientConditionPreProcValidatorsApplied()
+ throws ServiceException {
+ ValidationContext ctx = of(anUnfinalizedVersionManager());
+ RequestValidations validations = loadValidations(ctx);
+
+ validations.validateRequest(aCreateKeyRequest(olderClientVersion()));
+
+ validationListener.assertNumOfEvents(2);
+ validationListener.assertExactListOfValidatorsCalled(
+ "preFinalizePreProcessCreateKeyValidator",
+ "oldClientPreProcessCreateKeyValidator");
+ }
+
+ @Test
+ public void testPreFinalizedWithOldClientConditionPostProcValidatorsApplied()
+ throws ServiceException {
+ ValidationContext ctx = of(anUnfinalizedVersionManager());
+ RequestValidations validations = loadValidations(ctx);
+
+ validations.validateResponse(
+ aCreateKeyRequest(olderClientVersion()), aCreateKeyResponse());
+
+ validationListener.assertNumOfEvents(3);
+ validationListener.assertExactListOfValidatorsCalled(
+ "preFinalizePostProcessCreateKeyValidator",
+ "oldClientPostProcessCreateKeyValidator",
+ "oldClientPostProcessCreateKeyValidator2");
+ }
+
+ private RequestValidations loadValidations(ValidationContext ctx) {
+ return new RequestValidations()
+ .fromPackage(PACKAGE)
+ .withinContext(ctx)
+ .load();
+ }
+
+ private RequestValidations loadEmptyValidations(ValidationContext ctx) {
+ return new RequestValidations()
+ .fromPackage(PACKAGE_WO_VALIDATORS)
+ .withinContext(ctx)
+ .load();
+ }
+
+ private int olderClientVersion() {
+ return ClientVersion.CURRENT_VERSION - 1;
+ }
+
+ private int currentClientVersion() {
+ return ClientVersion.CURRENT_VERSION;
+ }
+
+ private OMRequest aCreateKeyRequest(int clientVersion) {
+ return aRequest(CreateKey, clientVersion);
+ }
+
+ private OMRequest aDeleteKeysRequest(int clientVersion) {
+ return aRequest(DeleteKeys, clientVersion);
+ }
+
+ private OMRequest aRenameKeyRequest(int clientVersion) {
+ return aRequest(RenameKey, clientVersion);
+ }
+
+ private OMRequest aRequest(Type type, int clientVersion) {
+ return OMRequest.newBuilder()
+ .setVersion(clientVersion)
+ .setCmdType(type)
+ .setClientId("TestClient")
+ .build();
+ }
+
+ private OMResponse aCreateKeyResponse() {
+ return aResponse(CreateKey);
+ }
+
+ private OMResponse aDeleteKeysResponse() {
+ return aResponse(DeleteKeys);
+ }
+
+ private OMResponse aRenameKeyResponse() {
+ return aResponse(RenameKey);
+ }
+
+ private OMResponse aResponse(Type type) {
+ return OMResponse.newBuilder()
+ .setCmdType(type)
+ .setStatus(OK)
+ .build();
+ }
+
+ private LayoutVersionManager aFinalizedVersionManager() {
+ LayoutVersionManager vm = mock(LayoutVersionManager.class);
+ when(vm.needsFinalization()).thenReturn(false);
+ return vm;
+ }
+
+ private LayoutVersionManager anUnfinalizedVersionManager() {
+ LayoutVersionManager vm = mock(LayoutVersionManager.class);
+ when(vm.needsFinalization()).thenReturn(true);
+ return vm;
+ }
+
+ private static class ValidationListenerImpl implements ValidationListener {
+ private List<String> calledMethods = new ArrayList<>();
+
+ @Override
+ public void validationCalled(String calledMethodName) {
+ calledMethods.add(calledMethodName);
+ }
+
+ public void attach() {
+ GeneralValidatorsForTesting.addListener(this);
+ }
+
+ public void detach() {
+ GeneralValidatorsForTesting.removeListener(this);
+ reset();
+ }
+
+ public void reset() {
+ calledMethods = new ArrayList<>();
+ }
+
+ public void assertExactListOfValidatorsCalled(String... methodNames) {
+ List<String> calls = new ArrayList<>(calledMethods);
+ for (String methodName : methodNames) {
+ if (!calls.remove(methodName)) {
+ fail("Expected method call for " + methodName + " did not happened.");
+ }
+ }
+ if (!calls.isEmpty()) {
+ fail("Some of the methods were not called."
+ + "Missing calls for: " + calls);
+ }
+ }
+
+ public void assertNumOfEvents(int count) {
+ if (calledMethods.size() != count) {
+ fail("Unexpected validation call count."
+ + " Expected: " + count + "; Happened: " + calledMethods.size());
+ }
+ }
+ }
+}
+
+
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestValidatorRegistry.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestValidatorRegistry.java
new file mode 100644
index 0000000..e11fa51
--- /dev/null
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/TestValidatorRegistry.java
@@ -0,0 +1,215 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.ozone.om.request.validation;
+
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.reflections.util.ClasspathHelper;
+
+import java.lang.reflect.Method;
+import java.net.MalformedURLException;
+import java.net.URL;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import static java.util.Arrays.asList;
+import static java.util.Collections.emptyList;
+import static org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase.POST_PROCESS;
+import static org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase.PRE_PROCESS;
+import static org.apache.hadoop.ozone.om.request.validation.ValidationCondition.CLUSTER_NEEDS_FINALIZATION;
+import static org.apache.hadoop.ozone.om.request.validation.ValidationCondition.OLDER_CLIENT_REQUESTS;
+import static org.apache.hadoop.ozone.om.request.validation.testvalidatorset1.GeneralValidatorsForTesting.startValidatorTest;
+import static org.apache.hadoop.ozone.om.request.validation.testvalidatorset1.GeneralValidatorsForTesting.finishValidatorTest;
+import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type.CreateDirectory;
+import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type.CreateKey;
+import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type.CreateVolume;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * Validator registry tests.
+ * For validator method declarations see the GeneralValidatorsForTesting
+ * and ValidatorsForOnlyNewClientValidations (in ../avalidation2) classes.
+ */
+public class TestValidatorRegistry {
+ private static final String PACKAGE =
+ "org.apache.hadoop.ozone.om.request.validation.testvalidatorset1";
+
+ private static final String PACKAGE2 =
+ "org.apache.hadoop.ozone.om.request.validation.testvalidatorset2";
+
+ private static final String PACKAGE_WO_VALIDATORS =
+ "org.apache.hadoop.hdds.annotation";
+
+ @Before
+ public void setup() {
+ startValidatorTest();
+ }
+
+ @After
+ public void tearDown() {
+ finishValidatorTest();
+ }
+
+ @Test
+ public void testNoValidatorsReturnedForEmptyConditionList() {
+ ValidatorRegistry registry = new ValidatorRegistry(PACKAGE);
+ List<Method> validators =
+ registry.validationsFor(emptyList(), CreateKey, PRE_PROCESS);
+
+ assertTrue(validators.isEmpty());
+ }
+
+ @Test
+ public void testRegistryHasThePreFinalizePreProcessCreateKeyValidator() {
+ ValidatorRegistry registry = new ValidatorRegistry(PACKAGE);
+ List<Method> validators =
+ registry.validationsFor(
+ asList(CLUSTER_NEEDS_FINALIZATION), CreateKey, PRE_PROCESS);
+
+ assertEquals(1, validators.size());
+ String expectedMethodName = "preFinalizePreProcessCreateKeyValidator";
+ assertEquals(expectedMethodName, validators.get(0).getName());
+ }
+
+ @Test
+ public void testRegistryHasThePreFinalizePostProcessCreateKeyValidator() {
+ ValidatorRegistry registry = new ValidatorRegistry(PACKAGE);
+ List<Method> validators =
+ registry.validationsFor(
+ asList(CLUSTER_NEEDS_FINALIZATION), CreateKey, POST_PROCESS);
+
+ assertEquals(1, validators.size());
+ String expectedMethodName = "preFinalizePostProcessCreateKeyValidator";
+ assertEquals(expectedMethodName, validators.get(0).getName());
+ }
+
+ @Test
+ public void testRegistryHasTheOldClientPreProcessCreateKeyValidator() {
+ ValidatorRegistry registry = new ValidatorRegistry(PACKAGE);
+ List<Method> validators =
+ registry.validationsFor(
+ asList(OLDER_CLIENT_REQUESTS), CreateKey, PRE_PROCESS);
+
+ assertEquals(2, validators.size());
+ List<String> methodNames =
+ validators.stream().map(Method::getName).collect(Collectors.toList());
+ assertTrue(methodNames.contains("oldClientPreProcessCreateKeyValidator"));
+ assertTrue(methodNames.contains("oldClientPreProcessCreateKeyValidator2"));
+ }
+
+ @Test
+ public void testRegistryHasTheOldClientPostProcessCreateKeyValidator() {
+ ValidatorRegistry registry = new ValidatorRegistry(PACKAGE);
+ List<Method> validators =
+ registry.validationsFor(
+ asList(OLDER_CLIENT_REQUESTS), CreateKey, POST_PROCESS);
+
+ assertEquals(2, validators.size());
+ List<String> methodNames =
+ validators.stream().map(Method::getName).collect(Collectors.toList());
+ assertTrue(methodNames.contains("oldClientPostProcessCreateKeyValidator"));
+ assertTrue(methodNames.contains("oldClientPostProcessCreateKeyValidator2"));
+ }
+
+ @Test
+ public void testRegistryHasTheMultiPurposePreProcessCreateVolumeValidator() {
+ ValidatorRegistry registry = new ValidatorRegistry(PACKAGE);
+ List<Method> preFinalizeValidators =
+ registry.validationsFor(
+ asList(CLUSTER_NEEDS_FINALIZATION), CreateVolume, PRE_PROCESS);
+ List<Method> newClientValidators =
+ registry.validationsFor(
+ asList(OLDER_CLIENT_REQUESTS), CreateVolume, PRE_PROCESS);
+
+ assertEquals(1, preFinalizeValidators.size());
+ assertEquals(1, newClientValidators.size());
+ String expectedMethodName = "multiPurposePreProcessCreateVolumeValidator";
+ assertEquals(expectedMethodName, preFinalizeValidators.get(0).getName());
+ assertEquals(expectedMethodName, newClientValidators.get(0).getName());
+ }
+
+ @Test
+ public void testRegistryHasTheMultiPurposePostProcessCreateVolumeValidator() {
+ ValidatorRegistry registry = new ValidatorRegistry(PACKAGE);
+ List<Method> preFinalizeValidators =
+ registry.validationsFor(
+ asList(CLUSTER_NEEDS_FINALIZATION), CreateVolume, POST_PROCESS);
+ List<Method> oldClientValidators =
+ registry.validationsFor(
+ asList(OLDER_CLIENT_REQUESTS), CreateVolume, POST_PROCESS);
+
+ assertEquals(1, preFinalizeValidators.size());
+ assertEquals(1, oldClientValidators.size());
+ String expectedMethodName = "multiPurposePostProcessCreateVolumeValidator";
+ assertEquals(expectedMethodName, preFinalizeValidators.get(0).getName());
+ assertEquals(expectedMethodName, oldClientValidators.get(0).getName());
+ }
+
+ @Test
+ public void testValidatorsAreReturnedForMultiCondition() {
+ ValidatorRegistry registry = new ValidatorRegistry(PACKAGE);
+ List<Method> validators =
+ registry.validationsFor(
+ asList(CLUSTER_NEEDS_FINALIZATION, OLDER_CLIENT_REQUESTS),
+ CreateKey, POST_PROCESS);
+
+ assertEquals(3, validators.size());
+ List<String> methodNames =
+ validators.stream().map(Method::getName).collect(Collectors.toList());
+ assertTrue(
+ methodNames.contains("preFinalizePostProcessCreateKeyValidator"));
+ assertTrue(
+ methodNames.contains("oldClientPostProcessCreateKeyValidator"));
+ assertTrue(
+ methodNames.contains("oldClientPostProcessCreateKeyValidator2"));
+ }
+
+ @Test
+ public void testNoValidatorForRequestsAtAllReturnsEmptyList() {
+ ValidatorRegistry registry = new ValidatorRegistry(PACKAGE_WO_VALIDATORS);
+
+ assertTrue(registry.validationsFor(
+ asList(OLDER_CLIENT_REQUESTS), CreateKey, PRE_PROCESS).isEmpty());
+ }
+
+ @Test
+ public void testNoValidatorForConditionReturnsEmptyList()
+ throws MalformedURLException {
+ Collection<URL> urls = ClasspathHelper.forPackage(PACKAGE2);
+ Collection<URL> urlsToUse = new ArrayList<>();
+ for (URL url : urls) {
+ urlsToUse.add(new URL(url, PACKAGE2.replaceAll("\\.", "/")));
+ }
+ ValidatorRegistry registry = new ValidatorRegistry(urlsToUse);
+
+ assertTrue(registry.validationsFor(
+ asList(CLUSTER_NEEDS_FINALIZATION), CreateKey, PRE_PROCESS).isEmpty());
+ }
+
+ @Test
+ public void testNoDefinedValidationForRequestReturnsEmptyList() {
+ ValidatorRegistry registry = new ValidatorRegistry(PACKAGE);
+
+ assertTrue(registry.validationsFor(
+ asList(OLDER_CLIENT_REQUESTS), CreateDirectory, null).isEmpty());
+ }
+
+}
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/testvalidatorset1/GeneralValidatorsForTesting.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/testvalidatorset1/GeneralValidatorsForTesting.java
new file mode 100644
index 0000000..35c3afa
--- /dev/null
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/testvalidatorset1/GeneralValidatorsForTesting.java
@@ -0,0 +1,190 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.ozone.om.request.validation.testvalidatorset1;
+
+import org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator;
+import org.apache.hadoop.ozone.om.request.validation.TestRequestValidations;
+import org.apache.hadoop.ozone.om.request.validation.ValidationContext;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMResponse;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import static org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase.POST_PROCESS;
+import static org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase.PRE_PROCESS;
+import static org.apache.hadoop.ozone.om.request.validation.ValidationCondition.CLUSTER_NEEDS_FINALIZATION;
+import static org.apache.hadoop.ozone.om.request.validation.ValidationCondition.OLDER_CLIENT_REQUESTS;
+import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type.CreateKey;
+import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type.CreateVolume;
+import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type.DeleteKeys;
+
+/**
+ * Some annotated request validator method, and facilities to help check if
+ * validations were properly called from tests where applicable.
+ */
+public final class GeneralValidatorsForTesting {
+
+ /**
+ * As the validators written here does not override any request or response
+ * but throw exceptions for specific tests, a test that wants to directly
+ * use a validator here, has to turn on this boolean, and the method that
+ * the test relies on has to check for this value.
+ *
+ * This is necessary to do not affect other tests that are testing requests
+ * processing, as for some of those tests this package is on the classpath
+ * and therefore the annotated validations are loadede for them.
+ */
+ private static boolean validatorTestsRunning = false;
+
+ public static void startValidatorTest() {
+ validatorTestsRunning = true;
+ }
+
+ public static void finishValidatorTest() {
+ validatorTestsRunning = false;
+ }
+
+ private GeneralValidatorsForTesting() { }
+
+ /**
+ * Interface to easily add listeners that get notified if a certain validator
+ * method defined in this class was called.
+ *
+ * @see TestRequestValidations for more details on how this intercace is
+ * being used.
+ */
+ @FunctionalInterface
+ public interface ValidationListener {
+ void validationCalled(String calledMethodName);
+ }
+
+ private static List<ValidationListener> listeners = new ArrayList<>();
+
+ public static void addListener(ValidationListener listener) {
+ listeners.add(listener);
+ }
+
+ public static void removeListener(ValidationListener listener) {
+ listeners.remove(listener);
+ }
+
+ private static void fireValidationEvent(String calledMethodName) {
+ listeners.forEach(l -> l.validationCalled(calledMethodName));
+ }
+
+ @RequestFeatureValidator(
+ conditions = { CLUSTER_NEEDS_FINALIZATION },
+ processingPhase = PRE_PROCESS,
+ requestType = CreateKey)
+ public static OMRequest preFinalizePreProcessCreateKeyValidator(
+ OMRequest req, ValidationContext ctx) {
+ fireValidationEvent("preFinalizePreProcessCreateKeyValidator");
+ return req;
+ }
+
+ @RequestFeatureValidator(
+ conditions = { CLUSTER_NEEDS_FINALIZATION },
+ processingPhase = POST_PROCESS,
+ requestType = CreateKey)
+ public static OMResponse preFinalizePostProcessCreateKeyValidator(
+ OMRequest req, OMResponse resp, ValidationContext ctx) {
+ fireValidationEvent("preFinalizePostProcessCreateKeyValidator");
+ return resp;
+ }
+
+ @RequestFeatureValidator(
+ conditions = { OLDER_CLIENT_REQUESTS },
+ processingPhase = PRE_PROCESS,
+ requestType = CreateKey)
+ public static OMRequest oldClientPreProcessCreateKeyValidator(
+ OMRequest req, ValidationContext ctx) {
+ fireValidationEvent("oldClientPreProcessCreateKeyValidator");
+ return req;
+ }
+
+ @RequestFeatureValidator(
+ conditions = { OLDER_CLIENT_REQUESTS },
+ processingPhase = POST_PROCESS,
+ requestType = CreateKey)
+ public static OMResponse oldClientPostProcessCreateKeyValidator(
+ OMRequest req, OMResponse resp, ValidationContext ctx) {
+ fireValidationEvent("oldClientPostProcessCreateKeyValidator");
+ return resp;
+ }
+
+ @RequestFeatureValidator(
+ conditions = { CLUSTER_NEEDS_FINALIZATION, OLDER_CLIENT_REQUESTS },
+ processingPhase = PRE_PROCESS,
+ requestType = CreateVolume)
+ public static OMRequest multiPurposePreProcessCreateVolumeValidator(
+ OMRequest req, ValidationContext ctx) {
+ fireValidationEvent("multiPurposePreProcessCreateVolumeValidator");
+ return req;
+ }
+
+ @RequestFeatureValidator(
+ conditions = { OLDER_CLIENT_REQUESTS, CLUSTER_NEEDS_FINALIZATION },
+ processingPhase = POST_PROCESS,
+ requestType = CreateVolume)
+ public static OMResponse multiPurposePostProcessCreateVolumeValidator(
+ OMRequest req, OMResponse resp, ValidationContext ctx) {
+ fireValidationEvent("multiPurposePostProcessCreateVolumeValidator");
+ return resp;
+ }
+
+ @RequestFeatureValidator(
+ conditions = { OLDER_CLIENT_REQUESTS },
+ processingPhase = POST_PROCESS,
+ requestType = CreateKey)
+ public static OMResponse oldClientPostProcessCreateKeyValidator2(
+ OMRequest req, OMResponse resp, ValidationContext ctx) {
+ fireValidationEvent("oldClientPostProcessCreateKeyValidator2");
+ return resp;
+ }
+
+ @RequestFeatureValidator(
+ conditions = {OLDER_CLIENT_REQUESTS},
+ processingPhase = PRE_PROCESS,
+ requestType = DeleteKeys
+ )
+ public static OMRequest throwingPreProcessValidator(
+ OMRequest req, ValidationContext ctx) throws IOException {
+ fireValidationEvent("throwingPreProcessValidator");
+ if (validatorTestsRunning) {
+ throw new IOException("IOException: fail for testing...");
+ }
+ return req;
+ }
+
+ @RequestFeatureValidator(
+ conditions = {OLDER_CLIENT_REQUESTS},
+ processingPhase = POST_PROCESS,
+ requestType = DeleteKeys
+ )
+ public static OMResponse throwingPostProcessValidator(
+ OMRequest req, OMResponse resp, ValidationContext ctx)
+ throws IOException {
+ fireValidationEvent("throwingPostProcessValidator");
+ if (validatorTestsRunning) {
+ throw new IOException("IOException: fail for testing...");
+ }
+ return resp;
+ }
+
+}
diff --git a/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/testvalidatorset2/ValidatorsForOnlyOldClientValidations.java b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/testvalidatorset2/ValidatorsForOnlyOldClientValidations.java
new file mode 100644
index 0000000..447b9ab
--- /dev/null
+++ b/hadoop-ozone/ozone-manager/src/test/java/org/apache/hadoop/ozone/om/request/validation/testvalidatorset2/ValidatorsForOnlyOldClientValidations.java
@@ -0,0 +1,43 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with this
+ * work for additional information regarding copyright ownership. The ASF
+ * licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ * <p>
+ * http://www.apache.org/licenses/LICENSE-2.0
+ * <p>
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
+ * License for the specific language governing permissions and limitations under
+ * the License.
+ */
+package org.apache.hadoop.ozone.om.request.validation.testvalidatorset2;
+
+import org.apache.hadoop.ozone.om.request.validation.RequestFeatureValidator;
+import org.apache.hadoop.ozone.om.request.validation.ValidationContext;
+import org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.OMRequest;
+
+import static org.apache.hadoop.ozone.om.request.validation.RequestProcessingPhase.PRE_PROCESS;
+import static org.apache.hadoop.ozone.om.request.validation.ValidationCondition.OLDER_CLIENT_REQUESTS;
+import static org.apache.hadoop.ozone.protocol.proto.OzoneManagerProtocolProtos.Type.CreateKey;
+
+/**
+ * Separate validator methods for a few specific tests that covers cases where
+ * there are almost no validators added.
+ */
+public final class ValidatorsForOnlyOldClientValidations {
+
+ private ValidatorsForOnlyOldClientValidations() { }
+
+ @RequestFeatureValidator(
+ conditions = { OLDER_CLIENT_REQUESTS },
+ processingPhase = PRE_PROCESS,
+ requestType = CreateKey)
+ public static OMRequest oldClientPreProcessCreateKeyValidator2(
+ OMRequest req, ValidationContext ctx) {
+ return req;
+ }
+}
diff --git a/pom.xml b/pom.xml
index 82582d6..a745233 100644
--- a/pom.xml
+++ b/pom.xml
@@ -24,6 +24,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs
<packaging>pom</packaging>
<modules>
+ <module>dev-support/annotations</module>
<module>hadoop-hdds</module>
<module>hadoop-ozone</module>
</modules>
@@ -163,6 +164,10 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs
<hadoop-thirdparty.version>1.1.1</hadoop-thirdparty.version>
<findbugs.version>3.0.0</findbugs.version>
+ <!--
+ | Keep this in sync with the version used in
+ | /dev-support/annotatons/pom.xml
+ -->
<spotbugs.version>3.1.12</spotbugs.version>
<dnsjava.version>2.1.7</dnsjava.version>
@@ -232,8 +237,16 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs
<exec-maven-plugin.version>1.3.1</exec-maven-plugin.version>
<make-maven-plugin.version>1.0-beta-1</make-maven-plugin.version>
<native-maven-plugin.version>1.0-alpha-8</native-maven-plugin.version>
+ <!--
+ | Keep this in sync with the version used in
+ | /dev-support/annotatons/pom.xml
+ -->
<maven-checkstyle-plugin.version>3.1.2</maven-checkstyle-plugin.version>
<maven-site-plugin.version>3.9.1</maven-site-plugin.version>
+ <!--
+ | Keep this in sync with the version used in
+ | /dev-support/annotatons/pom.xml
+ -->
<checkstyle.version>9.3</checkstyle.version>
<surefire.fork.timeout>1200</surefire.fork.timeout>
<aws-java-sdk.version>1.12.124</aws-java-sdk.version>
@@ -1293,6 +1306,12 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs
<scope>test</scope>
</dependency>
<dependency>
+ <groupId>com.google.testing.compile</groupId>
+ <artifactId>compile-testing</artifactId>
+ <version>0.19</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
<groupId>org.objenesis</groupId>
<artifactId>objenesis</artifactId>
<version>1.0</version>
@@ -1626,6 +1645,13 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xs
</dependency>
</dependencies>
</dependencyManagement>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.ozone</groupId>
+ <artifactId>ozone-annotation-processing</artifactId>
+ <version>${ozone.version}</version>
+ </dependency>
+ </dependencies>
<build>
<extensions>
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@ozone.apache.org
For additional commands, e-mail: commits-help@ozone.apache.org