You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by bb...@apache.org on 2020/11/03 20:39:13 UTC
[nifi] 03/07: NIFI-1121: Added property dependencies to MergeContent
This is an automated email from the ASF dual-hosted git repository.
bbende pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/nifi.git
commit e2e901b6b97c8099038230b430905bd5c85bf6ae
Author: Mark Payne <ma...@hotmail.com>
AuthorDate: Fri Aug 14 16:42:47 2020 -0400
NIFI-1121: Added property dependencies to MergeContent
Signed-off-by: Bryan Bende <bb...@apache.org>
---
.../nifi/processors/standard/MergeContent.java | 60 +++++++++++++---------
1 file changed, 37 insertions(+), 23 deletions(-)
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/MergeContent.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/MergeContent.java
index 5d4506c..215822e 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/MergeContent.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/MergeContent.java
@@ -166,19 +166,6 @@ public class MergeContent extends BinFiles {
public static final AllowableValue METADATA_STRATEGY_DO_NOT_MERGE = new AllowableValue("Do Not Merge Uncommon Metadata", "Do Not Merge Uncommon Metadata",
"For any input format that supports metadata (Avro, e.g.), any FlowFile whose metadata values do not match those of the first FlowFile in the bin will not be merged.");
- public static final PropertyDescriptor METADATA_STRATEGY = new PropertyDescriptor.Builder()
- .required(true)
- .name("mergecontent-metadata-strategy")
- .displayName("Metadata Strategy")
- .description("For FlowFiles whose input format supports metadata (Avro, e.g.), this property determines which metadata should be added to the bundle. "
- + "If 'Use First Metadata' is selected, the metadata keys/values from the first FlowFile to be bundled will be used. If 'Keep Only Common Metadata' is selected, "
- + "only the metadata that exists on all FlowFiles in the bundle, with the same value, will be preserved. If 'Ignore Metadata' is selected, no metadata is transferred to "
- + "the outgoing bundled FlowFile. If 'Do Not Merge Uncommon Metadata' is selected, any FlowFile whose metadata values do not match those of the first bundled FlowFile "
- + "will not be merged.")
- .allowableValues(METADATA_STRATEGY_USE_FIRST, METADATA_STRATEGY_ALL_COMMON, METADATA_STRATEGY_DO_NOT_MERGE, METADATA_STRATEGY_IGNORE)
- .defaultValue(METADATA_STRATEGY_DO_NOT_MERGE.getValue())
- .build();
-
public static final AllowableValue MERGE_STRATEGY_BIN_PACK = new AllowableValue(
"Bin-Packing Algorithm",
"Bin-Packing Algorithm",
@@ -197,6 +184,8 @@ public class MergeContent extends BinFiles {
"Filename", "Filename", "The values of Header, Footer, and Demarcator will be retrieved from the contents of a file");
public static final AllowableValue DELIMITER_STRATEGY_TEXT = new AllowableValue(
"Text", "Text", "The values of Header, Footer, and Demarcator will be specified as property values");
+ public static final AllowableValue DELIMITER_STRATEGY_NONE = new AllowableValue(
+ "Do Not Use Delimiters", "Do Not Use Delimiters", "No Header, Footer, or Demarcator will be used");
public static final String MERGE_FORMAT_TAR_VALUE = "TAR";
public static final String MERGE_FORMAT_ZIP_VALUE = "ZIP";
@@ -263,6 +252,20 @@ public class MergeContent extends BinFiles {
.defaultValue(MERGE_FORMAT_CONCAT.getValue())
.build();
+ public static final PropertyDescriptor METADATA_STRATEGY = new PropertyDescriptor.Builder()
+ .required(true)
+ .name("mergecontent-metadata-strategy")
+ .displayName("Metadata Strategy")
+ .description("For FlowFiles whose input format supports metadata (Avro, e.g.), this property determines which metadata should be added to the bundle. "
+ + "If 'Use First Metadata' is selected, the metadata keys/values from the first FlowFile to be bundled will be used. If 'Keep Only Common Metadata' is selected, "
+ + "only the metadata that exists on all FlowFiles in the bundle, with the same value, will be preserved. If 'Ignore Metadata' is selected, no metadata is transferred to "
+ + "the outgoing bundled FlowFile. If 'Do Not Merge Uncommon Metadata' is selected, any FlowFile whose metadata values do not match those of the first bundled FlowFile "
+ + "will not be merged.")
+ .allowableValues(METADATA_STRATEGY_USE_FIRST, METADATA_STRATEGY_ALL_COMMON, METADATA_STRATEGY_DO_NOT_MERGE, METADATA_STRATEGY_IGNORE)
+ .defaultValue(METADATA_STRATEGY_DO_NOT_MERGE.getValue())
+ .dependsOn(MERGE_FORMAT, MERGE_FORMAT_AVRO)
+ .build();
+
public static final PropertyDescriptor CORRELATION_ATTRIBUTE_NAME = new PropertyDescriptor.Builder()
.name("Correlation Attribute Name")
.description("If specified, like FlowFiles will be binned together, where 'like FlowFiles' means FlowFiles that have the same value for "
@@ -271,6 +274,7 @@ public class MergeContent extends BinFiles {
.expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES)
.addValidator(StandardValidators.ATTRIBUTE_KEY_VALIDATOR)
.defaultValue(null)
+ .dependsOn(MERGE_STRATEGY, MERGE_STRATEGY_BIN_PACK)
.build();
public static final PropertyDescriptor DELIMITER_STRATEGY = new PropertyDescriptor.Builder()
@@ -278,8 +282,9 @@ public class MergeContent extends BinFiles {
.name("Delimiter Strategy")
.description("Determines if Header, Footer, and Demarcator should point to files containing the respective content, or if "
+ "the values of the properties should be used as the content.")
- .allowableValues(DELIMITER_STRATEGY_FILENAME, DELIMITER_STRATEGY_TEXT)
- .defaultValue(DELIMITER_STRATEGY_FILENAME.getValue())
+ .allowableValues(DELIMITER_STRATEGY_NONE, DELIMITER_STRATEGY_FILENAME, DELIMITER_STRATEGY_TEXT)
+ .defaultValue(DELIMITER_STRATEGY_NONE.getValue())
+ .dependsOn(MERGE_STRATEGY, MERGE_STRATEGY_BIN_PACK)
.build();
public static final PropertyDescriptor HEADER = new PropertyDescriptor.Builder()
.name("Header File")
@@ -289,6 +294,7 @@ public class MergeContent extends BinFiles {
.required(false)
.addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
.expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES)
+ .dependsOn(DELIMITER_STRATEGY, DELIMITER_STRATEGY_FILENAME, DELIMITER_STRATEGY_TEXT)
.build();
public static final PropertyDescriptor FOOTER = new PropertyDescriptor.Builder()
.name("Footer File")
@@ -298,6 +304,7 @@ public class MergeContent extends BinFiles {
.required(false)
.addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
.expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES)
+ .dependsOn(DELIMITER_STRATEGY, DELIMITER_STRATEGY_FILENAME, DELIMITER_STRATEGY_TEXT)
.build();
public static final PropertyDescriptor DEMARCATOR = new PropertyDescriptor.Builder()
.name("Demarcator File")
@@ -307,6 +314,7 @@ public class MergeContent extends BinFiles {
.required(false)
.addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
.expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES)
+ .dependsOn(DELIMITER_STRATEGY, DELIMITER_STRATEGY_FILENAME, DELIMITER_STRATEGY_TEXT)
.build();
public static final PropertyDescriptor COMPRESSION_LEVEL = new PropertyDescriptor.Builder()
.name("Compression Level")
@@ -315,24 +323,25 @@ public class MergeContent extends BinFiles {
.required(true)
.allowableValues("0", "1", "2", "3", "4", "5", "6", "7", "8", "9")
.defaultValue("1")
+ .dependsOn(MERGE_FORMAT, MERGE_FORMAT_ZIP)
.build();
public static final PropertyDescriptor KEEP_PATH = new PropertyDescriptor.Builder()
.name("Keep Path")
- .description("If using the Zip or Tar Merge Format, specifies whether or not the FlowFiles' paths should be included in their entry "
- + "names; if using other merge strategy, this value is ignored")
+ .description("If using the Zip or Tar Merge Format, specifies whether or not the FlowFiles' paths should be included in their entry names.")
.required(true)
.allowableValues("true", "false")
.defaultValue("false")
+ .dependsOn(MERGE_FORMAT, MERGE_FORMAT_TAR, MERGE_FORMAT_ZIP)
.build();
public static final PropertyDescriptor TAR_MODIFIED_TIME = new PropertyDescriptor.Builder()
.name("Tar Modified Time")
.description("If using the Tar Merge Format, specifies if the Tar entry should store the modified timestamp either by expression "
- + "(e.g. ${file.lastModifiedTime} or static value, both of which must match the ISO8601 format 'yyyy-MM-dd'T'HH:mm:ssZ'; if using "
- + "other merge strategy or left blank, this value is ignored")
+ + "(e.g. ${file.lastModifiedTime} or static value, both of which must match the ISO8601 format 'yyyy-MM-dd'T'HH:mm:ssZ'.")
.required(false)
.expressionLanguageSupported(ExpressionLanguageScope.FLOWFILE_ATTRIBUTES)
.addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
.defaultValue("${file.lastModifiedTime}")
+ .dependsOn(MERGE_FORMAT, MERGE_FORMAT_TAR)
.build();
public static final Relationship REL_MERGED = new Relationship.Builder().name("merged").description("The FlowFile containing the merged content").build();
@@ -356,10 +365,10 @@ public class MergeContent extends BinFiles {
descriptors.add(AttributeStrategyUtil.ATTRIBUTE_STRATEGY);
descriptors.add(CORRELATION_ATTRIBUTE_NAME);
descriptors.add(METADATA_STRATEGY);
- descriptors.add(MIN_ENTRIES);
- descriptors.add(MAX_ENTRIES);
- descriptors.add(MIN_SIZE);
- descriptors.add(MAX_SIZE);
+ descriptors.add(addBinPackingDependency(MIN_ENTRIES));
+ descriptors.add(addBinPackingDependency(MAX_ENTRIES));
+ descriptors.add(addBinPackingDependency(MIN_SIZE));
+ descriptors.add(addBinPackingDependency(MAX_SIZE));
descriptors.add(MAX_BIN_AGE);
descriptors.add(MAX_BIN_COUNT);
descriptors.add(DELIMITER_STRATEGY);
@@ -372,6 +381,11 @@ public class MergeContent extends BinFiles {
return descriptors;
}
+ // Convenience method to make creation of property descriptors cleaner
+ private PropertyDescriptor addBinPackingDependency(final PropertyDescriptor original) {
+ return new PropertyDescriptor.Builder().fromPropertyDescriptor(original).dependsOn(MERGE_STRATEGY, MERGE_STRATEGY_BIN_PACK).build();
+ }
+
@Override
protected Collection<ValidationResult> additionalCustomValidation(ValidationContext context) {
final Collection<ValidationResult> results = new ArrayList<>();