You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@nifi.apache.org by oz...@apache.org on 2016/11/09 17:12:03 UTC
nifi git commit: NIFI-3008 Enabled expression language on
BASE_DIRECTORY and FILENAME properties in TailFile
Repository: nifi
Updated Branches:
refs/heads/master 3e892c55e -> cc2fbcaac
NIFI-3008 Enabled expression language on BASE_DIRECTORY and FILENAME properties in TailFile
This closes #1190
Project: http://git-wip-us.apache.org/repos/asf/nifi/repo
Commit: http://git-wip-us.apache.org/repos/asf/nifi/commit/cc2fbcaa
Tree: http://git-wip-us.apache.org/repos/asf/nifi/tree/cc2fbcaa
Diff: http://git-wip-us.apache.org/repos/asf/nifi/diff/cc2fbcaa
Branch: refs/heads/master
Commit: cc2fbcaac457ebbfeaada4a0fba84db784bfc9a4
Parents: 3e892c5
Author: Jeff Storck <jt...@gmail.com>
Authored: Tue Nov 8 22:02:03 2016 -0500
Committer: Oleg Zhurakousky <ol...@suitcase.io>
Committed: Wed Nov 9 12:11:11 2016 -0500
----------------------------------------------------------------------
.../nifi/processors/standard/TailFile.java | 21 ++++++++++----------
.../nifi/processors/standard/TestTailFile.java | 21 ++++++++++++++++++++
2 files changed, 32 insertions(+), 10 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/nifi/blob/cc2fbcaa/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/TailFile.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/TailFile.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/TailFile.java
index ba2c9b8..a171977 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/TailFile.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/main/java/org/apache/nifi/processors/standard/TailFile.java
@@ -126,7 +126,7 @@ public class TailFile extends AbstractProcessor {
.name("tail-base-directory")
.displayName("Base directory")
.description("Base directory used to look for files to tail. This property is required when using Multifile mode.")
- .expressionLanguageSupported(false)
+ .expressionLanguageSupported(true)
.addValidator(StandardValidators.FILE_EXISTS_VALIDATOR)
.required(false)
.build();
@@ -148,8 +148,8 @@ public class TailFile extends AbstractProcessor {
.description("Path of the file to tail in case of single file mode. If using multifile mode, regular expression to find files "
+ "to tail in the base directory. In case recursivity is set to true, the regular expression will be used to match the "
+ "path starting from the base directory (see additional details for examples).")
- .expressionLanguageSupported(false)
- .addValidator(StandardValidators.REGULAR_EXPRESSION_VALIDATOR)
+ .expressionLanguageSupported(true)
+ .addValidator(StandardValidators.createRegexValidator(0, Integer.MAX_VALUE, true))
.required(true)
.build();
@@ -267,7 +267,7 @@ public class TailFile extends AbstractProcessor {
final List<ValidationResult> results = new ArrayList<>(super.customValidate(context));
if(context.getProperty(MODE).getValue().equals(MODE_MULTIFILE.getValue())) {
- String path = context.getProperty(BASE_DIRECTORY).getValue();
+ String path = context.getProperty(BASE_DIRECTORY).evaluateAttributeExpressions().getValue();
if(path == null) {
results.add(new ValidationResult.Builder().subject(BASE_DIRECTORY.getName()).valid(false)
.explanation("Base directory property cannot be empty in Multifile mode.").build());
@@ -291,8 +291,8 @@ public class TailFile extends AbstractProcessor {
}
} else {
long max = context.getProperty(MAXIMUM_AGE).getValue() == null ? Long.MAX_VALUE : context.getProperty(MAXIMUM_AGE).asTimePeriod(TimeUnit.MILLISECONDS);
- List<String> filesToTail = getFilesToTail(context.getProperty(BASE_DIRECTORY).getValue(),
- context.getProperty(FILENAME).getValue(),
+ List<String> filesToTail = getFilesToTail(context.getProperty(BASE_DIRECTORY).evaluateAttributeExpressions().getValue(),
+ context.getProperty(FILENAME).evaluateAttributeExpressions().getValue(),
context.getProperty(RECURSIVE).asBoolean(),
max);
@@ -322,12 +322,12 @@ public class TailFile extends AbstractProcessor {
List<String> filesToTail = new ArrayList<String>();
if(context.getProperty(MODE).getValue().equals(MODE_MULTIFILE.getValue())) {
- filesToTail.addAll(getFilesToTail(context.getProperty(BASE_DIRECTORY).getValue(),
- context.getProperty(FILENAME).getValue(),
+ filesToTail.addAll(getFilesToTail(context.getProperty(BASE_DIRECTORY).evaluateAttributeExpressions().getValue(),
+ context.getProperty(FILENAME).evaluateAttributeExpressions().getValue(),
context.getProperty(RECURSIVE).asBoolean(),
maxAge));
} else {
- filesToTail.add(context.getProperty(FILENAME).getValue());
+ filesToTail.add(context.getProperty(FILENAME).evaluateAttributeExpressions().getValue());
}
@@ -1130,7 +1130,8 @@ public class TailFile extends AbstractProcessor {
// use a timestamp of lastModified() + 1 so that we do not ingest this file again.
cleanup();
- tfo.setState(new TailFileState(context.getProperty(FILENAME).getValue(), null, null, 0L, file.lastModified() + 1L, file.length(), null, tfo.getState().getBuffer()));
+ tfo.setState(new TailFileState(context.getProperty(FILENAME).evaluateAttributeExpressions().getValue(), null, null, 0L, file.lastModified() + 1L, file.length(), null,
+ tfo.getState().getBuffer()));
// must ensure that we do session.commit() before persisting state in order to avoid data loss.
session.commit();
http://git-wip-us.apache.org/repos/asf/nifi/blob/cc2fbcaa/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestTailFile.java
----------------------------------------------------------------------
diff --git a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestTailFile.java b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestTailFile.java
index 73b8544..5e49a75 100644
--- a/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestTailFile.java
+++ b/nifi-nar-bundles/nifi-standard-bundle/nifi-standard-processors/src/test/java/org/apache/nifi/processors/standard/TestTailFile.java
@@ -620,6 +620,27 @@ public class TestTailFile {
runner.getFlowFilesForRelationship(TailFile.REL_SUCCESS).get(4).assertContentEquals("1\n");
}
+ @Test
+ public void testMultipleFilesWithBasedirAndFilenameEL() throws IOException, InterruptedException {
+ runner.setVariable("vrBaseDirectory", "target");
+ runner.setProperty(TailFile.BASE_DIRECTORY, "${vrBaseDirectory}");
+ runner.setProperty(TailFile.MODE, TailFile.MODE_MULTIFILE);
+ runner.setVariable("vrFilename", "(testDir/)?log(ging)?.txt");
+ runner.setProperty(TailFile.FILENAME, "${vrFilename}");
+ runner.setProperty(TailFile.ROLLING_FILENAME_PATTERN, "${filename}.?");
+ runner.setProperty(TailFile.START_POSITION, TailFile.START_CURRENT_FILE);
+ runner.setProperty(TailFile.RECURSIVE, "true");
+
+ runner.run(1);
+ runner.assertAllFlowFilesTransferred(TailFile.REL_SUCCESS, 0);
+
+ otherRaf.write("hi\n".getBytes());
+ raf.write("hello\n".getBytes());
+
+ runner.run(1);
+ runner.assertAllFlowFilesTransferred(TailFile.REL_SUCCESS, 2);
+ }
+
/**
* This test is used to check the case where we have multiple files in the same directory
* and where it is not possible to specify a single rolling pattern for all files.