You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@nifi.apache.org by GitBox <gi...@apache.org> on 2021/07/27 06:21:12 UTC
[GitHub] [nifi-minifi-cpp] szaszm commented on a change in pull request #1143: MINIFICPP-1522 Log invalid attribute in case of YAML parse failure
szaszm commented on a change in pull request #1143:
URL: https://github.com/apache/nifi-minifi-cpp/pull/1143#discussion_r677149209
##########
File path: extensions/standard-processors/tests/unit/YamlConfigurationTests.cpp
##########
@@ -37,160 +37,186 @@ TEST_CASE("Test YAML Config Processing", "[YamlConfiguration]") {
core::YamlConfiguration yamlConfig(testProvRepo, testFlowFileRepo, content_repo, streamFactory, configuration);
SECTION("loading YAML without optional component IDs works") {
- static const std::string CONFIG_YAML_WITHOUT_IDS = ""
- "MiNiFi Config Version: 1\n"
- "Flow Controller:\n"
- " name: MiNiFi Flow\n"
- " comment:\n"
- "\n"
- "Core Properties:\n"
- " flow controller graceful shutdown period: 10 sec\n"
- " flow service write delay interval: 500 ms\n"
- " administrative yield duration: 30 sec\n"
- " bored yield duration: 10 millis\n"
- "\n"
- "FlowFile Repository:\n"
- " partitions: 256\n"
- " checkpoint interval: 2 mins\n"
- " always sync: false\n"
- " Swap:\n"
- " threshold: 20000\n"
- " in period: 5 sec\n"
- " in threads: 1\n"
- " out period: 5 sec\n"
- " out threads: 4\n"
- "\n"
- "Provenance Repository:\n"
- " provenance rollover time: 1 min\n"
- "\n"
- "Content Repository:\n"
- " content claim max appendable size: 10 MB\n"
- " content claim max flow files: 100\n"
- " always sync: false\n"
- "\n"
- "Component Status Repository:\n"
- " buffer size: 1440\n"
- " snapshot frequency: 1 min\n"
- "\n"
- "Security Properties:\n"
- " keystore: /tmp/ssl/localhost-ks.jks\n"
- " keystore type: JKS\n"
- " keystore password: localtest\n"
- " key password: localtest\n"
- " truststore: /tmp/ssl/localhost-ts.jks\n"
- " truststore type: JKS\n"
- " truststore password: localtest\n"
- " ssl protocol: TLS\n"
- " Sensitive Props:\n"
- " key:\n"
- " algorithm: PBEWITHMD5AND256BITAES-CBC-OPENSSL\n"
- " provider: BC\n"
- "\n"
- "Processors:\n"
- " - name: TailFile\n"
- " class: org.apache.nifi.processors.standard.TailFile\n"
- " max concurrent tasks: 1\n"
- " scheduling strategy: TIMER_DRIVEN\n"
- " scheduling period: 1 sec\n"
- " penalization period: 30 sec\n"
- " yield period: 1 sec\n"
- " run duration nanos: 0\n"
- " auto-terminated relationships list:\n"
- " Properties:\n"
- " File to Tail: logs/minifi-app.log\n"
- " Rolling Filename Pattern: minifi-app*\n"
- " Initial Start Position: Beginning of File\n"
- "\n"
- "Connections:\n"
- " - name: TailToS2S\n"
- " source name: TailFile\n"
- " source relationship name: success\n"
- " destination name: 8644cbcc-a45c-40e0-964d-5e536e2ada61\n"
- " max work queue size: 0\n"
- " max work queue data size: 1 MB\n"
- " flowfile expiration: 60 sec\n"
- " queue prioritizer class: org.apache.nifi.prioritizer.NewestFlowFileFirstPrioritizer\n"
- "\n"
- "Remote Processing Groups:\n"
- " - name: NiFi Flow\n"
- " comment:\n"
- " url: https://localhost:8090/nifi\n"
- " timeout: 30 secs\n"
- " yield period: 10 sec\n"
- " Input Ports:\n"
- " - id: 8644cbcc-a45c-40e0-964d-5e536e2ada61\n"
- " name: tailed log\n"
- " comments:\n"
- " max concurrent tasks: 1\n"
- " use compression: false\n"
- "\n"
- "Provenance Reporting:\n"
- " comment:\n"
- " scheduling strategy: TIMER_DRIVEN\n"
- " scheduling period: 30 sec\n"
- " host: localhost\n"
- " port name: provenance\n"
- " port: 8090\n"
- " port uuid: 2f389b8d-83f2-48d3-b465-048f28a1cb56\n"
- " url: https://localhost:8090/\n"
- " originating url: http://${hostname(true)}:8081/nifi\n"
- " use compression: true\n"
- " timeout: 30 secs\n"
- " batch size: 1000";
-
- std::istringstream configYamlStream(CONFIG_YAML_WITHOUT_IDS);
- std::unique_ptr<core::ProcessGroup> rootFlowConfig = yamlConfig.getYamlRoot(configYamlStream);
+ static const std::string CONFIG_YAML_WITHOUT_IDS =
+ R"(
+MiNiFi Config Version: 1
+Flow Controller:
+ name: MiNiFi Flow
+ comment:
- REQUIRE(rootFlowConfig);
- REQUIRE(rootFlowConfig->findProcessorByName("TailFile"));
- utils::Identifier uuid = rootFlowConfig->findProcessorByName("TailFile")->getUUID();
- REQUIRE(uuid);
- REQUIRE(!rootFlowConfig->findProcessorByName("TailFile")->getUUIDStr().empty());
- REQUIRE(1 == rootFlowConfig->findProcessorByName("TailFile")->getMaxConcurrentTasks());
- REQUIRE(
- core::SchedulingStrategy::TIMER_DRIVEN == rootFlowConfig->findProcessorByName("TailFile")->getSchedulingStrategy());
- REQUIRE(1 == rootFlowConfig->findProcessorByName("TailFile")->getMaxConcurrentTasks());
- REQUIRE(1 * 1000 * 1000 * 1000 == rootFlowConfig->findProcessorByName("TailFile")->getSchedulingPeriodNano());
- REQUIRE(std::chrono::seconds(30) == rootFlowConfig->findProcessorByName("TailFile")->getPenalizationPeriod());
- REQUIRE(1 * 1000 == rootFlowConfig->findProcessorByName("TailFile")->getYieldPeriodMsec());
- REQUIRE(0 == rootFlowConfig->findProcessorByName("TailFile")->getRunDurationNano());
+Core Properties:
+ flow controller graceful shutdown period: 10 sec
+ flow service write delay interval: 500 ms
+ administrative yield duration: 30 sec
+ bored yield duration: 10 millis
- std::map<std::string, std::shared_ptr<minifi::Connection>> connectionMap;
- rootFlowConfig->getConnections(connectionMap);
- REQUIRE(2 == connectionMap.size());
- // This is a map of UUID->Connection, and we don't know UUID, so just going to loop over it
- for (auto it : connectionMap) {
- REQUIRE(it.second);
- REQUIRE(!it.second->getUUIDStr().empty());
- REQUIRE(it.second->getDestination());
- REQUIRE(it.second->getSource());
- REQUIRE(60000 == it.second->getFlowExpirationDuration());
+FlowFile Repository:
+ partitions: 256
+ checkpoint interval: 2 mins
+ always sync: false
+ Swap:
+ threshold: 20000
+ in period: 5 sec
+ in threads: 1
+ out period: 5 sec
+ out threads: 4
+
+Provenance Repository:
+ provenance rollover time: 1 min
+
+Content Repository:
+ content claim max appendable size: 10 MB
+ content claim max flow files: 100
+ always sync: false
+
+Component Status Repository:
+ buffer size: 1440
+ snapshot frequency: 1 min
+
+Security Properties:
+ keystore: /tmp/ssl/localhost-ks.jks
+ keystore type: JKS
+ keystore password: localtest
+ key password: localtest
+ truststore: /tmp/ssl/localhost-ts.jks
+ truststore type: JKS
+ truststore password: localtest
+ ssl protocol: TLS
+ Sensitive Props:
+ key:
+ algorithm: PBEWITHMD5AND256BITAES-CBC-OPENSSL
+ provider: BC
+
+Processors:
+ - name: TailFile
+ class: org.apache.nifi.processors.standard.TailFile
+ max concurrent tasks: 1
+ scheduling strategy: TIMER_DRIVEN
+ scheduling period: 1 sec
+ penalization period: 30 sec
+ yield period: 1 sec
+ run duration nanos: 0
+ auto-terminated relationships list:
+ Properties:
+ File to Tail: logs/minifi-app.log
+ Rolling Filename Pattern: minifi-app*
+ Initial Start Position: Beginning of File
+
+Connections:
+ - name: TailToS2S
+ source name: TailFile
+ source relationship name: success
+ destination name: 8644cbcc-a45c-40e0-964d-5e536e2ada61
+ max work queue size: 0
+ max work queue data size: 1 MB
+ flowfile expiration: 60 sec
+ queue prioritizer class: org.apache.nifi.prioritizer.NewestFlowFileFirstPrioritizer
+
+Remote Processing Groups:
+ - name: NiFi Flow
+ comment:
+ url: https://localhost:8090/nifi
+ timeout: 30 secs
+ yield period: 10 sec
+ Input Ports:
+ - id: 8644cbcc-a45c-40e0-964d-5e536e2ada61
+ name: tailed log
+ comments:
+ max concurrent tasks: 1
+ use compression: false
+
+Provenance Reporting:
+ comment:
+ scheduling strategy: TIMER_DRIVEN
+ scheduling period: 30 sec
+ host: localhost
+ port name: provenance
+ port: 8090
+ port uuid: 2f389b8d-83f2-48d3-b465-048f28a1cb56
+ url: https://localhost:8090/
+ originating url: http://${hostname(true)}:8081/nifi
+ use compression: true
+ timeout: 30 secs
+ batch size: 1000;
+ )";
+
+ std::istringstream configYamlStream(CONFIG_YAML_WITHOUT_IDS);
+ std::unique_ptr<core::ProcessGroup> rootFlowConfig = yamlConfig.getYamlRoot(configYamlStream);
+
+ REQUIRE(rootFlowConfig);
+ REQUIRE(rootFlowConfig->findProcessorByName("TailFile"));
+ utils::Identifier uuid = rootFlowConfig->findProcessorByName("TailFile")->getUUID();
+ REQUIRE(uuid);
+ REQUIRE(!rootFlowConfig->findProcessorByName("TailFile")->getUUIDStr().empty());
+ REQUIRE(1 == rootFlowConfig->findProcessorByName("TailFile")->getMaxConcurrentTasks());
+ REQUIRE(
+ core::SchedulingStrategy::TIMER_DRIVEN == rootFlowConfig->findProcessorByName("TailFile")->getSchedulingStrategy());
+ REQUIRE(1 == rootFlowConfig->findProcessorByName("TailFile")->getMaxConcurrentTasks());
Review comment:
Duplicate condition
##########
File path: extensions/standard-processors/tests/unit/YamlConfigurationTests.cpp
##########
@@ -37,160 +37,186 @@ TEST_CASE("Test YAML Config Processing", "[YamlConfiguration]") {
core::YamlConfiguration yamlConfig(testProvRepo, testFlowFileRepo, content_repo, streamFactory, configuration);
SECTION("loading YAML without optional component IDs works") {
- static const std::string CONFIG_YAML_WITHOUT_IDS = ""
- "MiNiFi Config Version: 1\n"
- "Flow Controller:\n"
- " name: MiNiFi Flow\n"
- " comment:\n"
- "\n"
- "Core Properties:\n"
- " flow controller graceful shutdown period: 10 sec\n"
- " flow service write delay interval: 500 ms\n"
- " administrative yield duration: 30 sec\n"
- " bored yield duration: 10 millis\n"
- "\n"
- "FlowFile Repository:\n"
- " partitions: 256\n"
- " checkpoint interval: 2 mins\n"
- " always sync: false\n"
- " Swap:\n"
- " threshold: 20000\n"
- " in period: 5 sec\n"
- " in threads: 1\n"
- " out period: 5 sec\n"
- " out threads: 4\n"
- "\n"
- "Provenance Repository:\n"
- " provenance rollover time: 1 min\n"
- "\n"
- "Content Repository:\n"
- " content claim max appendable size: 10 MB\n"
- " content claim max flow files: 100\n"
- " always sync: false\n"
- "\n"
- "Component Status Repository:\n"
- " buffer size: 1440\n"
- " snapshot frequency: 1 min\n"
- "\n"
- "Security Properties:\n"
- " keystore: /tmp/ssl/localhost-ks.jks\n"
- " keystore type: JKS\n"
- " keystore password: localtest\n"
- " key password: localtest\n"
- " truststore: /tmp/ssl/localhost-ts.jks\n"
- " truststore type: JKS\n"
- " truststore password: localtest\n"
- " ssl protocol: TLS\n"
- " Sensitive Props:\n"
- " key:\n"
- " algorithm: PBEWITHMD5AND256BITAES-CBC-OPENSSL\n"
- " provider: BC\n"
- "\n"
- "Processors:\n"
- " - name: TailFile\n"
- " class: org.apache.nifi.processors.standard.TailFile\n"
- " max concurrent tasks: 1\n"
- " scheduling strategy: TIMER_DRIVEN\n"
- " scheduling period: 1 sec\n"
- " penalization period: 30 sec\n"
- " yield period: 1 sec\n"
- " run duration nanos: 0\n"
- " auto-terminated relationships list:\n"
- " Properties:\n"
- " File to Tail: logs/minifi-app.log\n"
- " Rolling Filename Pattern: minifi-app*\n"
- " Initial Start Position: Beginning of File\n"
- "\n"
- "Connections:\n"
- " - name: TailToS2S\n"
- " source name: TailFile\n"
- " source relationship name: success\n"
- " destination name: 8644cbcc-a45c-40e0-964d-5e536e2ada61\n"
- " max work queue size: 0\n"
- " max work queue data size: 1 MB\n"
- " flowfile expiration: 60 sec\n"
- " queue prioritizer class: org.apache.nifi.prioritizer.NewestFlowFileFirstPrioritizer\n"
- "\n"
- "Remote Processing Groups:\n"
- " - name: NiFi Flow\n"
- " comment:\n"
- " url: https://localhost:8090/nifi\n"
- " timeout: 30 secs\n"
- " yield period: 10 sec\n"
- " Input Ports:\n"
- " - id: 8644cbcc-a45c-40e0-964d-5e536e2ada61\n"
- " name: tailed log\n"
- " comments:\n"
- " max concurrent tasks: 1\n"
- " use compression: false\n"
- "\n"
- "Provenance Reporting:\n"
- " comment:\n"
- " scheduling strategy: TIMER_DRIVEN\n"
- " scheduling period: 30 sec\n"
- " host: localhost\n"
- " port name: provenance\n"
- " port: 8090\n"
- " port uuid: 2f389b8d-83f2-48d3-b465-048f28a1cb56\n"
- " url: https://localhost:8090/\n"
- " originating url: http://${hostname(true)}:8081/nifi\n"
- " use compression: true\n"
- " timeout: 30 secs\n"
- " batch size: 1000";
-
- std::istringstream configYamlStream(CONFIG_YAML_WITHOUT_IDS);
- std::unique_ptr<core::ProcessGroup> rootFlowConfig = yamlConfig.getYamlRoot(configYamlStream);
+ static const std::string CONFIG_YAML_WITHOUT_IDS =
+ R"(
+MiNiFi Config Version: 1
+Flow Controller:
+ name: MiNiFi Flow
+ comment:
- REQUIRE(rootFlowConfig);
- REQUIRE(rootFlowConfig->findProcessorByName("TailFile"));
- utils::Identifier uuid = rootFlowConfig->findProcessorByName("TailFile")->getUUID();
- REQUIRE(uuid);
- REQUIRE(!rootFlowConfig->findProcessorByName("TailFile")->getUUIDStr().empty());
- REQUIRE(1 == rootFlowConfig->findProcessorByName("TailFile")->getMaxConcurrentTasks());
- REQUIRE(
- core::SchedulingStrategy::TIMER_DRIVEN == rootFlowConfig->findProcessorByName("TailFile")->getSchedulingStrategy());
- REQUIRE(1 == rootFlowConfig->findProcessorByName("TailFile")->getMaxConcurrentTasks());
- REQUIRE(1 * 1000 * 1000 * 1000 == rootFlowConfig->findProcessorByName("TailFile")->getSchedulingPeriodNano());
- REQUIRE(std::chrono::seconds(30) == rootFlowConfig->findProcessorByName("TailFile")->getPenalizationPeriod());
- REQUIRE(1 * 1000 == rootFlowConfig->findProcessorByName("TailFile")->getYieldPeriodMsec());
- REQUIRE(0 == rootFlowConfig->findProcessorByName("TailFile")->getRunDurationNano());
+Core Properties:
+ flow controller graceful shutdown period: 10 sec
+ flow service write delay interval: 500 ms
+ administrative yield duration: 30 sec
+ bored yield duration: 10 millis
- std::map<std::string, std::shared_ptr<minifi::Connection>> connectionMap;
- rootFlowConfig->getConnections(connectionMap);
- REQUIRE(2 == connectionMap.size());
- // This is a map of UUID->Connection, and we don't know UUID, so just going to loop over it
- for (auto it : connectionMap) {
- REQUIRE(it.second);
- REQUIRE(!it.second->getUUIDStr().empty());
- REQUIRE(it.second->getDestination());
- REQUIRE(it.second->getSource());
- REQUIRE(60000 == it.second->getFlowExpirationDuration());
+FlowFile Repository:
+ partitions: 256
+ checkpoint interval: 2 mins
+ always sync: false
+ Swap:
+ threshold: 20000
+ in period: 5 sec
+ in threads: 1
+ out period: 5 sec
+ out threads: 4
+
+Provenance Repository:
+ provenance rollover time: 1 min
+
+Content Repository:
+ content claim max appendable size: 10 MB
+ content claim max flow files: 100
+ always sync: false
+
+Component Status Repository:
+ buffer size: 1440
+ snapshot frequency: 1 min
+
+Security Properties:
+ keystore: /tmp/ssl/localhost-ks.jks
+ keystore type: JKS
+ keystore password: localtest
+ key password: localtest
+ truststore: /tmp/ssl/localhost-ts.jks
+ truststore type: JKS
+ truststore password: localtest
+ ssl protocol: TLS
+ Sensitive Props:
+ key:
+ algorithm: PBEWITHMD5AND256BITAES-CBC-OPENSSL
+ provider: BC
+
+Processors:
+ - name: TailFile
+ class: org.apache.nifi.processors.standard.TailFile
+ max concurrent tasks: 1
+ scheduling strategy: TIMER_DRIVEN
+ scheduling period: 1 sec
+ penalization period: 30 sec
+ yield period: 1 sec
+ run duration nanos: 0
+ auto-terminated relationships list:
+ Properties:
+ File to Tail: logs/minifi-app.log
+ Rolling Filename Pattern: minifi-app*
+ Initial Start Position: Beginning of File
+
+Connections:
+ - name: TailToS2S
+ source name: TailFile
+ source relationship name: success
+ destination name: 8644cbcc-a45c-40e0-964d-5e536e2ada61
+ max work queue size: 0
+ max work queue data size: 1 MB
+ flowfile expiration: 60 sec
+ queue prioritizer class: org.apache.nifi.prioritizer.NewestFlowFileFirstPrioritizer
+
+Remote Processing Groups:
+ - name: NiFi Flow
+ comment:
+ url: https://localhost:8090/nifi
+ timeout: 30 secs
+ yield period: 10 sec
+ Input Ports:
+ - id: 8644cbcc-a45c-40e0-964d-5e536e2ada61
+ name: tailed log
+ comments:
+ max concurrent tasks: 1
+ use compression: false
+
+Provenance Reporting:
+ comment:
+ scheduling strategy: TIMER_DRIVEN
+ scheduling period: 30 sec
+ host: localhost
+ port name: provenance
+ port: 8090
+ port uuid: 2f389b8d-83f2-48d3-b465-048f28a1cb56
+ url: https://localhost:8090/
+ originating url: http://${hostname(true)}:8081/nifi
+ use compression: true
+ timeout: 30 secs
+ batch size: 1000;
+ )";
+
+ std::istringstream configYamlStream(CONFIG_YAML_WITHOUT_IDS);
+ std::unique_ptr<core::ProcessGroup> rootFlowConfig = yamlConfig.getYamlRoot(configYamlStream);
+
+ REQUIRE(rootFlowConfig);
+ REQUIRE(rootFlowConfig->findProcessorByName("TailFile"));
+ utils::Identifier uuid = rootFlowConfig->findProcessorByName("TailFile")->getUUID();
+ REQUIRE(uuid);
+ REQUIRE(!rootFlowConfig->findProcessorByName("TailFile")->getUUIDStr().empty());
+ REQUIRE(1 == rootFlowConfig->findProcessorByName("TailFile")->getMaxConcurrentTasks());
+ REQUIRE(
+ core::SchedulingStrategy::TIMER_DRIVEN == rootFlowConfig->findProcessorByName("TailFile")->getSchedulingStrategy());
+ REQUIRE(1 == rootFlowConfig->findProcessorByName("TailFile")->getMaxConcurrentTasks());
+ REQUIRE(1 * 1000 * 1000 * 1000 == rootFlowConfig->findProcessorByName("TailFile")->getSchedulingPeriodNano());
+ REQUIRE(std::chrono::seconds(30) == rootFlowConfig->findProcessorByName("TailFile")->getPenalizationPeriod());
Review comment:
Consider using standard literal suffixes. I also recognize that this is old code just reformatted, so I'm 100% fine with not touching these parts.
##########
File path: extensions/standard-processors/tests/unit/YamlConfigurationTests.cpp
##########
@@ -37,160 +37,186 @@ TEST_CASE("Test YAML Config Processing", "[YamlConfiguration]") {
core::YamlConfiguration yamlConfig(testProvRepo, testFlowFileRepo, content_repo, streamFactory, configuration);
SECTION("loading YAML without optional component IDs works") {
- static const std::string CONFIG_YAML_WITHOUT_IDS = ""
- "MiNiFi Config Version: 1\n"
- "Flow Controller:\n"
- " name: MiNiFi Flow\n"
- " comment:\n"
- "\n"
- "Core Properties:\n"
- " flow controller graceful shutdown period: 10 sec\n"
- " flow service write delay interval: 500 ms\n"
- " administrative yield duration: 30 sec\n"
- " bored yield duration: 10 millis\n"
- "\n"
- "FlowFile Repository:\n"
- " partitions: 256\n"
- " checkpoint interval: 2 mins\n"
- " always sync: false\n"
- " Swap:\n"
- " threshold: 20000\n"
- " in period: 5 sec\n"
- " in threads: 1\n"
- " out period: 5 sec\n"
- " out threads: 4\n"
- "\n"
- "Provenance Repository:\n"
- " provenance rollover time: 1 min\n"
- "\n"
- "Content Repository:\n"
- " content claim max appendable size: 10 MB\n"
- " content claim max flow files: 100\n"
- " always sync: false\n"
- "\n"
- "Component Status Repository:\n"
- " buffer size: 1440\n"
- " snapshot frequency: 1 min\n"
- "\n"
- "Security Properties:\n"
- " keystore: /tmp/ssl/localhost-ks.jks\n"
- " keystore type: JKS\n"
- " keystore password: localtest\n"
- " key password: localtest\n"
- " truststore: /tmp/ssl/localhost-ts.jks\n"
- " truststore type: JKS\n"
- " truststore password: localtest\n"
- " ssl protocol: TLS\n"
- " Sensitive Props:\n"
- " key:\n"
- " algorithm: PBEWITHMD5AND256BITAES-CBC-OPENSSL\n"
- " provider: BC\n"
- "\n"
- "Processors:\n"
- " - name: TailFile\n"
- " class: org.apache.nifi.processors.standard.TailFile\n"
- " max concurrent tasks: 1\n"
- " scheduling strategy: TIMER_DRIVEN\n"
- " scheduling period: 1 sec\n"
- " penalization period: 30 sec\n"
- " yield period: 1 sec\n"
- " run duration nanos: 0\n"
- " auto-terminated relationships list:\n"
- " Properties:\n"
- " File to Tail: logs/minifi-app.log\n"
- " Rolling Filename Pattern: minifi-app*\n"
- " Initial Start Position: Beginning of File\n"
- "\n"
- "Connections:\n"
- " - name: TailToS2S\n"
- " source name: TailFile\n"
- " source relationship name: success\n"
- " destination name: 8644cbcc-a45c-40e0-964d-5e536e2ada61\n"
- " max work queue size: 0\n"
- " max work queue data size: 1 MB\n"
- " flowfile expiration: 60 sec\n"
- " queue prioritizer class: org.apache.nifi.prioritizer.NewestFlowFileFirstPrioritizer\n"
- "\n"
- "Remote Processing Groups:\n"
- " - name: NiFi Flow\n"
- " comment:\n"
- " url: https://localhost:8090/nifi\n"
- " timeout: 30 secs\n"
- " yield period: 10 sec\n"
- " Input Ports:\n"
- " - id: 8644cbcc-a45c-40e0-964d-5e536e2ada61\n"
- " name: tailed log\n"
- " comments:\n"
- " max concurrent tasks: 1\n"
- " use compression: false\n"
- "\n"
- "Provenance Reporting:\n"
- " comment:\n"
- " scheduling strategy: TIMER_DRIVEN\n"
- " scheduling period: 30 sec\n"
- " host: localhost\n"
- " port name: provenance\n"
- " port: 8090\n"
- " port uuid: 2f389b8d-83f2-48d3-b465-048f28a1cb56\n"
- " url: https://localhost:8090/\n"
- " originating url: http://${hostname(true)}:8081/nifi\n"
- " use compression: true\n"
- " timeout: 30 secs\n"
- " batch size: 1000";
-
- std::istringstream configYamlStream(CONFIG_YAML_WITHOUT_IDS);
- std::unique_ptr<core::ProcessGroup> rootFlowConfig = yamlConfig.getYamlRoot(configYamlStream);
+ static const std::string CONFIG_YAML_WITHOUT_IDS =
+ R"(
+MiNiFi Config Version: 1
+Flow Controller:
+ name: MiNiFi Flow
+ comment:
- REQUIRE(rootFlowConfig);
- REQUIRE(rootFlowConfig->findProcessorByName("TailFile"));
- utils::Identifier uuid = rootFlowConfig->findProcessorByName("TailFile")->getUUID();
- REQUIRE(uuid);
- REQUIRE(!rootFlowConfig->findProcessorByName("TailFile")->getUUIDStr().empty());
- REQUIRE(1 == rootFlowConfig->findProcessorByName("TailFile")->getMaxConcurrentTasks());
- REQUIRE(
- core::SchedulingStrategy::TIMER_DRIVEN == rootFlowConfig->findProcessorByName("TailFile")->getSchedulingStrategy());
- REQUIRE(1 == rootFlowConfig->findProcessorByName("TailFile")->getMaxConcurrentTasks());
- REQUIRE(1 * 1000 * 1000 * 1000 == rootFlowConfig->findProcessorByName("TailFile")->getSchedulingPeriodNano());
- REQUIRE(std::chrono::seconds(30) == rootFlowConfig->findProcessorByName("TailFile")->getPenalizationPeriod());
- REQUIRE(1 * 1000 == rootFlowConfig->findProcessorByName("TailFile")->getYieldPeriodMsec());
- REQUIRE(0 == rootFlowConfig->findProcessorByName("TailFile")->getRunDurationNano());
+Core Properties:
+ flow controller graceful shutdown period: 10 sec
+ flow service write delay interval: 500 ms
+ administrative yield duration: 30 sec
+ bored yield duration: 10 millis
- std::map<std::string, std::shared_ptr<minifi::Connection>> connectionMap;
- rootFlowConfig->getConnections(connectionMap);
- REQUIRE(2 == connectionMap.size());
- // This is a map of UUID->Connection, and we don't know UUID, so just going to loop over it
- for (auto it : connectionMap) {
- REQUIRE(it.second);
- REQUIRE(!it.second->getUUIDStr().empty());
- REQUIRE(it.second->getDestination());
- REQUIRE(it.second->getSource());
- REQUIRE(60000 == it.second->getFlowExpirationDuration());
+FlowFile Repository:
+ partitions: 256
+ checkpoint interval: 2 mins
+ always sync: false
+ Swap:
+ threshold: 20000
+ in period: 5 sec
+ in threads: 1
+ out period: 5 sec
+ out threads: 4
+
+Provenance Repository:
+ provenance rollover time: 1 min
+
+Content Repository:
+ content claim max appendable size: 10 MB
+ content claim max flow files: 100
+ always sync: false
+
+Component Status Repository:
+ buffer size: 1440
+ snapshot frequency: 1 min
+
+Security Properties:
+ keystore: /tmp/ssl/localhost-ks.jks
+ keystore type: JKS
+ keystore password: localtest
+ key password: localtest
+ truststore: /tmp/ssl/localhost-ts.jks
+ truststore type: JKS
+ truststore password: localtest
+ ssl protocol: TLS
+ Sensitive Props:
+ key:
+ algorithm: PBEWITHMD5AND256BITAES-CBC-OPENSSL
+ provider: BC
+
+Processors:
+ - name: TailFile
+ class: org.apache.nifi.processors.standard.TailFile
+ max concurrent tasks: 1
+ scheduling strategy: TIMER_DRIVEN
+ scheduling period: 1 sec
+ penalization period: 30 sec
+ yield period: 1 sec
+ run duration nanos: 0
+ auto-terminated relationships list:
+ Properties:
+ File to Tail: logs/minifi-app.log
+ Rolling Filename Pattern: minifi-app*
+ Initial Start Position: Beginning of File
+
+Connections:
+ - name: TailToS2S
+ source name: TailFile
+ source relationship name: success
+ destination name: 8644cbcc-a45c-40e0-964d-5e536e2ada61
+ max work queue size: 0
+ max work queue data size: 1 MB
+ flowfile expiration: 60 sec
+ queue prioritizer class: org.apache.nifi.prioritizer.NewestFlowFileFirstPrioritizer
+
+Remote Processing Groups:
+ - name: NiFi Flow
+ comment:
+ url: https://localhost:8090/nifi
+ timeout: 30 secs
+ yield period: 10 sec
+ Input Ports:
+ - id: 8644cbcc-a45c-40e0-964d-5e536e2ada61
+ name: tailed log
+ comments:
+ max concurrent tasks: 1
+ use compression: false
+
+Provenance Reporting:
+ comment:
+ scheduling strategy: TIMER_DRIVEN
+ scheduling period: 30 sec
+ host: localhost
+ port name: provenance
+ port: 8090
+ port uuid: 2f389b8d-83f2-48d3-b465-048f28a1cb56
+ url: https://localhost:8090/
+ originating url: http://${hostname(true)}:8081/nifi
+ use compression: true
+ timeout: 30 secs
+ batch size: 1000;
+ )";
+
+ std::istringstream configYamlStream(CONFIG_YAML_WITHOUT_IDS);
+ std::unique_ptr<core::ProcessGroup> rootFlowConfig = yamlConfig.getYamlRoot(configYamlStream);
+
+ REQUIRE(rootFlowConfig);
+ REQUIRE(rootFlowConfig->findProcessorByName("TailFile"));
+ utils::Identifier uuid = rootFlowConfig->findProcessorByName("TailFile")->getUUID();
+ REQUIRE(uuid);
+ REQUIRE(!rootFlowConfig->findProcessorByName("TailFile")->getUUIDStr().empty());
+ REQUIRE(1 == rootFlowConfig->findProcessorByName("TailFile")->getMaxConcurrentTasks());
+ REQUIRE(
+ core::SchedulingStrategy::TIMER_DRIVEN == rootFlowConfig->findProcessorByName("TailFile")->getSchedulingStrategy());
+ REQUIRE(1 == rootFlowConfig->findProcessorByName("TailFile")->getMaxConcurrentTasks());
+ REQUIRE(1 * 1000 * 1000 * 1000 == rootFlowConfig->findProcessorByName("TailFile")->getSchedulingPeriodNano());
+ REQUIRE(std::chrono::seconds(30) == rootFlowConfig->findProcessorByName("TailFile")->getPenalizationPeriod());
+ REQUIRE(1 * 1000 == rootFlowConfig->findProcessorByName("TailFile")->getYieldPeriodMsec());
+ REQUIRE(0 == rootFlowConfig->findProcessorByName("TailFile")->getRunDurationNano());
+
+ std::map<std::string, std::shared_ptr<minifi::Connection>> connectionMap;
+ rootFlowConfig->getConnections(connectionMap);
+ REQUIRE(2 == connectionMap.size());
+ // This is a map of UUID->Connection, and we don't know UUID, so just going to loop over it
+ for (auto it : connectionMap) {
+ REQUIRE(it.second);
+ REQUIRE(!it.second->getUUIDStr().empty());
+ REQUIRE(it.second->getDestination());
+ REQUIRE(it.second->getSource());
+ REQUIRE(60000 == it.second->getFlowExpirationDuration());
+ }
}
-}
SECTION("missing required field in YAML throws exception") {
- static const std::string CONFIG_YAML_NO_RPG_PORT_ID = ""
- "MiNiFi Config Version: 1\n"
- "Flow Controller:\n"
- " name: MiNiFi Flow\n"
- "Processors: []\n"
- "Connections: []\n"
- "Remote Processing Groups:\n"
- " - name: NiFi Flow\n"
- " comment:\n"
- " url: https://localhost:8090/nifi\n"
- " timeout: 30 secs\n"
- " yield period: 10 sec\n"
- " Input Ports:\n"
- " - name: tailed log\n"
- " comments:\n"
- " max concurrent tasks: 1\n"
- " use compression: false\n"
- "\n";
-
- std::istringstream configYamlStream(CONFIG_YAML_NO_RPG_PORT_ID);
- REQUIRE_THROWS_AS(yamlConfig.getYamlRoot(configYamlStream), std::invalid_argument);
-}
+ static const std::string CONFIG_YAML_NO_RPG_PORT_ID =
+ R"(
+MiNiFi Config Version: 1
+Flow Controller:
+ name: MiNiFi Flow
+Processors: []
+Connections: []
+Remote Processing Groups:
+ - name: NiFi Flow
+ comment:
+ url: https://localhost:8090/nifi
+ timeout: 30 secs
+ yield period: 10 sec
+ Input Ports:
+ - name: tailed log
+ comments:
+ max concurrent tasks: 1
+ use compression: false
+ )";
+
+ std::istringstream configYamlStream(CONFIG_YAML_NO_RPG_PORT_ID);
+ REQUIRE_THROWS_AS(yamlConfig.getYamlRoot(configYamlStream), std::invalid_argument);
+ }
+
+ SECTION("Validated YAML property failure throws exception and logs invalid attribute name") {
+ static const std::string CONFIG_YAML_EMPTY_RETRY_ATTRIBUTE =
+ R"(
+Flow Controller:
+ name: MiNiFi Flow
+Processors:
+ - name: RetryFlowFile
+ class: org.apache.nifi.processors.standard.RetryFlowFile
+ scheduling strategy: TIMER_DRIVEN
+ scheduling period: 1 sec
+ auto-terminated relationships list:
+ Properties:
+ Retry Attribute: ""
+Connections: []
+Remote Processing Groups: []
+Provenance Reporting:
+ )";
+
+ std::istringstream configYamlStream(CONFIG_YAML_EMPTY_RETRY_ATTRIBUTE);
+ REQUIRE_THROWS_AS(yamlConfig.getYamlRoot(configYamlStream), utils::internal::InvalidValueException);
+ REQUIRE(LogTestController::getInstance().contains("Invalid value was set for property 'Retry Attribute'"));
Review comment:
Can we log the value itself?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@nifi.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org