You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@nifi.apache.org by GitBox <gi...@apache.org> on 2022/09/08 13:17:29 UTC
[GitHub] [nifi] ferencerdei opened a new pull request, #6376: NIFI-10455 Get workday report processor
ferencerdei opened a new pull request, #6376:
URL: https://github.com/apache/nifi/pull/6376
<!-- Licensed to the Apache Software Foundation (ASF) under one or more -->
<!-- contributor license agreements. See the NOTICE file distributed with -->
<!-- this work for additional information regarding copyright ownership. -->
<!-- The ASF licenses this file to You under the Apache License, Version 2.0 -->
<!-- (the "License"); you may not use this file except in compliance with -->
<!-- the License. You may obtain a copy of the License at -->
<!-- http://www.apache.org/licenses/LICENSE-2.0 -->
<!-- Unless required by applicable law or agreed to in writing, software -->
<!-- distributed under the License is distributed on an "AS IS" BASIS, -->
<!-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -->
<!-- See the License for the specific language governing permissions and -->
<!-- limitations under the License. -->
# Summary
Adding a processor which can fetch data from Workday RaaS, transform the response, or hash specific fields.
[NIFI-10455](https://issues.apache.org/jira/browse/NIFI-10455)
# Tracking
Please complete the following tracking steps prior to pull request creation.
### Issue Tracking
- [ ] [Apache NiFi Jira](https://issues.apache.org/jira/browse/NIFI) issue created
### Pull Request Tracking
- [ ] Pull Request title starts with Apache NiFi Jira issue number, such as `NIFI-00000`
- [ ] Pull Request commit message starts with Apache NiFi Jira issue number, as such `NIFI-00000`
### Pull Request Formatting
- [ ] Pull Request based on current revision of the `main` branch
- [ ] Pull Request refers to a feature branch with one commit containing changes
# Verification
Please indicate the verification steps performed prior to pull request creation.
### Build
- [ ] Build completed using `mvn clean install -P contrib-check`
- [ ] JDK 8
- [ ] JDK 11
- [ ] JDK 17
### Licensing
- [ ] New dependencies are compatible with the [Apache License 2.0](https://apache.org/licenses/LICENSE-2.0) according to the [License Policy](https://www.apache.org/legal/resolved.html)
- [ ] New dependencies are documented in applicable `LICENSE` and `NOTICE` files
### Documentation
- [ ] Documentation formatting appears as expected in rendered files
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@nifi.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [nifi] ferencerdei commented on a diff in pull request #6376: NIFI-10455 Get workday report processor
Posted by GitBox <gi...@apache.org>.
ferencerdei commented on code in PR #6376:
URL: https://github.com/apache/nifi/pull/6376#discussion_r968132771
##########
nifi-nar-bundles/nifi-workday-bundle/nifi-workday-processors/src/main/java/org/apache/nifi/processors/workday/GetWorkdayReport.java:
##########
@@ -0,0 +1,432 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.processors.workday;
+
+import static org.apache.nifi.expression.ExpressionLanguageScope.FLOWFILE_ATTRIBUTES;
+import static org.apache.nifi.expression.ExpressionLanguageScope.NONE;
+import static org.apache.nifi.processor.util.StandardValidators.NON_BLANK_VALIDATOR;
+import static org.apache.nifi.processor.util.StandardValidators.URL_VALIDATOR;
+
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Base64;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import org.apache.nifi.annotation.behavior.EventDriven;
+import org.apache.nifi.annotation.behavior.InputRequirement;
+import org.apache.nifi.annotation.behavior.InputRequirement.Requirement;
+import org.apache.nifi.annotation.behavior.SideEffectFree;
+import org.apache.nifi.annotation.behavior.SupportsBatching;
+import org.apache.nifi.annotation.behavior.WritesAttribute;
+import org.apache.nifi.annotation.behavior.WritesAttributes;
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.annotation.lifecycle.OnScheduled;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.ValidationContext;
+import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.flowfile.attributes.CoreAttributes;
+import org.apache.nifi.logging.ComponentLog;
+import org.apache.nifi.processor.AbstractProcessor;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.processor.util.StandardValidators;
+import org.apache.nifi.record.path.RecordPath;
+import org.apache.nifi.record.path.RecordPathResult;
+import org.apache.nifi.schema.access.SchemaNotFoundException;
+import org.apache.nifi.security.util.crypto.HashAlgorithm;
+import org.apache.nifi.security.util.crypto.HashService;
+import org.apache.nifi.serialization.MalformedRecordException;
+import org.apache.nifi.serialization.RecordReader;
+import org.apache.nifi.serialization.RecordReaderFactory;
+import org.apache.nifi.serialization.RecordSetWriter;
+import org.apache.nifi.serialization.RecordSetWriterFactory;
+import org.apache.nifi.serialization.record.Record;
+import org.apache.nifi.serialization.record.RecordFieldType;
+import org.apache.nifi.serialization.record.RecordSchema;
+import org.apache.nifi.web.client.api.HttpResponseEntity;
+import org.apache.nifi.web.client.api.WebClientService;
+import org.apache.nifi.web.client.provider.api.WebClientServiceProvider;
+
+@Tags({"Workday", "report", "get"})
+@InputRequirement(Requirement.INPUT_ALLOWED)
+@CapabilityDescription("A processor which can interact with a configurable Workday Report. The processor can forward the content without modification, or you can transform it by"
+ + " providing the specific Record Reader and Record Writer services based on your needs. You can also hash, or remove fields using the input parameters and schemes in the Record writer. "
+ + "Supported Workday report formats are: csv, simplexml, json")
+@EventDriven
+@SideEffectFree
+@SupportsBatching
+@WritesAttributes({
+ @WritesAttribute(attribute = GetWorkdayReport.GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS, description = "The Java exception class raised when the processor fails"),
+ @WritesAttribute(attribute = GetWorkdayReport.GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE, description = "The Java exception message raised when the processor fails"),
+ @WritesAttribute(attribute = "mime.type", description = "Sets the mime.type attribute to the MIME Type specified by the Source / Record Writer"),
+ @WritesAttribute(attribute= GetWorkdayReport.RECORD_COUNT, description = "The number of records in an outgoing FlowFile. This is only populated on the 'success' relationship "
+ + "when Record Reader and Writer is set.")})
+public class GetWorkdayReport extends AbstractProcessor {
+
+ protected static final String STATUS_CODE = "getworkdayreport.status.code";
+ protected static final String REQUEST_URL = "getworkdayreport.request.url";
+ protected static final String REQUEST_DURATION = "getworkdayreport.request.duration";
+ protected static final String TRANSACTION_ID = "getworkdayreport.tx.id";
+ protected static final String GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS = "getworkdayreport.java.exception.class";
+ protected static final String GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE = "getworkdayreport.java.exception.message";
+ protected static final String RECORD_COUNT = "record.count";
+ protected static final String BASIC_PREFIX = "Basic ";
+ protected static final String COLUMNS_TO_HASH_SEPARATOR = ",";
+ protected static final String HEADER_AUTHORIZATION = "Authorization";
+ protected static final String HEADER_CONTENT_TYPE = "Content-Type";
+ protected static final String USERNAME_PASSWORD_SEPARATOR = ":";
+
+ protected static final PropertyDescriptor REPORT_URL = new PropertyDescriptor.Builder()
+ .name("Workday report URL")
+ .displayName("Workday report URL")
+ .description("HTTP remote URL of Workday report including a scheme of http or https, as well as a hostname or IP address with optional port and path elements.")
+ .required(true)
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .addValidator(URL_VALIDATOR)
+ .build();
+
+ protected static final PropertyDescriptor WORKDAY_USERNAME = new PropertyDescriptor.Builder()
+ .name("Workday Username")
+ .displayName("Workday Username")
+ .description("The username provided for authentication of Workday requests. Encoded using Base64 for HTTP Basic Authentication as described in RFC 7617.")
+ .required(true)
+ .addValidator(StandardValidators.createRegexMatchingValidator(Pattern.compile("^[\\x20-\\x39\\x3b-\\x7e\\x80-\\xff]+$")))
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .build();
+
+ protected static final PropertyDescriptor WORKDAY_PASSWORD = new PropertyDescriptor.Builder()
+ .name("Workday Password")
+ .displayName("Workday Password")
+ .description("The password provided for authentication of Workday requests. Encoded using Base64 for HTTP Basic Authentication as described in RFC 7617.")
+ .required(true)
+ .sensitive(true)
+ .addValidator(StandardValidators.createRegexMatchingValidator(Pattern.compile("^[\\x20-\\x7e\\x80-\\xff]+$")))
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .build();
+
+ protected static final PropertyDescriptor WEB_CLIENT_SERVICE = new PropertyDescriptor.Builder()
+ .name("Standard Web Client Service")
+ .description("Web client which is used to communicate with the Workday API.")
+ .required(true)
+ .identifiesControllerService(WebClientServiceProvider.class)
+ .build();
+
+ protected static final PropertyDescriptor FIELDS_TO_HASH = new PropertyDescriptor.Builder()
+ .name("Fields to hash")
+ .displayName("Fields to hash")
+ .description("Comma separated record paths to replace with hash.")
+ .required(false)
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .addValidator(NON_BLANK_VALIDATOR)
+ .build();
+
+ protected static final PropertyDescriptor HASHING_ALGORITHM = new PropertyDescriptor.Builder()
+ .name("Hashing algorithm")
+ .displayName("Hashing algorithm")
+ .description("Determines what hashing algorithm should be used to perform the hashing function.")
+ .required(true)
+ .allowableValues(HashService.buildHashAlgorithmAllowableValues())
+ .defaultValue(HashAlgorithm.SHA256.getName())
+ .addValidator(NON_BLANK_VALIDATOR)
+ .expressionLanguageSupported(NONE)
+ .dependsOn(FIELDS_TO_HASH)
+ .build();
+
+ protected static final PropertyDescriptor RECORD_READER_FACTORY = new PropertyDescriptor.Builder()
+ .name("record-reader")
+ .displayName("Record Reader")
+ .description("Specifies the Controller Service to use for parsing incoming data and determining the data's schema.")
+ .identifiesControllerService(RecordReaderFactory.class)
+ .required(false)
+ .build();
+
+ protected static final PropertyDescriptor RECORD_WRITER_FACTORY = new PropertyDescriptor.Builder()
+ .name("record-writer")
+ .displayName("Record Writer")
+ .description("The Record Writer to use for serializing Records to an output FlowFile.")
+ .identifiesControllerService(RecordSetWriterFactory.class)
+ .dependsOn(RECORD_READER_FACTORY)
+ .required(true)
+ .build();
+
+ protected static final Relationship ORIGINAL = new Relationship.Builder()
+ .name("Original")
+ .description("Request FlowFiles transferred when receiving HTTP responses with a status code between 200 and 299.")
+ .build();
+
+ protected static final Relationship FAILURE = new Relationship.Builder()
+ .name("Failure")
+ .description("Request FlowFiles transferred when receiving socket communication errors.")
+ .build();
+
+ protected static final Relationship RESPONSE = new Relationship.Builder()
+ .name("Response")
+ .description("Response FlowFiles transferred when receiving HTTP responses with a status code between 200 and 299.")
+ .build();
+
+ protected static final Set<Relationship> RELATIONSHIPS = Collections.unmodifiableSet(new HashSet<>(Arrays.asList(ORIGINAL, RESPONSE, FAILURE)));
+ protected static final List<PropertyDescriptor> PROPERTIES = Collections.unmodifiableList(Arrays.asList(REPORT_URL, WORKDAY_USERNAME, WORKDAY_PASSWORD, WEB_CLIENT_SERVICE,
+ FIELDS_TO_HASH, HASHING_ALGORITHM, RECORD_READER_FACTORY, RECORD_WRITER_FACTORY));
+
+ private final AtomicReference<WebClientService> webClientAtomicReference = new AtomicReference<>();
+ private final AtomicReference<RecordReaderFactory> recordReaderFactoryAtomicReference = new AtomicReference<>();
+ private final AtomicReference<RecordSetWriterFactory> recordSetWriterFactoryAtomicReference = new AtomicReference<>();
+
+ @Override
+ protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
+ return PROPERTIES;
+ }
+
+ @Override
+ public Set<Relationship> getRelationships() {
+ return RELATIONSHIPS;
+ }
+
+ @OnScheduled
+ public void setUpClient(final ProcessContext context) {
+ WebClientServiceProvider standardWebClientServiceProvider = context.getProperty(WEB_CLIENT_SERVICE).asControllerService(WebClientServiceProvider.class);
+ RecordReaderFactory recordReaderFactory = context.getProperty(RECORD_READER_FACTORY).asControllerService(RecordReaderFactory.class);
+ RecordSetWriterFactory recordSetWriterFactory = context.getProperty(RECORD_WRITER_FACTORY).asControllerService(RecordSetWriterFactory.class);
+ WebClientService webClientService = standardWebClientServiceProvider.getWebClientService();
+ webClientAtomicReference.set(webClientService);
+ recordReaderFactoryAtomicReference.set(recordReaderFactory);
+ recordSetWriterFactoryAtomicReference.set(recordSetWriterFactory);
+ }
+
+ @Override
+ public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException {
+ FlowFile flowfile = session.get();
+
+ if (skipExecution(context, flowfile)) {
+ return;
+ }
+
+ ComponentLog logger = getLogger();
+ FlowFile responseFlowFile = null;
+
+ try {
+ WebClientService webClientService = webClientAtomicReference.get();
+ URI uri = new URI(context.getProperty(REPORT_URL).evaluateAttributeExpressions(flowfile).getValue().trim());
+ long startNanos = System.nanoTime();
+ String authorization = createAuthorizationHeader(context, flowfile);
+
+ try(HttpResponseEntity httpResponseEntity = webClientService.get().uri(uri).header(HEADER_AUTHORIZATION, authorization).retrieve()) {
+ responseFlowFile = createResponseFlowFile(flowfile, session, context, httpResponseEntity);
+ long elapsedTime = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos);
+ Map<String, String> commonAttributes = createCommonAttributes(uri, httpResponseEntity, elapsedTime);
+
+ if (flowfile != null) {
+ flowfile = session.putAllAttributes(flowfile, decorateWithMimeAttribute(commonAttributes, httpResponseEntity));
+ }
+ if (responseFlowFile != null) {
+ responseFlowFile = session.putAllAttributes(responseFlowFile, commonAttributes);
+ if (flowfile != null) {
+ session.getProvenanceReporter().fetch(responseFlowFile, uri.toString(), elapsedTime);
+ } else {
+ session.getProvenanceReporter().receive(responseFlowFile, uri.toString(), elapsedTime);
+ }
Review Comment:
done
##########
nifi-nar-bundles/nifi-workday-bundle/nifi-workday-processors/src/main/java/org/apache/nifi/processors/workday/GetWorkdayReport.java:
##########
@@ -0,0 +1,432 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.processors.workday;
+
+import static org.apache.nifi.expression.ExpressionLanguageScope.FLOWFILE_ATTRIBUTES;
+import static org.apache.nifi.expression.ExpressionLanguageScope.NONE;
+import static org.apache.nifi.processor.util.StandardValidators.NON_BLANK_VALIDATOR;
+import static org.apache.nifi.processor.util.StandardValidators.URL_VALIDATOR;
+
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Base64;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import org.apache.nifi.annotation.behavior.EventDriven;
+import org.apache.nifi.annotation.behavior.InputRequirement;
+import org.apache.nifi.annotation.behavior.InputRequirement.Requirement;
+import org.apache.nifi.annotation.behavior.SideEffectFree;
+import org.apache.nifi.annotation.behavior.SupportsBatching;
+import org.apache.nifi.annotation.behavior.WritesAttribute;
+import org.apache.nifi.annotation.behavior.WritesAttributes;
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.annotation.lifecycle.OnScheduled;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.ValidationContext;
+import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.flowfile.attributes.CoreAttributes;
+import org.apache.nifi.logging.ComponentLog;
+import org.apache.nifi.processor.AbstractProcessor;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.processor.util.StandardValidators;
+import org.apache.nifi.record.path.RecordPath;
+import org.apache.nifi.record.path.RecordPathResult;
+import org.apache.nifi.schema.access.SchemaNotFoundException;
+import org.apache.nifi.security.util.crypto.HashAlgorithm;
+import org.apache.nifi.security.util.crypto.HashService;
+import org.apache.nifi.serialization.MalformedRecordException;
+import org.apache.nifi.serialization.RecordReader;
+import org.apache.nifi.serialization.RecordReaderFactory;
+import org.apache.nifi.serialization.RecordSetWriter;
+import org.apache.nifi.serialization.RecordSetWriterFactory;
+import org.apache.nifi.serialization.record.Record;
+import org.apache.nifi.serialization.record.RecordFieldType;
+import org.apache.nifi.serialization.record.RecordSchema;
+import org.apache.nifi.web.client.api.HttpResponseEntity;
+import org.apache.nifi.web.client.api.WebClientService;
+import org.apache.nifi.web.client.provider.api.WebClientServiceProvider;
+
+@Tags({"Workday", "report", "get"})
+@InputRequirement(Requirement.INPUT_ALLOWED)
+@CapabilityDescription("A processor which can interact with a configurable Workday Report. The processor can forward the content without modification, or you can transform it by"
+ + " providing the specific Record Reader and Record Writer services based on your needs. You can also hash, or remove fields using the input parameters and schemes in the Record writer. "
+ + "Supported Workday report formats are: csv, simplexml, json")
+@EventDriven
+@SideEffectFree
+@SupportsBatching
+@WritesAttributes({
+ @WritesAttribute(attribute = GetWorkdayReport.GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS, description = "The Java exception class raised when the processor fails"),
+ @WritesAttribute(attribute = GetWorkdayReport.GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE, description = "The Java exception message raised when the processor fails"),
+ @WritesAttribute(attribute = "mime.type", description = "Sets the mime.type attribute to the MIME Type specified by the Source / Record Writer"),
+ @WritesAttribute(attribute= GetWorkdayReport.RECORD_COUNT, description = "The number of records in an outgoing FlowFile. This is only populated on the 'success' relationship "
+ + "when Record Reader and Writer is set.")})
+public class GetWorkdayReport extends AbstractProcessor {
+
+ protected static final String STATUS_CODE = "getworkdayreport.status.code";
+ protected static final String REQUEST_URL = "getworkdayreport.request.url";
+ protected static final String REQUEST_DURATION = "getworkdayreport.request.duration";
+ protected static final String TRANSACTION_ID = "getworkdayreport.tx.id";
+ protected static final String GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS = "getworkdayreport.java.exception.class";
+ protected static final String GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE = "getworkdayreport.java.exception.message";
+ protected static final String RECORD_COUNT = "record.count";
+ protected static final String BASIC_PREFIX = "Basic ";
+ protected static final String COLUMNS_TO_HASH_SEPARATOR = ",";
+ protected static final String HEADER_AUTHORIZATION = "Authorization";
+ protected static final String HEADER_CONTENT_TYPE = "Content-Type";
+ protected static final String USERNAME_PASSWORD_SEPARATOR = ":";
+
+ protected static final PropertyDescriptor REPORT_URL = new PropertyDescriptor.Builder()
+ .name("Workday report URL")
+ .displayName("Workday report URL")
+ .description("HTTP remote URL of Workday report including a scheme of http or https, as well as a hostname or IP address with optional port and path elements.")
+ .required(true)
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .addValidator(URL_VALIDATOR)
+ .build();
+
+ protected static final PropertyDescriptor WORKDAY_USERNAME = new PropertyDescriptor.Builder()
+ .name("Workday Username")
+ .displayName("Workday Username")
+ .description("The username provided for authentication of Workday requests. Encoded using Base64 for HTTP Basic Authentication as described in RFC 7617.")
+ .required(true)
+ .addValidator(StandardValidators.createRegexMatchingValidator(Pattern.compile("^[\\x20-\\x39\\x3b-\\x7e\\x80-\\xff]+$")))
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .build();
+
+ protected static final PropertyDescriptor WORKDAY_PASSWORD = new PropertyDescriptor.Builder()
+ .name("Workday Password")
+ .displayName("Workday Password")
+ .description("The password provided for authentication of Workday requests. Encoded using Base64 for HTTP Basic Authentication as described in RFC 7617.")
+ .required(true)
+ .sensitive(true)
+ .addValidator(StandardValidators.createRegexMatchingValidator(Pattern.compile("^[\\x20-\\x7e\\x80-\\xff]+$")))
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .build();
+
+ protected static final PropertyDescriptor WEB_CLIENT_SERVICE = new PropertyDescriptor.Builder()
+ .name("Standard Web Client Service")
+ .description("Web client which is used to communicate with the Workday API.")
+ .required(true)
+ .identifiesControllerService(WebClientServiceProvider.class)
+ .build();
+
+ protected static final PropertyDescriptor FIELDS_TO_HASH = new PropertyDescriptor.Builder()
+ .name("Fields to hash")
+ .displayName("Fields to hash")
+ .description("Comma separated record paths to replace with hash.")
+ .required(false)
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .addValidator(NON_BLANK_VALIDATOR)
+ .build();
+
+ protected static final PropertyDescriptor HASHING_ALGORITHM = new PropertyDescriptor.Builder()
+ .name("Hashing algorithm")
+ .displayName("Hashing algorithm")
+ .description("Determines what hashing algorithm should be used to perform the hashing function.")
+ .required(true)
+ .allowableValues(HashService.buildHashAlgorithmAllowableValues())
+ .defaultValue(HashAlgorithm.SHA256.getName())
+ .addValidator(NON_BLANK_VALIDATOR)
+ .expressionLanguageSupported(NONE)
+ .dependsOn(FIELDS_TO_HASH)
+ .build();
+
+ protected static final PropertyDescriptor RECORD_READER_FACTORY = new PropertyDescriptor.Builder()
+ .name("record-reader")
+ .displayName("Record Reader")
+ .description("Specifies the Controller Service to use for parsing incoming data and determining the data's schema.")
+ .identifiesControllerService(RecordReaderFactory.class)
+ .required(false)
+ .build();
+
+ protected static final PropertyDescriptor RECORD_WRITER_FACTORY = new PropertyDescriptor.Builder()
+ .name("record-writer")
+ .displayName("Record Writer")
+ .description("The Record Writer to use for serializing Records to an output FlowFile.")
+ .identifiesControllerService(RecordSetWriterFactory.class)
+ .dependsOn(RECORD_READER_FACTORY)
+ .required(true)
+ .build();
+
+ protected static final Relationship ORIGINAL = new Relationship.Builder()
+ .name("Original")
+ .description("Request FlowFiles transferred when receiving HTTP responses with a status code between 200 and 299.")
+ .build();
+
+ protected static final Relationship FAILURE = new Relationship.Builder()
+ .name("Failure")
+ .description("Request FlowFiles transferred when receiving socket communication errors.")
+ .build();
+
+ protected static final Relationship RESPONSE = new Relationship.Builder()
+ .name("Response")
+ .description("Response FlowFiles transferred when receiving HTTP responses with a status code between 200 and 299.")
+ .build();
+
+ protected static final Set<Relationship> RELATIONSHIPS = Collections.unmodifiableSet(new HashSet<>(Arrays.asList(ORIGINAL, RESPONSE, FAILURE)));
+ protected static final List<PropertyDescriptor> PROPERTIES = Collections.unmodifiableList(Arrays.asList(REPORT_URL, WORKDAY_USERNAME, WORKDAY_PASSWORD, WEB_CLIENT_SERVICE,
+ FIELDS_TO_HASH, HASHING_ALGORITHM, RECORD_READER_FACTORY, RECORD_WRITER_FACTORY));
+
+ private final AtomicReference<WebClientService> webClientAtomicReference = new AtomicReference<>();
+ private final AtomicReference<RecordReaderFactory> recordReaderFactoryAtomicReference = new AtomicReference<>();
+ private final AtomicReference<RecordSetWriterFactory> recordSetWriterFactoryAtomicReference = new AtomicReference<>();
+
+ @Override
+ protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
+ return PROPERTIES;
+ }
+
+ @Override
+ public Set<Relationship> getRelationships() {
+ return RELATIONSHIPS;
+ }
+
+ @OnScheduled
+ public void setUpClient(final ProcessContext context) {
+ WebClientServiceProvider standardWebClientServiceProvider = context.getProperty(WEB_CLIENT_SERVICE).asControllerService(WebClientServiceProvider.class);
+ RecordReaderFactory recordReaderFactory = context.getProperty(RECORD_READER_FACTORY).asControllerService(RecordReaderFactory.class);
+ RecordSetWriterFactory recordSetWriterFactory = context.getProperty(RECORD_WRITER_FACTORY).asControllerService(RecordSetWriterFactory.class);
+ WebClientService webClientService = standardWebClientServiceProvider.getWebClientService();
+ webClientAtomicReference.set(webClientService);
+ recordReaderFactoryAtomicReference.set(recordReaderFactory);
+ recordSetWriterFactoryAtomicReference.set(recordSetWriterFactory);
+ }
+
+ @Override
+ public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException {
+ FlowFile flowfile = session.get();
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@nifi.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [nifi] ferencerdei commented on a diff in pull request #6376: NIFI-10455 Get workday report processor
Posted by GitBox <gi...@apache.org>.
ferencerdei commented on code in PR #6376:
URL: https://github.com/apache/nifi/pull/6376#discussion_r968132059
##########
nifi-nar-bundles/nifi-workday-bundle/nifi-workday-processors/src/main/java/org/apache/nifi/processors/workday/GetWorkdayReport.java:
##########
@@ -0,0 +1,432 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.processors.workday;
+
+import static org.apache.nifi.expression.ExpressionLanguageScope.FLOWFILE_ATTRIBUTES;
+import static org.apache.nifi.expression.ExpressionLanguageScope.NONE;
+import static org.apache.nifi.processor.util.StandardValidators.NON_BLANK_VALIDATOR;
+import static org.apache.nifi.processor.util.StandardValidators.URL_VALIDATOR;
+
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Base64;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import org.apache.nifi.annotation.behavior.EventDriven;
+import org.apache.nifi.annotation.behavior.InputRequirement;
+import org.apache.nifi.annotation.behavior.InputRequirement.Requirement;
+import org.apache.nifi.annotation.behavior.SideEffectFree;
+import org.apache.nifi.annotation.behavior.SupportsBatching;
+import org.apache.nifi.annotation.behavior.WritesAttribute;
+import org.apache.nifi.annotation.behavior.WritesAttributes;
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.annotation.lifecycle.OnScheduled;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.ValidationContext;
+import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.flowfile.attributes.CoreAttributes;
+import org.apache.nifi.logging.ComponentLog;
+import org.apache.nifi.processor.AbstractProcessor;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.processor.util.StandardValidators;
+import org.apache.nifi.record.path.RecordPath;
+import org.apache.nifi.record.path.RecordPathResult;
+import org.apache.nifi.schema.access.SchemaNotFoundException;
+import org.apache.nifi.security.util.crypto.HashAlgorithm;
+import org.apache.nifi.security.util.crypto.HashService;
+import org.apache.nifi.serialization.MalformedRecordException;
+import org.apache.nifi.serialization.RecordReader;
+import org.apache.nifi.serialization.RecordReaderFactory;
+import org.apache.nifi.serialization.RecordSetWriter;
+import org.apache.nifi.serialization.RecordSetWriterFactory;
+import org.apache.nifi.serialization.record.Record;
+import org.apache.nifi.serialization.record.RecordFieldType;
+import org.apache.nifi.serialization.record.RecordSchema;
+import org.apache.nifi.web.client.api.HttpResponseEntity;
+import org.apache.nifi.web.client.api.WebClientService;
+import org.apache.nifi.web.client.provider.api.WebClientServiceProvider;
+
+@Tags({"Workday", "report", "get"})
+@InputRequirement(Requirement.INPUT_ALLOWED)
+@CapabilityDescription("A processor which can interact with a configurable Workday Report. The processor can forward the content without modification, or you can transform it by"
+ + " providing the specific Record Reader and Record Writer services based on your needs. You can also hash, or remove fields using the input parameters and schemes in the Record writer. "
+ + "Supported Workday report formats are: csv, simplexml, json")
+@EventDriven
+@SideEffectFree
+@SupportsBatching
+@WritesAttributes({
+ @WritesAttribute(attribute = GetWorkdayReport.GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS, description = "The Java exception class raised when the processor fails"),
+ @WritesAttribute(attribute = GetWorkdayReport.GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE, description = "The Java exception message raised when the processor fails"),
+ @WritesAttribute(attribute = "mime.type", description = "Sets the mime.type attribute to the MIME Type specified by the Source / Record Writer"),
+ @WritesAttribute(attribute= GetWorkdayReport.RECORD_COUNT, description = "The number of records in an outgoing FlowFile. This is only populated on the 'success' relationship "
+ + "when Record Reader and Writer is set.")})
+public class GetWorkdayReport extends AbstractProcessor {
+
+ protected static final String STATUS_CODE = "getworkdayreport.status.code";
+ protected static final String REQUEST_URL = "getworkdayreport.request.url";
+ protected static final String REQUEST_DURATION = "getworkdayreport.request.duration";
+ protected static final String TRANSACTION_ID = "getworkdayreport.tx.id";
+ protected static final String GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS = "getworkdayreport.java.exception.class";
+ protected static final String GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE = "getworkdayreport.java.exception.message";
+ protected static final String RECORD_COUNT = "record.count";
+ protected static final String BASIC_PREFIX = "Basic ";
+ protected static final String COLUMNS_TO_HASH_SEPARATOR = ",";
+ protected static final String HEADER_AUTHORIZATION = "Authorization";
+ protected static final String HEADER_CONTENT_TYPE = "Content-Type";
+ protected static final String USERNAME_PASSWORD_SEPARATOR = ":";
+
+ protected static final PropertyDescriptor REPORT_URL = new PropertyDescriptor.Builder()
+ .name("Workday report URL")
+ .displayName("Workday report URL")
+ .description("HTTP remote URL of Workday report including a scheme of http or https, as well as a hostname or IP address with optional port and path elements.")
+ .required(true)
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .addValidator(URL_VALIDATOR)
+ .build();
+
+ protected static final PropertyDescriptor WORKDAY_USERNAME = new PropertyDescriptor.Builder()
+ .name("Workday Username")
+ .displayName("Workday Username")
+ .description("The username provided for authentication of Workday requests. Encoded using Base64 for HTTP Basic Authentication as described in RFC 7617.")
+ .required(true)
+ .addValidator(StandardValidators.createRegexMatchingValidator(Pattern.compile("^[\\x20-\\x39\\x3b-\\x7e\\x80-\\xff]+$")))
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .build();
+
+ protected static final PropertyDescriptor WORKDAY_PASSWORD = new PropertyDescriptor.Builder()
+ .name("Workday Password")
+ .displayName("Workday Password")
+ .description("The password provided for authentication of Workday requests. Encoded using Base64 for HTTP Basic Authentication as described in RFC 7617.")
+ .required(true)
+ .sensitive(true)
+ .addValidator(StandardValidators.createRegexMatchingValidator(Pattern.compile("^[\\x20-\\x7e\\x80-\\xff]+$")))
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .build();
+
+ protected static final PropertyDescriptor WEB_CLIENT_SERVICE = new PropertyDescriptor.Builder()
+ .name("Standard Web Client Service")
+ .description("Web client which is used to communicate with the Workday API.")
+ .required(true)
+ .identifiesControllerService(WebClientServiceProvider.class)
+ .build();
+
+ protected static final PropertyDescriptor FIELDS_TO_HASH = new PropertyDescriptor.Builder()
+ .name("Fields to hash")
+ .displayName("Fields to hash")
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@nifi.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [nifi] exceptionfactory commented on a diff in pull request #6376: NIFI-10455 Get workday report processor
Posted by GitBox <gi...@apache.org>.
exceptionfactory commented on code in PR #6376:
URL: https://github.com/apache/nifi/pull/6376#discussion_r966418883
##########
nifi-nar-bundles/nifi-workday-bundle/nifi-workday-processors/src/main/java/org/apache/nifi/processors/workday/GetWorkdayReport.java:
##########
@@ -0,0 +1,432 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.processors.workday;
+
+import static org.apache.nifi.expression.ExpressionLanguageScope.FLOWFILE_ATTRIBUTES;
+import static org.apache.nifi.expression.ExpressionLanguageScope.NONE;
+import static org.apache.nifi.processor.util.StandardValidators.NON_BLANK_VALIDATOR;
+import static org.apache.nifi.processor.util.StandardValidators.URL_VALIDATOR;
+
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Base64;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import org.apache.nifi.annotation.behavior.EventDriven;
+import org.apache.nifi.annotation.behavior.InputRequirement;
+import org.apache.nifi.annotation.behavior.InputRequirement.Requirement;
+import org.apache.nifi.annotation.behavior.SideEffectFree;
+import org.apache.nifi.annotation.behavior.SupportsBatching;
+import org.apache.nifi.annotation.behavior.WritesAttribute;
+import org.apache.nifi.annotation.behavior.WritesAttributes;
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.annotation.lifecycle.OnScheduled;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.ValidationContext;
+import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.flowfile.attributes.CoreAttributes;
+import org.apache.nifi.logging.ComponentLog;
+import org.apache.nifi.processor.AbstractProcessor;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.processor.util.StandardValidators;
+import org.apache.nifi.record.path.RecordPath;
+import org.apache.nifi.record.path.RecordPathResult;
+import org.apache.nifi.schema.access.SchemaNotFoundException;
+import org.apache.nifi.security.util.crypto.HashAlgorithm;
+import org.apache.nifi.security.util.crypto.HashService;
+import org.apache.nifi.serialization.MalformedRecordException;
+import org.apache.nifi.serialization.RecordReader;
+import org.apache.nifi.serialization.RecordReaderFactory;
+import org.apache.nifi.serialization.RecordSetWriter;
+import org.apache.nifi.serialization.RecordSetWriterFactory;
+import org.apache.nifi.serialization.record.Record;
+import org.apache.nifi.serialization.record.RecordFieldType;
+import org.apache.nifi.serialization.record.RecordSchema;
+import org.apache.nifi.web.client.api.HttpResponseEntity;
+import org.apache.nifi.web.client.api.WebClientService;
+import org.apache.nifi.web.client.provider.api.WebClientServiceProvider;
+
+@Tags({"Workday", "report", "get"})
+@InputRequirement(Requirement.INPUT_ALLOWED)
+@CapabilityDescription("A processor which can interact with a configurable Workday Report. The processor can forward the content without modification, or you can transform it by"
+ + " providing the specific Record Reader and Record Writer services based on your needs. You can also hash, or remove fields using the input parameters and schemes in the Record writer. "
+ + "Supported Workday report formats are: csv, simplexml, json")
+@EventDriven
+@SideEffectFree
+@SupportsBatching
+@WritesAttributes({
+ @WritesAttribute(attribute = GetWorkdayReport.GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS, description = "The Java exception class raised when the processor fails"),
+ @WritesAttribute(attribute = GetWorkdayReport.GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE, description = "The Java exception message raised when the processor fails"),
+ @WritesAttribute(attribute = "mime.type", description = "Sets the mime.type attribute to the MIME Type specified by the Source / Record Writer"),
+ @WritesAttribute(attribute= GetWorkdayReport.RECORD_COUNT, description = "The number of records in an outgoing FlowFile. This is only populated on the 'success' relationship "
+ + "when Record Reader and Writer is set.")})
+public class GetWorkdayReport extends AbstractProcessor {
+
+ protected static final String STATUS_CODE = "getworkdayreport.status.code";
+ protected static final String REQUEST_URL = "getworkdayreport.request.url";
+ protected static final String REQUEST_DURATION = "getworkdayreport.request.duration";
+ protected static final String TRANSACTION_ID = "getworkdayreport.tx.id";
+ protected static final String GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS = "getworkdayreport.java.exception.class";
+ protected static final String GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE = "getworkdayreport.java.exception.message";
+ protected static final String RECORD_COUNT = "record.count";
+ protected static final String BASIC_PREFIX = "Basic ";
+ protected static final String COLUMNS_TO_HASH_SEPARATOR = ",";
+ protected static final String HEADER_AUTHORIZATION = "Authorization";
+ protected static final String HEADER_CONTENT_TYPE = "Content-Type";
+ protected static final String USERNAME_PASSWORD_SEPARATOR = ":";
+
+ protected static final PropertyDescriptor REPORT_URL = new PropertyDescriptor.Builder()
+ .name("Workday report URL")
+ .displayName("Workday report URL")
Review Comment:
In general, property names should have all words capitalized:
```suggestion
.name("Workday Report URL")
.displayName("Workday Report URL")
```
##########
nifi-assembly/pom.xml:
##########
@@ -227,6 +227,12 @@ language governing permissions and limitations under the License. -->
<version>1.18.0-SNAPSHOT</version>
<type>nar</type>
</dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-web-client-provider-service-nar</artifactId>
+ <version>1.18.0-SNAPSHOT</version>
+ <type>nar</type>
+ </dependency>
Review Comment:
This declaration duplicates an existing dependency and should be removed.
##########
nifi-nar-bundles/nifi-workday-bundle/nifi-workday-processors/src/main/java/org/apache/nifi/processors/workday/GetWorkdayReport.java:
##########
@@ -0,0 +1,432 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.processors.workday;
+
+import static org.apache.nifi.expression.ExpressionLanguageScope.FLOWFILE_ATTRIBUTES;
+import static org.apache.nifi.expression.ExpressionLanguageScope.NONE;
+import static org.apache.nifi.processor.util.StandardValidators.NON_BLANK_VALIDATOR;
+import static org.apache.nifi.processor.util.StandardValidators.URL_VALIDATOR;
+
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Base64;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import org.apache.nifi.annotation.behavior.EventDriven;
+import org.apache.nifi.annotation.behavior.InputRequirement;
+import org.apache.nifi.annotation.behavior.InputRequirement.Requirement;
+import org.apache.nifi.annotation.behavior.SideEffectFree;
+import org.apache.nifi.annotation.behavior.SupportsBatching;
+import org.apache.nifi.annotation.behavior.WritesAttribute;
+import org.apache.nifi.annotation.behavior.WritesAttributes;
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.annotation.lifecycle.OnScheduled;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.ValidationContext;
+import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.flowfile.attributes.CoreAttributes;
+import org.apache.nifi.logging.ComponentLog;
+import org.apache.nifi.processor.AbstractProcessor;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.processor.util.StandardValidators;
+import org.apache.nifi.record.path.RecordPath;
+import org.apache.nifi.record.path.RecordPathResult;
+import org.apache.nifi.schema.access.SchemaNotFoundException;
+import org.apache.nifi.security.util.crypto.HashAlgorithm;
+import org.apache.nifi.security.util.crypto.HashService;
+import org.apache.nifi.serialization.MalformedRecordException;
+import org.apache.nifi.serialization.RecordReader;
+import org.apache.nifi.serialization.RecordReaderFactory;
+import org.apache.nifi.serialization.RecordSetWriter;
+import org.apache.nifi.serialization.RecordSetWriterFactory;
+import org.apache.nifi.serialization.record.Record;
+import org.apache.nifi.serialization.record.RecordFieldType;
+import org.apache.nifi.serialization.record.RecordSchema;
+import org.apache.nifi.web.client.api.HttpResponseEntity;
+import org.apache.nifi.web.client.api.WebClientService;
+import org.apache.nifi.web.client.provider.api.WebClientServiceProvider;
+
+@Tags({"Workday", "report", "get"})
+@InputRequirement(Requirement.INPUT_ALLOWED)
+@CapabilityDescription("A processor which can interact with a configurable Workday Report. The processor can forward the content without modification, or you can transform it by"
+ + " providing the specific Record Reader and Record Writer services based on your needs. You can also hash, or remove fields using the input parameters and schemes in the Record writer. "
+ + "Supported Workday report formats are: csv, simplexml, json")
+@EventDriven
+@SideEffectFree
+@SupportsBatching
+@WritesAttributes({
+ @WritesAttribute(attribute = GetWorkdayReport.GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS, description = "The Java exception class raised when the processor fails"),
+ @WritesAttribute(attribute = GetWorkdayReport.GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE, description = "The Java exception message raised when the processor fails"),
+ @WritesAttribute(attribute = "mime.type", description = "Sets the mime.type attribute to the MIME Type specified by the Source / Record Writer"),
+ @WritesAttribute(attribute= GetWorkdayReport.RECORD_COUNT, description = "The number of records in an outgoing FlowFile. This is only populated on the 'success' relationship "
+ + "when Record Reader and Writer is set.")})
+public class GetWorkdayReport extends AbstractProcessor {
+
+ protected static final String STATUS_CODE = "getworkdayreport.status.code";
+ protected static final String REQUEST_URL = "getworkdayreport.request.url";
+ protected static final String REQUEST_DURATION = "getworkdayreport.request.duration";
+ protected static final String TRANSACTION_ID = "getworkdayreport.tx.id";
+ protected static final String GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS = "getworkdayreport.java.exception.class";
+ protected static final String GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE = "getworkdayreport.java.exception.message";
+ protected static final String RECORD_COUNT = "record.count";
+ protected static final String BASIC_PREFIX = "Basic ";
+ protected static final String COLUMNS_TO_HASH_SEPARATOR = ",";
+ protected static final String HEADER_AUTHORIZATION = "Authorization";
+ protected static final String HEADER_CONTENT_TYPE = "Content-Type";
+ protected static final String USERNAME_PASSWORD_SEPARATOR = ":";
+
+ protected static final PropertyDescriptor REPORT_URL = new PropertyDescriptor.Builder()
+ .name("Workday report URL")
+ .displayName("Workday report URL")
+ .description("HTTP remote URL of Workday report including a scheme of http or https, as well as a hostname or IP address with optional port and path elements.")
+ .required(true)
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .addValidator(URL_VALIDATOR)
+ .build();
+
+ protected static final PropertyDescriptor WORKDAY_USERNAME = new PropertyDescriptor.Builder()
+ .name("Workday Username")
+ .displayName("Workday Username")
+ .description("The username provided for authentication of Workday requests. Encoded using Base64 for HTTP Basic Authentication as described in RFC 7617.")
+ .required(true)
+ .addValidator(StandardValidators.createRegexMatchingValidator(Pattern.compile("^[\\x20-\\x39\\x3b-\\x7e\\x80-\\xff]+$")))
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .build();
+
+ protected static final PropertyDescriptor WORKDAY_PASSWORD = new PropertyDescriptor.Builder()
+ .name("Workday Password")
+ .displayName("Workday Password")
+ .description("The password provided for authentication of Workday requests. Encoded using Base64 for HTTP Basic Authentication as described in RFC 7617.")
+ .required(true)
+ .sensitive(true)
+ .addValidator(StandardValidators.createRegexMatchingValidator(Pattern.compile("^[\\x20-\\x7e\\x80-\\xff]+$")))
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .build();
+
+ protected static final PropertyDescriptor WEB_CLIENT_SERVICE = new PropertyDescriptor.Builder()
+ .name("Standard Web Client Service")
Review Comment:
The `Standard` word should be removed to reflect the general interface as opposed to the implementation:
```suggestion
.name("Web Client Service Provider")
```
##########
nifi-nar-bundles/nifi-workday-bundle/nifi-workday-processors/src/main/java/org/apache/nifi/processors/workday/GetWorkdayReport.java:
##########
@@ -0,0 +1,432 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.processors.workday;
+
+import static org.apache.nifi.expression.ExpressionLanguageScope.FLOWFILE_ATTRIBUTES;
+import static org.apache.nifi.expression.ExpressionLanguageScope.NONE;
+import static org.apache.nifi.processor.util.StandardValidators.NON_BLANK_VALIDATOR;
+import static org.apache.nifi.processor.util.StandardValidators.URL_VALIDATOR;
+
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Base64;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import org.apache.nifi.annotation.behavior.EventDriven;
+import org.apache.nifi.annotation.behavior.InputRequirement;
+import org.apache.nifi.annotation.behavior.InputRequirement.Requirement;
+import org.apache.nifi.annotation.behavior.SideEffectFree;
+import org.apache.nifi.annotation.behavior.SupportsBatching;
+import org.apache.nifi.annotation.behavior.WritesAttribute;
+import org.apache.nifi.annotation.behavior.WritesAttributes;
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.annotation.lifecycle.OnScheduled;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.ValidationContext;
+import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.flowfile.attributes.CoreAttributes;
+import org.apache.nifi.logging.ComponentLog;
+import org.apache.nifi.processor.AbstractProcessor;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.processor.util.StandardValidators;
+import org.apache.nifi.record.path.RecordPath;
+import org.apache.nifi.record.path.RecordPathResult;
+import org.apache.nifi.schema.access.SchemaNotFoundException;
+import org.apache.nifi.security.util.crypto.HashAlgorithm;
+import org.apache.nifi.security.util.crypto.HashService;
+import org.apache.nifi.serialization.MalformedRecordException;
+import org.apache.nifi.serialization.RecordReader;
+import org.apache.nifi.serialization.RecordReaderFactory;
+import org.apache.nifi.serialization.RecordSetWriter;
+import org.apache.nifi.serialization.RecordSetWriterFactory;
+import org.apache.nifi.serialization.record.Record;
+import org.apache.nifi.serialization.record.RecordFieldType;
+import org.apache.nifi.serialization.record.RecordSchema;
+import org.apache.nifi.web.client.api.HttpResponseEntity;
+import org.apache.nifi.web.client.api.WebClientService;
+import org.apache.nifi.web.client.provider.api.WebClientServiceProvider;
+
+@Tags({"Workday", "report", "get"})
+@InputRequirement(Requirement.INPUT_ALLOWED)
+@CapabilityDescription("A processor which can interact with a configurable Workday Report. The processor can forward the content without modification, or you can transform it by"
+ + " providing the specific Record Reader and Record Writer services based on your needs. You can also hash, or remove fields using the input parameters and schemes in the Record writer. "
+ + "Supported Workday report formats are: csv, simplexml, json")
+@EventDriven
+@SideEffectFree
+@SupportsBatching
+@WritesAttributes({
+ @WritesAttribute(attribute = GetWorkdayReport.GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS, description = "The Java exception class raised when the processor fails"),
+ @WritesAttribute(attribute = GetWorkdayReport.GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE, description = "The Java exception message raised when the processor fails"),
+ @WritesAttribute(attribute = "mime.type", description = "Sets the mime.type attribute to the MIME Type specified by the Source / Record Writer"),
+ @WritesAttribute(attribute= GetWorkdayReport.RECORD_COUNT, description = "The number of records in an outgoing FlowFile. This is only populated on the 'success' relationship "
+ + "when Record Reader and Writer is set.")})
+public class GetWorkdayReport extends AbstractProcessor {
+
+ protected static final String STATUS_CODE = "getworkdayreport.status.code";
+ protected static final String REQUEST_URL = "getworkdayreport.request.url";
+ protected static final String REQUEST_DURATION = "getworkdayreport.request.duration";
+ protected static final String TRANSACTION_ID = "getworkdayreport.tx.id";
+ protected static final String GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS = "getworkdayreport.java.exception.class";
+ protected static final String GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE = "getworkdayreport.java.exception.message";
+ protected static final String RECORD_COUNT = "record.count";
+ protected static final String BASIC_PREFIX = "Basic ";
+ protected static final String COLUMNS_TO_HASH_SEPARATOR = ",";
+ protected static final String HEADER_AUTHORIZATION = "Authorization";
+ protected static final String HEADER_CONTENT_TYPE = "Content-Type";
+ protected static final String USERNAME_PASSWORD_SEPARATOR = ":";
+
+ protected static final PropertyDescriptor REPORT_URL = new PropertyDescriptor.Builder()
+ .name("Workday report URL")
+ .displayName("Workday report URL")
+ .description("HTTP remote URL of Workday report including a scheme of http or https, as well as a hostname or IP address with optional port and path elements.")
+ .required(true)
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .addValidator(URL_VALIDATOR)
+ .build();
+
+ protected static final PropertyDescriptor WORKDAY_USERNAME = new PropertyDescriptor.Builder()
+ .name("Workday Username")
+ .displayName("Workday Username")
+ .description("The username provided for authentication of Workday requests. Encoded using Base64 for HTTP Basic Authentication as described in RFC 7617.")
+ .required(true)
+ .addValidator(StandardValidators.createRegexMatchingValidator(Pattern.compile("^[\\x20-\\x39\\x3b-\\x7e\\x80-\\xff]+$")))
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .build();
+
+ protected static final PropertyDescriptor WORKDAY_PASSWORD = new PropertyDescriptor.Builder()
+ .name("Workday Password")
+ .displayName("Workday Password")
+ .description("The password provided for authentication of Workday requests. Encoded using Base64 for HTTP Basic Authentication as described in RFC 7617.")
+ .required(true)
+ .sensitive(true)
+ .addValidator(StandardValidators.createRegexMatchingValidator(Pattern.compile("^[\\x20-\\x7e\\x80-\\xff]+$")))
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .build();
+
+ protected static final PropertyDescriptor WEB_CLIENT_SERVICE = new PropertyDescriptor.Builder()
+ .name("Standard Web Client Service")
+ .description("Web client which is used to communicate with the Workday API.")
+ .required(true)
+ .identifiesControllerService(WebClientServiceProvider.class)
+ .build();
+
+ protected static final PropertyDescriptor FIELDS_TO_HASH = new PropertyDescriptor.Builder()
+ .name("Fields to hash")
+ .displayName("Fields to hash")
+ .description("Comma separated record paths to replace with hash.")
+ .required(false)
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .addValidator(NON_BLANK_VALIDATOR)
+ .build();
+
+ protected static final PropertyDescriptor HASHING_ALGORITHM = new PropertyDescriptor.Builder()
+ .name("Hashing algorithm")
+ .displayName("Hashing algorithm")
Review Comment:
```suggestion
.name("Hashing Algorithm")
.displayName("Hashing Algorithm")
```
##########
nifi-nar-bundles/nifi-workday-bundle/nifi-workday-processors/src/main/java/org/apache/nifi/processors/workday/GetWorkdayReport.java:
##########
@@ -0,0 +1,432 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.processors.workday;
+
+import static org.apache.nifi.expression.ExpressionLanguageScope.FLOWFILE_ATTRIBUTES;
+import static org.apache.nifi.expression.ExpressionLanguageScope.NONE;
+import static org.apache.nifi.processor.util.StandardValidators.NON_BLANK_VALIDATOR;
+import static org.apache.nifi.processor.util.StandardValidators.URL_VALIDATOR;
+
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Base64;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import org.apache.nifi.annotation.behavior.EventDriven;
+import org.apache.nifi.annotation.behavior.InputRequirement;
+import org.apache.nifi.annotation.behavior.InputRequirement.Requirement;
+import org.apache.nifi.annotation.behavior.SideEffectFree;
+import org.apache.nifi.annotation.behavior.SupportsBatching;
+import org.apache.nifi.annotation.behavior.WritesAttribute;
+import org.apache.nifi.annotation.behavior.WritesAttributes;
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.annotation.lifecycle.OnScheduled;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.ValidationContext;
+import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.flowfile.attributes.CoreAttributes;
+import org.apache.nifi.logging.ComponentLog;
+import org.apache.nifi.processor.AbstractProcessor;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.processor.util.StandardValidators;
+import org.apache.nifi.record.path.RecordPath;
+import org.apache.nifi.record.path.RecordPathResult;
+import org.apache.nifi.schema.access.SchemaNotFoundException;
+import org.apache.nifi.security.util.crypto.HashAlgorithm;
+import org.apache.nifi.security.util.crypto.HashService;
+import org.apache.nifi.serialization.MalformedRecordException;
+import org.apache.nifi.serialization.RecordReader;
+import org.apache.nifi.serialization.RecordReaderFactory;
+import org.apache.nifi.serialization.RecordSetWriter;
+import org.apache.nifi.serialization.RecordSetWriterFactory;
+import org.apache.nifi.serialization.record.Record;
+import org.apache.nifi.serialization.record.RecordFieldType;
+import org.apache.nifi.serialization.record.RecordSchema;
+import org.apache.nifi.web.client.api.HttpResponseEntity;
+import org.apache.nifi.web.client.api.WebClientService;
+import org.apache.nifi.web.client.provider.api.WebClientServiceProvider;
+
+@Tags({"Workday", "report", "get"})
+@InputRequirement(Requirement.INPUT_ALLOWED)
+@CapabilityDescription("A processor which can interact with a configurable Workday Report. The processor can forward the content without modification, or you can transform it by"
+ + " providing the specific Record Reader and Record Writer services based on your needs. You can also hash, or remove fields using the input parameters and schemes in the Record writer. "
+ + "Supported Workday report formats are: csv, simplexml, json")
+@EventDriven
+@SideEffectFree
+@SupportsBatching
+@WritesAttributes({
+ @WritesAttribute(attribute = GetWorkdayReport.GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS, description = "The Java exception class raised when the processor fails"),
+ @WritesAttribute(attribute = GetWorkdayReport.GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE, description = "The Java exception message raised when the processor fails"),
+ @WritesAttribute(attribute = "mime.type", description = "Sets the mime.type attribute to the MIME Type specified by the Source / Record Writer"),
+ @WritesAttribute(attribute= GetWorkdayReport.RECORD_COUNT, description = "The number of records in an outgoing FlowFile. This is only populated on the 'success' relationship "
+ + "when Record Reader and Writer is set.")})
+public class GetWorkdayReport extends AbstractProcessor {
+
+ protected static final String STATUS_CODE = "getworkdayreport.status.code";
+ protected static final String REQUEST_URL = "getworkdayreport.request.url";
+ protected static final String REQUEST_DURATION = "getworkdayreport.request.duration";
+ protected static final String TRANSACTION_ID = "getworkdayreport.tx.id";
+ protected static final String GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS = "getworkdayreport.java.exception.class";
+ protected static final String GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE = "getworkdayreport.java.exception.message";
+ protected static final String RECORD_COUNT = "record.count";
+ protected static final String BASIC_PREFIX = "Basic ";
+ protected static final String COLUMNS_TO_HASH_SEPARATOR = ",";
+ protected static final String HEADER_AUTHORIZATION = "Authorization";
+ protected static final String HEADER_CONTENT_TYPE = "Content-Type";
+ protected static final String USERNAME_PASSWORD_SEPARATOR = ":";
+
+ protected static final PropertyDescriptor REPORT_URL = new PropertyDescriptor.Builder()
+ .name("Workday report URL")
+ .displayName("Workday report URL")
+ .description("HTTP remote URL of Workday report including a scheme of http or https, as well as a hostname or IP address with optional port and path elements.")
+ .required(true)
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .addValidator(URL_VALIDATOR)
+ .build();
+
+ protected static final PropertyDescriptor WORKDAY_USERNAME = new PropertyDescriptor.Builder()
+ .name("Workday Username")
+ .displayName("Workday Username")
+ .description("The username provided for authentication of Workday requests. Encoded using Base64 for HTTP Basic Authentication as described in RFC 7617.")
+ .required(true)
+ .addValidator(StandardValidators.createRegexMatchingValidator(Pattern.compile("^[\\x20-\\x39\\x3b-\\x7e\\x80-\\xff]+$")))
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .build();
+
+ protected static final PropertyDescriptor WORKDAY_PASSWORD = new PropertyDescriptor.Builder()
+ .name("Workday Password")
+ .displayName("Workday Password")
+ .description("The password provided for authentication of Workday requests. Encoded using Base64 for HTTP Basic Authentication as described in RFC 7617.")
+ .required(true)
+ .sensitive(true)
+ .addValidator(StandardValidators.createRegexMatchingValidator(Pattern.compile("^[\\x20-\\x7e\\x80-\\xff]+$")))
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .build();
+
+ protected static final PropertyDescriptor WEB_CLIENT_SERVICE = new PropertyDescriptor.Builder()
+ .name("Standard Web Client Service")
+ .description("Web client which is used to communicate with the Workday API.")
+ .required(true)
+ .identifiesControllerService(WebClientServiceProvider.class)
+ .build();
+
+ protected static final PropertyDescriptor FIELDS_TO_HASH = new PropertyDescriptor.Builder()
+ .name("Fields to hash")
+ .displayName("Fields to hash")
+ .description("Comma separated record paths to replace with hash.")
+ .required(false)
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .addValidator(NON_BLANK_VALIDATOR)
+ .build();
+
+ protected static final PropertyDescriptor HASHING_ALGORITHM = new PropertyDescriptor.Builder()
+ .name("Hashing algorithm")
+ .displayName("Hashing algorithm")
+ .description("Determines what hashing algorithm should be used to perform the hashing function.")
+ .required(true)
+ .allowableValues(HashService.buildHashAlgorithmAllowableValues())
+ .defaultValue(HashAlgorithm.SHA256.getName())
+ .addValidator(NON_BLANK_VALIDATOR)
+ .expressionLanguageSupported(NONE)
+ .dependsOn(FIELDS_TO_HASH)
+ .build();
+
+ protected static final PropertyDescriptor RECORD_READER_FACTORY = new PropertyDescriptor.Builder()
+ .name("record-reader")
+ .displayName("Record Reader")
+ .description("Specifies the Controller Service to use for parsing incoming data and determining the data's schema.")
+ .identifiesControllerService(RecordReaderFactory.class)
+ .required(false)
+ .build();
+
+ protected static final PropertyDescriptor RECORD_WRITER_FACTORY = new PropertyDescriptor.Builder()
+ .name("record-writer")
+ .displayName("Record Writer")
+ .description("The Record Writer to use for serializing Records to an output FlowFile.")
+ .identifiesControllerService(RecordSetWriterFactory.class)
+ .dependsOn(RECORD_READER_FACTORY)
+ .required(true)
+ .build();
+
+ protected static final Relationship ORIGINAL = new Relationship.Builder()
+ .name("Original")
+ .description("Request FlowFiles transferred when receiving HTTP responses with a status code between 200 and 299.")
+ .build();
+
+ protected static final Relationship FAILURE = new Relationship.Builder()
+ .name("Failure")
+ .description("Request FlowFiles transferred when receiving socket communication errors.")
+ .build();
+
+ protected static final Relationship RESPONSE = new Relationship.Builder()
+ .name("Response")
+ .description("Response FlowFiles transferred when receiving HTTP responses with a status code between 200 and 299.")
+ .build();
+
+ protected static final Set<Relationship> RELATIONSHIPS = Collections.unmodifiableSet(new HashSet<>(Arrays.asList(ORIGINAL, RESPONSE, FAILURE)));
+ protected static final List<PropertyDescriptor> PROPERTIES = Collections.unmodifiableList(Arrays.asList(REPORT_URL, WORKDAY_USERNAME, WORKDAY_PASSWORD, WEB_CLIENT_SERVICE,
+ FIELDS_TO_HASH, HASHING_ALGORITHM, RECORD_READER_FACTORY, RECORD_WRITER_FACTORY));
+
+ private final AtomicReference<WebClientService> webClientAtomicReference = new AtomicReference<>();
+ private final AtomicReference<RecordReaderFactory> recordReaderFactoryAtomicReference = new AtomicReference<>();
+ private final AtomicReference<RecordSetWriterFactory> recordSetWriterFactoryAtomicReference = new AtomicReference<>();
+
+ @Override
+ protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
+ return PROPERTIES;
+ }
+
+ @Override
+ public Set<Relationship> getRelationships() {
+ return RELATIONSHIPS;
+ }
+
+ @OnScheduled
+ public void setUpClient(final ProcessContext context) {
+ WebClientServiceProvider standardWebClientServiceProvider = context.getProperty(WEB_CLIENT_SERVICE).asControllerService(WebClientServiceProvider.class);
+ RecordReaderFactory recordReaderFactory = context.getProperty(RECORD_READER_FACTORY).asControllerService(RecordReaderFactory.class);
+ RecordSetWriterFactory recordSetWriterFactory = context.getProperty(RECORD_WRITER_FACTORY).asControllerService(RecordSetWriterFactory.class);
+ WebClientService webClientService = standardWebClientServiceProvider.getWebClientService();
+ webClientAtomicReference.set(webClientService);
+ recordReaderFactoryAtomicReference.set(recordReaderFactory);
+ recordSetWriterFactoryAtomicReference.set(recordSetWriterFactory);
+ }
+
+ @Override
+ public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException {
+ FlowFile flowfile = session.get();
+
+ if (skipExecution(context, flowfile)) {
+ return;
+ }
+
+ ComponentLog logger = getLogger();
+ FlowFile responseFlowFile = null;
+
+ try {
+ WebClientService webClientService = webClientAtomicReference.get();
+ URI uri = new URI(context.getProperty(REPORT_URL).evaluateAttributeExpressions(flowfile).getValue().trim());
+ long startNanos = System.nanoTime();
+ String authorization = createAuthorizationHeader(context, flowfile);
+
+ try(HttpResponseEntity httpResponseEntity = webClientService.get().uri(uri).header(HEADER_AUTHORIZATION, authorization).retrieve()) {
Review Comment:
Recommend formatting the request call using multiple lines for easier readability.
##########
nifi-nar-bundles/nifi-workday-bundle/nifi-workday-processors/src/main/java/org/apache/nifi/processors/workday/GetWorkdayReport.java:
##########
@@ -0,0 +1,432 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.processors.workday;
+
+import static org.apache.nifi.expression.ExpressionLanguageScope.FLOWFILE_ATTRIBUTES;
+import static org.apache.nifi.expression.ExpressionLanguageScope.NONE;
+import static org.apache.nifi.processor.util.StandardValidators.NON_BLANK_VALIDATOR;
+import static org.apache.nifi.processor.util.StandardValidators.URL_VALIDATOR;
+
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Base64;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import org.apache.nifi.annotation.behavior.EventDriven;
+import org.apache.nifi.annotation.behavior.InputRequirement;
+import org.apache.nifi.annotation.behavior.InputRequirement.Requirement;
+import org.apache.nifi.annotation.behavior.SideEffectFree;
+import org.apache.nifi.annotation.behavior.SupportsBatching;
+import org.apache.nifi.annotation.behavior.WritesAttribute;
+import org.apache.nifi.annotation.behavior.WritesAttributes;
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.annotation.lifecycle.OnScheduled;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.ValidationContext;
+import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.flowfile.attributes.CoreAttributes;
+import org.apache.nifi.logging.ComponentLog;
+import org.apache.nifi.processor.AbstractProcessor;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.processor.util.StandardValidators;
+import org.apache.nifi.record.path.RecordPath;
+import org.apache.nifi.record.path.RecordPathResult;
+import org.apache.nifi.schema.access.SchemaNotFoundException;
+import org.apache.nifi.security.util.crypto.HashAlgorithm;
+import org.apache.nifi.security.util.crypto.HashService;
+import org.apache.nifi.serialization.MalformedRecordException;
+import org.apache.nifi.serialization.RecordReader;
+import org.apache.nifi.serialization.RecordReaderFactory;
+import org.apache.nifi.serialization.RecordSetWriter;
+import org.apache.nifi.serialization.RecordSetWriterFactory;
+import org.apache.nifi.serialization.record.Record;
+import org.apache.nifi.serialization.record.RecordFieldType;
+import org.apache.nifi.serialization.record.RecordSchema;
+import org.apache.nifi.web.client.api.HttpResponseEntity;
+import org.apache.nifi.web.client.api.WebClientService;
+import org.apache.nifi.web.client.provider.api.WebClientServiceProvider;
+
+@Tags({"Workday", "report", "get"})
+@InputRequirement(Requirement.INPUT_ALLOWED)
+@CapabilityDescription("A processor which can interact with a configurable Workday Report. The processor can forward the content without modification, or you can transform it by"
+ + " providing the specific Record Reader and Record Writer services based on your needs. You can also hash, or remove fields using the input parameters and schemes in the Record writer. "
+ + "Supported Workday report formats are: csv, simplexml, json")
+@EventDriven
+@SideEffectFree
+@SupportsBatching
+@WritesAttributes({
+ @WritesAttribute(attribute = GetWorkdayReport.GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS, description = "The Java exception class raised when the processor fails"),
+ @WritesAttribute(attribute = GetWorkdayReport.GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE, description = "The Java exception message raised when the processor fails"),
+ @WritesAttribute(attribute = "mime.type", description = "Sets the mime.type attribute to the MIME Type specified by the Source / Record Writer"),
+ @WritesAttribute(attribute= GetWorkdayReport.RECORD_COUNT, description = "The number of records in an outgoing FlowFile. This is only populated on the 'success' relationship "
+ + "when Record Reader and Writer is set.")})
+public class GetWorkdayReport extends AbstractProcessor {
+
+ protected static final String STATUS_CODE = "getworkdayreport.status.code";
+ protected static final String REQUEST_URL = "getworkdayreport.request.url";
+ protected static final String REQUEST_DURATION = "getworkdayreport.request.duration";
+ protected static final String TRANSACTION_ID = "getworkdayreport.tx.id";
+ protected static final String GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS = "getworkdayreport.java.exception.class";
+ protected static final String GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE = "getworkdayreport.java.exception.message";
+ protected static final String RECORD_COUNT = "record.count";
+ protected static final String BASIC_PREFIX = "Basic ";
+ protected static final String COLUMNS_TO_HASH_SEPARATOR = ",";
+ protected static final String HEADER_AUTHORIZATION = "Authorization";
+ protected static final String HEADER_CONTENT_TYPE = "Content-Type";
+ protected static final String USERNAME_PASSWORD_SEPARATOR = ":";
+
+ protected static final PropertyDescriptor REPORT_URL = new PropertyDescriptor.Builder()
+ .name("Workday report URL")
+ .displayName("Workday report URL")
+ .description("HTTP remote URL of Workday report including a scheme of http or https, as well as a hostname or IP address with optional port and path elements.")
+ .required(true)
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .addValidator(URL_VALIDATOR)
+ .build();
+
+ protected static final PropertyDescriptor WORKDAY_USERNAME = new PropertyDescriptor.Builder()
+ .name("Workday Username")
+ .displayName("Workday Username")
+ .description("The username provided for authentication of Workday requests. Encoded using Base64 for HTTP Basic Authentication as described in RFC 7617.")
+ .required(true)
+ .addValidator(StandardValidators.createRegexMatchingValidator(Pattern.compile("^[\\x20-\\x39\\x3b-\\x7e\\x80-\\xff]+$")))
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .build();
+
+ protected static final PropertyDescriptor WORKDAY_PASSWORD = new PropertyDescriptor.Builder()
+ .name("Workday Password")
+ .displayName("Workday Password")
+ .description("The password provided for authentication of Workday requests. Encoded using Base64 for HTTP Basic Authentication as described in RFC 7617.")
+ .required(true)
+ .sensitive(true)
+ .addValidator(StandardValidators.createRegexMatchingValidator(Pattern.compile("^[\\x20-\\x7e\\x80-\\xff]+$")))
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .build();
+
+ protected static final PropertyDescriptor WEB_CLIENT_SERVICE = new PropertyDescriptor.Builder()
+ .name("Standard Web Client Service")
+ .description("Web client which is used to communicate with the Workday API.")
+ .required(true)
+ .identifiesControllerService(WebClientServiceProvider.class)
+ .build();
+
+ protected static final PropertyDescriptor FIELDS_TO_HASH = new PropertyDescriptor.Builder()
+ .name("Fields to hash")
+ .displayName("Fields to hash")
Review Comment:
```suggestion
.name("Hashed Fields")
.displayName("Hashed Fields")
```
##########
nifi-nar-bundles/nifi-workday-bundle/nifi-workday-processors/src/main/java/org/apache/nifi/processors/workday/GetWorkdayReport.java:
##########
@@ -0,0 +1,432 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.processors.workday;
+
+import static org.apache.nifi.expression.ExpressionLanguageScope.FLOWFILE_ATTRIBUTES;
+import static org.apache.nifi.expression.ExpressionLanguageScope.NONE;
+import static org.apache.nifi.processor.util.StandardValidators.NON_BLANK_VALIDATOR;
+import static org.apache.nifi.processor.util.StandardValidators.URL_VALIDATOR;
+
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Base64;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import org.apache.nifi.annotation.behavior.EventDriven;
+import org.apache.nifi.annotation.behavior.InputRequirement;
+import org.apache.nifi.annotation.behavior.InputRequirement.Requirement;
+import org.apache.nifi.annotation.behavior.SideEffectFree;
+import org.apache.nifi.annotation.behavior.SupportsBatching;
+import org.apache.nifi.annotation.behavior.WritesAttribute;
+import org.apache.nifi.annotation.behavior.WritesAttributes;
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.annotation.lifecycle.OnScheduled;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.ValidationContext;
+import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.flowfile.attributes.CoreAttributes;
+import org.apache.nifi.logging.ComponentLog;
+import org.apache.nifi.processor.AbstractProcessor;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.processor.util.StandardValidators;
+import org.apache.nifi.record.path.RecordPath;
+import org.apache.nifi.record.path.RecordPathResult;
+import org.apache.nifi.schema.access.SchemaNotFoundException;
+import org.apache.nifi.security.util.crypto.HashAlgorithm;
+import org.apache.nifi.security.util.crypto.HashService;
+import org.apache.nifi.serialization.MalformedRecordException;
+import org.apache.nifi.serialization.RecordReader;
+import org.apache.nifi.serialization.RecordReaderFactory;
+import org.apache.nifi.serialization.RecordSetWriter;
+import org.apache.nifi.serialization.RecordSetWriterFactory;
+import org.apache.nifi.serialization.record.Record;
+import org.apache.nifi.serialization.record.RecordFieldType;
+import org.apache.nifi.serialization.record.RecordSchema;
+import org.apache.nifi.web.client.api.HttpResponseEntity;
+import org.apache.nifi.web.client.api.WebClientService;
+import org.apache.nifi.web.client.provider.api.WebClientServiceProvider;
+
+@Tags({"Workday", "report", "get"})
+@InputRequirement(Requirement.INPUT_ALLOWED)
+@CapabilityDescription("A processor which can interact with a configurable Workday Report. The processor can forward the content without modification, or you can transform it by"
+ + " providing the specific Record Reader and Record Writer services based on your needs. You can also hash, or remove fields using the input parameters and schemes in the Record writer. "
+ + "Supported Workday report formats are: csv, simplexml, json")
+@EventDriven
+@SideEffectFree
+@SupportsBatching
+@WritesAttributes({
+ @WritesAttribute(attribute = GetWorkdayReport.GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS, description = "The Java exception class raised when the processor fails"),
+ @WritesAttribute(attribute = GetWorkdayReport.GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE, description = "The Java exception message raised when the processor fails"),
+ @WritesAttribute(attribute = "mime.type", description = "Sets the mime.type attribute to the MIME Type specified by the Source / Record Writer"),
+ @WritesAttribute(attribute= GetWorkdayReport.RECORD_COUNT, description = "The number of records in an outgoing FlowFile. This is only populated on the 'success' relationship "
+ + "when Record Reader and Writer is set.")})
+public class GetWorkdayReport extends AbstractProcessor {
+
+ protected static final String STATUS_CODE = "getworkdayreport.status.code";
+ protected static final String REQUEST_URL = "getworkdayreport.request.url";
+ protected static final String REQUEST_DURATION = "getworkdayreport.request.duration";
+ protected static final String TRANSACTION_ID = "getworkdayreport.tx.id";
+ protected static final String GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS = "getworkdayreport.java.exception.class";
+ protected static final String GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE = "getworkdayreport.java.exception.message";
+ protected static final String RECORD_COUNT = "record.count";
+ protected static final String BASIC_PREFIX = "Basic ";
+ protected static final String COLUMNS_TO_HASH_SEPARATOR = ",";
+ protected static final String HEADER_AUTHORIZATION = "Authorization";
+ protected static final String HEADER_CONTENT_TYPE = "Content-Type";
+ protected static final String USERNAME_PASSWORD_SEPARATOR = ":";
+
+ protected static final PropertyDescriptor REPORT_URL = new PropertyDescriptor.Builder()
+ .name("Workday report URL")
+ .displayName("Workday report URL")
+ .description("HTTP remote URL of Workday report including a scheme of http or https, as well as a hostname or IP address with optional port and path elements.")
+ .required(true)
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .addValidator(URL_VALIDATOR)
+ .build();
+
+ protected static final PropertyDescriptor WORKDAY_USERNAME = new PropertyDescriptor.Builder()
+ .name("Workday Username")
+ .displayName("Workday Username")
+ .description("The username provided for authentication of Workday requests. Encoded using Base64 for HTTP Basic Authentication as described in RFC 7617.")
+ .required(true)
+ .addValidator(StandardValidators.createRegexMatchingValidator(Pattern.compile("^[\\x20-\\x39\\x3b-\\x7e\\x80-\\xff]+$")))
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .build();
+
+ protected static final PropertyDescriptor WORKDAY_PASSWORD = new PropertyDescriptor.Builder()
+ .name("Workday Password")
+ .displayName("Workday Password")
+ .description("The password provided for authentication of Workday requests. Encoded using Base64 for HTTP Basic Authentication as described in RFC 7617.")
+ .required(true)
+ .sensitive(true)
+ .addValidator(StandardValidators.createRegexMatchingValidator(Pattern.compile("^[\\x20-\\x7e\\x80-\\xff]+$")))
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .build();
+
+ protected static final PropertyDescriptor WEB_CLIENT_SERVICE = new PropertyDescriptor.Builder()
+ .name("Standard Web Client Service")
+ .description("Web client which is used to communicate with the Workday API.")
+ .required(true)
+ .identifiesControllerService(WebClientServiceProvider.class)
+ .build();
+
+ protected static final PropertyDescriptor FIELDS_TO_HASH = new PropertyDescriptor.Builder()
+ .name("Fields to hash")
+ .displayName("Fields to hash")
+ .description("Comma separated record paths to replace with hash.")
Review Comment:
This description is not quite clear, are the fields being replaced with a hash, or are the contents used to create a hash?
##########
nifi-nar-bundles/nifi-workday-bundle/nifi-workday-processors/src/main/java/org/apache/nifi/processors/workday/GetWorkdayReport.java:
##########
@@ -0,0 +1,432 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.processors.workday;
+
+import static org.apache.nifi.expression.ExpressionLanguageScope.FLOWFILE_ATTRIBUTES;
+import static org.apache.nifi.expression.ExpressionLanguageScope.NONE;
+import static org.apache.nifi.processor.util.StandardValidators.NON_BLANK_VALIDATOR;
+import static org.apache.nifi.processor.util.StandardValidators.URL_VALIDATOR;
+
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Base64;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import org.apache.nifi.annotation.behavior.EventDriven;
+import org.apache.nifi.annotation.behavior.InputRequirement;
+import org.apache.nifi.annotation.behavior.InputRequirement.Requirement;
+import org.apache.nifi.annotation.behavior.SideEffectFree;
+import org.apache.nifi.annotation.behavior.SupportsBatching;
+import org.apache.nifi.annotation.behavior.WritesAttribute;
+import org.apache.nifi.annotation.behavior.WritesAttributes;
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.annotation.lifecycle.OnScheduled;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.ValidationContext;
+import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.flowfile.attributes.CoreAttributes;
+import org.apache.nifi.logging.ComponentLog;
+import org.apache.nifi.processor.AbstractProcessor;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.processor.util.StandardValidators;
+import org.apache.nifi.record.path.RecordPath;
+import org.apache.nifi.record.path.RecordPathResult;
+import org.apache.nifi.schema.access.SchemaNotFoundException;
+import org.apache.nifi.security.util.crypto.HashAlgorithm;
+import org.apache.nifi.security.util.crypto.HashService;
+import org.apache.nifi.serialization.MalformedRecordException;
+import org.apache.nifi.serialization.RecordReader;
+import org.apache.nifi.serialization.RecordReaderFactory;
+import org.apache.nifi.serialization.RecordSetWriter;
+import org.apache.nifi.serialization.RecordSetWriterFactory;
+import org.apache.nifi.serialization.record.Record;
+import org.apache.nifi.serialization.record.RecordFieldType;
+import org.apache.nifi.serialization.record.RecordSchema;
+import org.apache.nifi.web.client.api.HttpResponseEntity;
+import org.apache.nifi.web.client.api.WebClientService;
+import org.apache.nifi.web.client.provider.api.WebClientServiceProvider;
+
+@Tags({"Workday", "report", "get"})
+@InputRequirement(Requirement.INPUT_ALLOWED)
+@CapabilityDescription("A processor which can interact with a configurable Workday Report. The processor can forward the content without modification, or you can transform it by"
+ + " providing the specific Record Reader and Record Writer services based on your needs. You can also hash, or remove fields using the input parameters and schemes in the Record writer. "
+ + "Supported Workday report formats are: csv, simplexml, json")
+@EventDriven
+@SideEffectFree
+@SupportsBatching
+@WritesAttributes({
+ @WritesAttribute(attribute = GetWorkdayReport.GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS, description = "The Java exception class raised when the processor fails"),
+ @WritesAttribute(attribute = GetWorkdayReport.GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE, description = "The Java exception message raised when the processor fails"),
+ @WritesAttribute(attribute = "mime.type", description = "Sets the mime.type attribute to the MIME Type specified by the Source / Record Writer"),
+ @WritesAttribute(attribute= GetWorkdayReport.RECORD_COUNT, description = "The number of records in an outgoing FlowFile. This is only populated on the 'success' relationship "
+ + "when Record Reader and Writer is set.")})
+public class GetWorkdayReport extends AbstractProcessor {
+
+ protected static final String STATUS_CODE = "getworkdayreport.status.code";
+ protected static final String REQUEST_URL = "getworkdayreport.request.url";
+ protected static final String REQUEST_DURATION = "getworkdayreport.request.duration";
+ protected static final String TRANSACTION_ID = "getworkdayreport.tx.id";
+ protected static final String GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS = "getworkdayreport.java.exception.class";
+ protected static final String GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE = "getworkdayreport.java.exception.message";
+ protected static final String RECORD_COUNT = "record.count";
+ protected static final String BASIC_PREFIX = "Basic ";
+ protected static final String COLUMNS_TO_HASH_SEPARATOR = ",";
+ protected static final String HEADER_AUTHORIZATION = "Authorization";
+ protected static final String HEADER_CONTENT_TYPE = "Content-Type";
+ protected static final String USERNAME_PASSWORD_SEPARATOR = ":";
+
+ protected static final PropertyDescriptor REPORT_URL = new PropertyDescriptor.Builder()
+ .name("Workday report URL")
+ .displayName("Workday report URL")
+ .description("HTTP remote URL of Workday report including a scheme of http or https, as well as a hostname or IP address with optional port and path elements.")
+ .required(true)
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .addValidator(URL_VALIDATOR)
+ .build();
+
+ protected static final PropertyDescriptor WORKDAY_USERNAME = new PropertyDescriptor.Builder()
+ .name("Workday Username")
+ .displayName("Workday Username")
+ .description("The username provided for authentication of Workday requests. Encoded using Base64 for HTTP Basic Authentication as described in RFC 7617.")
+ .required(true)
+ .addValidator(StandardValidators.createRegexMatchingValidator(Pattern.compile("^[\\x20-\\x39\\x3b-\\x7e\\x80-\\xff]+$")))
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .build();
+
+ protected static final PropertyDescriptor WORKDAY_PASSWORD = new PropertyDescriptor.Builder()
+ .name("Workday Password")
+ .displayName("Workday Password")
+ .description("The password provided for authentication of Workday requests. Encoded using Base64 for HTTP Basic Authentication as described in RFC 7617.")
+ .required(true)
+ .sensitive(true)
+ .addValidator(StandardValidators.createRegexMatchingValidator(Pattern.compile("^[\\x20-\\x7e\\x80-\\xff]+$")))
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .build();
+
+ protected static final PropertyDescriptor WEB_CLIENT_SERVICE = new PropertyDescriptor.Builder()
+ .name("Standard Web Client Service")
+ .description("Web client which is used to communicate with the Workday API.")
+ .required(true)
+ .identifiesControllerService(WebClientServiceProvider.class)
+ .build();
+
+ protected static final PropertyDescriptor FIELDS_TO_HASH = new PropertyDescriptor.Builder()
+ .name("Fields to hash")
+ .displayName("Fields to hash")
+ .description("Comma separated record paths to replace with hash.")
+ .required(false)
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .addValidator(NON_BLANK_VALIDATOR)
+ .build();
+
+ protected static final PropertyDescriptor HASHING_ALGORITHM = new PropertyDescriptor.Builder()
+ .name("Hashing algorithm")
+ .displayName("Hashing algorithm")
+ .description("Determines what hashing algorithm should be used to perform the hashing function.")
+ .required(true)
+ .allowableValues(HashService.buildHashAlgorithmAllowableValues())
+ .defaultValue(HashAlgorithm.SHA256.getName())
+ .addValidator(NON_BLANK_VALIDATOR)
+ .expressionLanguageSupported(NONE)
+ .dependsOn(FIELDS_TO_HASH)
+ .build();
+
+ protected static final PropertyDescriptor RECORD_READER_FACTORY = new PropertyDescriptor.Builder()
+ .name("record-reader")
+ .displayName("Record Reader")
+ .description("Specifies the Controller Service to use for parsing incoming data and determining the data's schema.")
+ .identifiesControllerService(RecordReaderFactory.class)
+ .required(false)
+ .build();
+
+ protected static final PropertyDescriptor RECORD_WRITER_FACTORY = new PropertyDescriptor.Builder()
+ .name("record-writer")
+ .displayName("Record Writer")
+ .description("The Record Writer to use for serializing Records to an output FlowFile.")
+ .identifiesControllerService(RecordSetWriterFactory.class)
+ .dependsOn(RECORD_READER_FACTORY)
+ .required(true)
+ .build();
+
+ protected static final Relationship ORIGINAL = new Relationship.Builder()
+ .name("Original")
+ .description("Request FlowFiles transferred when receiving HTTP responses with a status code between 200 and 299.")
+ .build();
+
+ protected static final Relationship FAILURE = new Relationship.Builder()
+ .name("Failure")
+ .description("Request FlowFiles transferred when receiving socket communication errors.")
+ .build();
+
+ protected static final Relationship RESPONSE = new Relationship.Builder()
+ .name("Response")
+ .description("Response FlowFiles transferred when receiving HTTP responses with a status code between 200 and 299.")
+ .build();
+
+ protected static final Set<Relationship> RELATIONSHIPS = Collections.unmodifiableSet(new HashSet<>(Arrays.asList(ORIGINAL, RESPONSE, FAILURE)));
+ protected static final List<PropertyDescriptor> PROPERTIES = Collections.unmodifiableList(Arrays.asList(REPORT_URL, WORKDAY_USERNAME, WORKDAY_PASSWORD, WEB_CLIENT_SERVICE,
+ FIELDS_TO_HASH, HASHING_ALGORITHM, RECORD_READER_FACTORY, RECORD_WRITER_FACTORY));
+
+ private final AtomicReference<WebClientService> webClientAtomicReference = new AtomicReference<>();
+ private final AtomicReference<RecordReaderFactory> recordReaderFactoryAtomicReference = new AtomicReference<>();
+ private final AtomicReference<RecordSetWriterFactory> recordSetWriterFactoryAtomicReference = new AtomicReference<>();
Review Comment:
Recommend shortening these names to remove `Atomic`:
```suggestion
private final AtomicReference<WebClientService> webClientReference = new AtomicReference<>();
private final AtomicReference<RecordReaderFactory> recordReaderFactoryReference = new AtomicReference<>();
private final AtomicReference<RecordSetWriterFactory> recordSetWriterFactoryReference = new AtomicReference<>();
```
##########
nifi-nar-bundles/nifi-workday-bundle/nifi-workday-processors/src/main/java/org/apache/nifi/processors/workday/GetWorkdayReport.java:
##########
@@ -0,0 +1,432 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.processors.workday;
+
+import static org.apache.nifi.expression.ExpressionLanguageScope.FLOWFILE_ATTRIBUTES;
+import static org.apache.nifi.expression.ExpressionLanguageScope.NONE;
+import static org.apache.nifi.processor.util.StandardValidators.NON_BLANK_VALIDATOR;
+import static org.apache.nifi.processor.util.StandardValidators.URL_VALIDATOR;
+
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Base64;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import org.apache.nifi.annotation.behavior.EventDriven;
+import org.apache.nifi.annotation.behavior.InputRequirement;
+import org.apache.nifi.annotation.behavior.InputRequirement.Requirement;
+import org.apache.nifi.annotation.behavior.SideEffectFree;
+import org.apache.nifi.annotation.behavior.SupportsBatching;
+import org.apache.nifi.annotation.behavior.WritesAttribute;
+import org.apache.nifi.annotation.behavior.WritesAttributes;
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.annotation.lifecycle.OnScheduled;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.ValidationContext;
+import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.flowfile.attributes.CoreAttributes;
+import org.apache.nifi.logging.ComponentLog;
+import org.apache.nifi.processor.AbstractProcessor;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.processor.util.StandardValidators;
+import org.apache.nifi.record.path.RecordPath;
+import org.apache.nifi.record.path.RecordPathResult;
+import org.apache.nifi.schema.access.SchemaNotFoundException;
+import org.apache.nifi.security.util.crypto.HashAlgorithm;
+import org.apache.nifi.security.util.crypto.HashService;
+import org.apache.nifi.serialization.MalformedRecordException;
+import org.apache.nifi.serialization.RecordReader;
+import org.apache.nifi.serialization.RecordReaderFactory;
+import org.apache.nifi.serialization.RecordSetWriter;
+import org.apache.nifi.serialization.RecordSetWriterFactory;
+import org.apache.nifi.serialization.record.Record;
+import org.apache.nifi.serialization.record.RecordFieldType;
+import org.apache.nifi.serialization.record.RecordSchema;
+import org.apache.nifi.web.client.api.HttpResponseEntity;
+import org.apache.nifi.web.client.api.WebClientService;
+import org.apache.nifi.web.client.provider.api.WebClientServiceProvider;
+
+@Tags({"Workday", "report", "get"})
+@InputRequirement(Requirement.INPUT_ALLOWED)
+@CapabilityDescription("A processor which can interact with a configurable Workday Report. The processor can forward the content without modification, or you can transform it by"
+ + " providing the specific Record Reader and Record Writer services based on your needs. You can also hash, or remove fields using the input parameters and schemes in the Record writer. "
+ + "Supported Workday report formats are: csv, simplexml, json")
+@EventDriven
+@SideEffectFree
+@SupportsBatching
+@WritesAttributes({
+ @WritesAttribute(attribute = GetWorkdayReport.GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS, description = "The Java exception class raised when the processor fails"),
+ @WritesAttribute(attribute = GetWorkdayReport.GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE, description = "The Java exception message raised when the processor fails"),
+ @WritesAttribute(attribute = "mime.type", description = "Sets the mime.type attribute to the MIME Type specified by the Source / Record Writer"),
+ @WritesAttribute(attribute= GetWorkdayReport.RECORD_COUNT, description = "The number of records in an outgoing FlowFile. This is only populated on the 'success' relationship "
+ + "when Record Reader and Writer is set.")})
+public class GetWorkdayReport extends AbstractProcessor {
+
+ protected static final String STATUS_CODE = "getworkdayreport.status.code";
+ protected static final String REQUEST_URL = "getworkdayreport.request.url";
+ protected static final String REQUEST_DURATION = "getworkdayreport.request.duration";
+ protected static final String TRANSACTION_ID = "getworkdayreport.tx.id";
+ protected static final String GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS = "getworkdayreport.java.exception.class";
+ protected static final String GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE = "getworkdayreport.java.exception.message";
+ protected static final String RECORD_COUNT = "record.count";
+ protected static final String BASIC_PREFIX = "Basic ";
+ protected static final String COLUMNS_TO_HASH_SEPARATOR = ",";
+ protected static final String HEADER_AUTHORIZATION = "Authorization";
+ protected static final String HEADER_CONTENT_TYPE = "Content-Type";
+ protected static final String USERNAME_PASSWORD_SEPARATOR = ":";
+
+ protected static final PropertyDescriptor REPORT_URL = new PropertyDescriptor.Builder()
+ .name("Workday report URL")
+ .displayName("Workday report URL")
+ .description("HTTP remote URL of Workday report including a scheme of http or https, as well as a hostname or IP address with optional port and path elements.")
+ .required(true)
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .addValidator(URL_VALIDATOR)
+ .build();
+
+ protected static final PropertyDescriptor WORKDAY_USERNAME = new PropertyDescriptor.Builder()
+ .name("Workday Username")
+ .displayName("Workday Username")
+ .description("The username provided for authentication of Workday requests. Encoded using Base64 for HTTP Basic Authentication as described in RFC 7617.")
+ .required(true)
+ .addValidator(StandardValidators.createRegexMatchingValidator(Pattern.compile("^[\\x20-\\x39\\x3b-\\x7e\\x80-\\xff]+$")))
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .build();
+
+ protected static final PropertyDescriptor WORKDAY_PASSWORD = new PropertyDescriptor.Builder()
+ .name("Workday Password")
+ .displayName("Workday Password")
+ .description("The password provided for authentication of Workday requests. Encoded using Base64 for HTTP Basic Authentication as described in RFC 7617.")
+ .required(true)
+ .sensitive(true)
+ .addValidator(StandardValidators.createRegexMatchingValidator(Pattern.compile("^[\\x20-\\x7e\\x80-\\xff]+$")))
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .build();
+
+ protected static final PropertyDescriptor WEB_CLIENT_SERVICE = new PropertyDescriptor.Builder()
+ .name("Standard Web Client Service")
+ .description("Web client which is used to communicate with the Workday API.")
+ .required(true)
+ .identifiesControllerService(WebClientServiceProvider.class)
+ .build();
+
+ protected static final PropertyDescriptor FIELDS_TO_HASH = new PropertyDescriptor.Builder()
+ .name("Fields to hash")
+ .displayName("Fields to hash")
+ .description("Comma separated record paths to replace with hash.")
+ .required(false)
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .addValidator(NON_BLANK_VALIDATOR)
+ .build();
+
+ protected static final PropertyDescriptor HASHING_ALGORITHM = new PropertyDescriptor.Builder()
+ .name("Hashing algorithm")
+ .displayName("Hashing algorithm")
+ .description("Determines what hashing algorithm should be used to perform the hashing function.")
+ .required(true)
+ .allowableValues(HashService.buildHashAlgorithmAllowableValues())
+ .defaultValue(HashAlgorithm.SHA256.getName())
+ .addValidator(NON_BLANK_VALIDATOR)
+ .expressionLanguageSupported(NONE)
+ .dependsOn(FIELDS_TO_HASH)
+ .build();
+
+ protected static final PropertyDescriptor RECORD_READER_FACTORY = new PropertyDescriptor.Builder()
+ .name("record-reader")
+ .displayName("Record Reader")
+ .description("Specifies the Controller Service to use for parsing incoming data and determining the data's schema.")
+ .identifiesControllerService(RecordReaderFactory.class)
+ .required(false)
+ .build();
+
+ protected static final PropertyDescriptor RECORD_WRITER_FACTORY = new PropertyDescriptor.Builder()
+ .name("record-writer")
+ .displayName("Record Writer")
+ .description("The Record Writer to use for serializing Records to an output FlowFile.")
+ .identifiesControllerService(RecordSetWriterFactory.class)
+ .dependsOn(RECORD_READER_FACTORY)
+ .required(true)
+ .build();
+
+ protected static final Relationship ORIGINAL = new Relationship.Builder()
+ .name("Original")
+ .description("Request FlowFiles transferred when receiving HTTP responses with a status code between 200 and 299.")
+ .build();
+
+ protected static final Relationship FAILURE = new Relationship.Builder()
+ .name("Failure")
+ .description("Request FlowFiles transferred when receiving socket communication errors.")
+ .build();
+
+ protected static final Relationship RESPONSE = new Relationship.Builder()
+ .name("Response")
+ .description("Response FlowFiles transferred when receiving HTTP responses with a status code between 200 and 299.")
+ .build();
+
+ protected static final Set<Relationship> RELATIONSHIPS = Collections.unmodifiableSet(new HashSet<>(Arrays.asList(ORIGINAL, RESPONSE, FAILURE)));
+ protected static final List<PropertyDescriptor> PROPERTIES = Collections.unmodifiableList(Arrays.asList(REPORT_URL, WORKDAY_USERNAME, WORKDAY_PASSWORD, WEB_CLIENT_SERVICE,
+ FIELDS_TO_HASH, HASHING_ALGORITHM, RECORD_READER_FACTORY, RECORD_WRITER_FACTORY));
+
+ private final AtomicReference<WebClientService> webClientAtomicReference = new AtomicReference<>();
+ private final AtomicReference<RecordReaderFactory> recordReaderFactoryAtomicReference = new AtomicReference<>();
+ private final AtomicReference<RecordSetWriterFactory> recordSetWriterFactoryAtomicReference = new AtomicReference<>();
+
+ @Override
+ protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
+ return PROPERTIES;
+ }
+
+ @Override
+ public Set<Relationship> getRelationships() {
+ return RELATIONSHIPS;
+ }
+
+ @OnScheduled
+ public void setUpClient(final ProcessContext context) {
+ WebClientServiceProvider standardWebClientServiceProvider = context.getProperty(WEB_CLIENT_SERVICE).asControllerService(WebClientServiceProvider.class);
+ RecordReaderFactory recordReaderFactory = context.getProperty(RECORD_READER_FACTORY).asControllerService(RecordReaderFactory.class);
+ RecordSetWriterFactory recordSetWriterFactory = context.getProperty(RECORD_WRITER_FACTORY).asControllerService(RecordSetWriterFactory.class);
+ WebClientService webClientService = standardWebClientServiceProvider.getWebClientService();
+ webClientAtomicReference.set(webClientService);
+ recordReaderFactoryAtomicReference.set(recordReaderFactory);
+ recordSetWriterFactoryAtomicReference.set(recordSetWriterFactory);
+ }
+
+ @Override
+ public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException {
+ FlowFile flowfile = session.get();
+
+ if (skipExecution(context, flowfile)) {
+ return;
+ }
+
+ ComponentLog logger = getLogger();
+ FlowFile responseFlowFile = null;
+
+ try {
+ WebClientService webClientService = webClientAtomicReference.get();
+ URI uri = new URI(context.getProperty(REPORT_URL).evaluateAttributeExpressions(flowfile).getValue().trim());
+ long startNanos = System.nanoTime();
+ String authorization = createAuthorizationHeader(context, flowfile);
+
+ try(HttpResponseEntity httpResponseEntity = webClientService.get().uri(uri).header(HEADER_AUTHORIZATION, authorization).retrieve()) {
+ responseFlowFile = createResponseFlowFile(flowfile, session, context, httpResponseEntity);
+ long elapsedTime = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos);
+ Map<String, String> commonAttributes = createCommonAttributes(uri, httpResponseEntity, elapsedTime);
+
+ if (flowfile != null) {
+ flowfile = session.putAllAttributes(flowfile, decorateWithMimeAttribute(commonAttributes, httpResponseEntity));
+ }
+ if (responseFlowFile != null) {
+ responseFlowFile = session.putAllAttributes(responseFlowFile, commonAttributes);
+ if (flowfile != null) {
+ session.getProvenanceReporter().fetch(responseFlowFile, uri.toString(), elapsedTime);
+ } else {
+ session.getProvenanceReporter().receive(responseFlowFile, uri.toString(), elapsedTime);
+ }
+ }
+
+ route(flowfile, responseFlowFile, session, context, httpResponseEntity.statusCode());
+ }
+ } catch (Exception e) {
+ if (flowfile == null) {
+ logger.error("Request Processing failed", e);
+ context.yield();
+ } else {
+ logger.error("Request Processing failed: {}", flowfile, e);
+ session.penalize(flowfile);
+ flowfile = session.putAttribute(flowfile, GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS, e.getClass().getName());
Review Comment:
Recommend using `getClass().getSimpleName()`. Although this is less specific, it avoids tighter coupling to the specific Exception class.
##########
nifi-nar-bundles/nifi-workday-bundle/nifi-workday-processors/src/main/java/org/apache/nifi/processors/workday/GetWorkdayReport.java:
##########
@@ -0,0 +1,432 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.processors.workday;
+
+import static org.apache.nifi.expression.ExpressionLanguageScope.FLOWFILE_ATTRIBUTES;
+import static org.apache.nifi.expression.ExpressionLanguageScope.NONE;
+import static org.apache.nifi.processor.util.StandardValidators.NON_BLANK_VALIDATOR;
+import static org.apache.nifi.processor.util.StandardValidators.URL_VALIDATOR;
+
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Base64;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import org.apache.nifi.annotation.behavior.EventDriven;
+import org.apache.nifi.annotation.behavior.InputRequirement;
+import org.apache.nifi.annotation.behavior.InputRequirement.Requirement;
+import org.apache.nifi.annotation.behavior.SideEffectFree;
+import org.apache.nifi.annotation.behavior.SupportsBatching;
+import org.apache.nifi.annotation.behavior.WritesAttribute;
+import org.apache.nifi.annotation.behavior.WritesAttributes;
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.annotation.lifecycle.OnScheduled;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.ValidationContext;
+import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.flowfile.attributes.CoreAttributes;
+import org.apache.nifi.logging.ComponentLog;
+import org.apache.nifi.processor.AbstractProcessor;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.processor.util.StandardValidators;
+import org.apache.nifi.record.path.RecordPath;
+import org.apache.nifi.record.path.RecordPathResult;
+import org.apache.nifi.schema.access.SchemaNotFoundException;
+import org.apache.nifi.security.util.crypto.HashAlgorithm;
+import org.apache.nifi.security.util.crypto.HashService;
+import org.apache.nifi.serialization.MalformedRecordException;
+import org.apache.nifi.serialization.RecordReader;
+import org.apache.nifi.serialization.RecordReaderFactory;
+import org.apache.nifi.serialization.RecordSetWriter;
+import org.apache.nifi.serialization.RecordSetWriterFactory;
+import org.apache.nifi.serialization.record.Record;
+import org.apache.nifi.serialization.record.RecordFieldType;
+import org.apache.nifi.serialization.record.RecordSchema;
+import org.apache.nifi.web.client.api.HttpResponseEntity;
+import org.apache.nifi.web.client.api.WebClientService;
+import org.apache.nifi.web.client.provider.api.WebClientServiceProvider;
+
+@Tags({"Workday", "report", "get"})
+@InputRequirement(Requirement.INPUT_ALLOWED)
+@CapabilityDescription("A processor which can interact with a configurable Workday Report. The processor can forward the content without modification, or you can transform it by"
+ + " providing the specific Record Reader and Record Writer services based on your needs. You can also hash, or remove fields using the input parameters and schemes in the Record writer. "
+ + "Supported Workday report formats are: csv, simplexml, json")
+@EventDriven
+@SideEffectFree
+@SupportsBatching
+@WritesAttributes({
+ @WritesAttribute(attribute = GetWorkdayReport.GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS, description = "The Java exception class raised when the processor fails"),
+ @WritesAttribute(attribute = GetWorkdayReport.GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE, description = "The Java exception message raised when the processor fails"),
+ @WritesAttribute(attribute = "mime.type", description = "Sets the mime.type attribute to the MIME Type specified by the Source / Record Writer"),
+ @WritesAttribute(attribute= GetWorkdayReport.RECORD_COUNT, description = "The number of records in an outgoing FlowFile. This is only populated on the 'success' relationship "
+ + "when Record Reader and Writer is set.")})
+public class GetWorkdayReport extends AbstractProcessor {
+
+ protected static final String STATUS_CODE = "getworkdayreport.status.code";
+ protected static final String REQUEST_URL = "getworkdayreport.request.url";
+ protected static final String REQUEST_DURATION = "getworkdayreport.request.duration";
+ protected static final String TRANSACTION_ID = "getworkdayreport.tx.id";
+ protected static final String GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS = "getworkdayreport.java.exception.class";
+ protected static final String GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE = "getworkdayreport.java.exception.message";
+ protected static final String RECORD_COUNT = "record.count";
+ protected static final String BASIC_PREFIX = "Basic ";
+ protected static final String COLUMNS_TO_HASH_SEPARATOR = ",";
+ protected static final String HEADER_AUTHORIZATION = "Authorization";
+ protected static final String HEADER_CONTENT_TYPE = "Content-Type";
+ protected static final String USERNAME_PASSWORD_SEPARATOR = ":";
+
+ protected static final PropertyDescriptor REPORT_URL = new PropertyDescriptor.Builder()
+ .name("Workday report URL")
+ .displayName("Workday report URL")
+ .description("HTTP remote URL of Workday report including a scheme of http or https, as well as a hostname or IP address with optional port and path elements.")
+ .required(true)
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .addValidator(URL_VALIDATOR)
+ .build();
+
+ protected static final PropertyDescriptor WORKDAY_USERNAME = new PropertyDescriptor.Builder()
+ .name("Workday Username")
+ .displayName("Workday Username")
+ .description("The username provided for authentication of Workday requests. Encoded using Base64 for HTTP Basic Authentication as described in RFC 7617.")
+ .required(true)
+ .addValidator(StandardValidators.createRegexMatchingValidator(Pattern.compile("^[\\x20-\\x39\\x3b-\\x7e\\x80-\\xff]+$")))
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .build();
+
+ protected static final PropertyDescriptor WORKDAY_PASSWORD = new PropertyDescriptor.Builder()
+ .name("Workday Password")
+ .displayName("Workday Password")
+ .description("The password provided for authentication of Workday requests. Encoded using Base64 for HTTP Basic Authentication as described in RFC 7617.")
+ .required(true)
+ .sensitive(true)
+ .addValidator(StandardValidators.createRegexMatchingValidator(Pattern.compile("^[\\x20-\\x7e\\x80-\\xff]+$")))
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .build();
+
+ protected static final PropertyDescriptor WEB_CLIENT_SERVICE = new PropertyDescriptor.Builder()
+ .name("Standard Web Client Service")
+ .description("Web client which is used to communicate with the Workday API.")
+ .required(true)
+ .identifiesControllerService(WebClientServiceProvider.class)
+ .build();
+
+ protected static final PropertyDescriptor FIELDS_TO_HASH = new PropertyDescriptor.Builder()
+ .name("Fields to hash")
+ .displayName("Fields to hash")
+ .description("Comma separated record paths to replace with hash.")
+ .required(false)
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .addValidator(NON_BLANK_VALIDATOR)
+ .build();
+
+ protected static final PropertyDescriptor HASHING_ALGORITHM = new PropertyDescriptor.Builder()
+ .name("Hashing algorithm")
+ .displayName("Hashing algorithm")
+ .description("Determines what hashing algorithm should be used to perform the hashing function.")
+ .required(true)
+ .allowableValues(HashService.buildHashAlgorithmAllowableValues())
+ .defaultValue(HashAlgorithm.SHA256.getName())
+ .addValidator(NON_BLANK_VALIDATOR)
+ .expressionLanguageSupported(NONE)
+ .dependsOn(FIELDS_TO_HASH)
+ .build();
+
+ protected static final PropertyDescriptor RECORD_READER_FACTORY = new PropertyDescriptor.Builder()
+ .name("record-reader")
+ .displayName("Record Reader")
+ .description("Specifies the Controller Service to use for parsing incoming data and determining the data's schema.")
+ .identifiesControllerService(RecordReaderFactory.class)
+ .required(false)
+ .build();
+
+ protected static final PropertyDescriptor RECORD_WRITER_FACTORY = new PropertyDescriptor.Builder()
+ .name("record-writer")
+ .displayName("Record Writer")
+ .description("The Record Writer to use for serializing Records to an output FlowFile.")
+ .identifiesControllerService(RecordSetWriterFactory.class)
+ .dependsOn(RECORD_READER_FACTORY)
+ .required(true)
+ .build();
+
+ protected static final Relationship ORIGINAL = new Relationship.Builder()
+ .name("Original")
+ .description("Request FlowFiles transferred when receiving HTTP responses with a status code between 200 and 299.")
+ .build();
+
+ protected static final Relationship FAILURE = new Relationship.Builder()
+ .name("Failure")
+ .description("Request FlowFiles transferred when receiving socket communication errors.")
+ .build();
+
+ protected static final Relationship RESPONSE = new Relationship.Builder()
+ .name("Response")
+ .description("Response FlowFiles transferred when receiving HTTP responses with a status code between 200 and 299.")
+ .build();
+
+ protected static final Set<Relationship> RELATIONSHIPS = Collections.unmodifiableSet(new HashSet<>(Arrays.asList(ORIGINAL, RESPONSE, FAILURE)));
+ protected static final List<PropertyDescriptor> PROPERTIES = Collections.unmodifiableList(Arrays.asList(REPORT_URL, WORKDAY_USERNAME, WORKDAY_PASSWORD, WEB_CLIENT_SERVICE,
+ FIELDS_TO_HASH, HASHING_ALGORITHM, RECORD_READER_FACTORY, RECORD_WRITER_FACTORY));
+
+ private final AtomicReference<WebClientService> webClientAtomicReference = new AtomicReference<>();
+ private final AtomicReference<RecordReaderFactory> recordReaderFactoryAtomicReference = new AtomicReference<>();
+ private final AtomicReference<RecordSetWriterFactory> recordSetWriterFactoryAtomicReference = new AtomicReference<>();
+
+ @Override
+ protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
+ return PROPERTIES;
+ }
+
+ @Override
+ public Set<Relationship> getRelationships() {
+ return RELATIONSHIPS;
+ }
+
+ @OnScheduled
+ public void setUpClient(final ProcessContext context) {
+ WebClientServiceProvider standardWebClientServiceProvider = context.getProperty(WEB_CLIENT_SERVICE).asControllerService(WebClientServiceProvider.class);
+ RecordReaderFactory recordReaderFactory = context.getProperty(RECORD_READER_FACTORY).asControllerService(RecordReaderFactory.class);
+ RecordSetWriterFactory recordSetWriterFactory = context.getProperty(RECORD_WRITER_FACTORY).asControllerService(RecordSetWriterFactory.class);
+ WebClientService webClientService = standardWebClientServiceProvider.getWebClientService();
+ webClientAtomicReference.set(webClientService);
+ recordReaderFactoryAtomicReference.set(recordReaderFactory);
+ recordSetWriterFactoryAtomicReference.set(recordSetWriterFactory);
+ }
+
+ @Override
+ public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException {
+ FlowFile flowfile = session.get();
+
+ if (skipExecution(context, flowfile)) {
+ return;
+ }
+
+ ComponentLog logger = getLogger();
+ FlowFile responseFlowFile = null;
+
+ try {
+ WebClientService webClientService = webClientAtomicReference.get();
+ URI uri = new URI(context.getProperty(REPORT_URL).evaluateAttributeExpressions(flowfile).getValue().trim());
+ long startNanos = System.nanoTime();
+ String authorization = createAuthorizationHeader(context, flowfile);
+
+ try(HttpResponseEntity httpResponseEntity = webClientService.get().uri(uri).header(HEADER_AUTHORIZATION, authorization).retrieve()) {
+ responseFlowFile = createResponseFlowFile(flowfile, session, context, httpResponseEntity);
+ long elapsedTime = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos);
+ Map<String, String> commonAttributes = createCommonAttributes(uri, httpResponseEntity, elapsedTime);
+
+ if (flowfile != null) {
+ flowfile = session.putAllAttributes(flowfile, decorateWithMimeAttribute(commonAttributes, httpResponseEntity));
+ }
+ if (responseFlowFile != null) {
+ responseFlowFile = session.putAllAttributes(responseFlowFile, commonAttributes);
+ if (flowfile != null) {
+ session.getProvenanceReporter().fetch(responseFlowFile, uri.toString(), elapsedTime);
+ } else {
+ session.getProvenanceReporter().receive(responseFlowFile, uri.toString(), elapsedTime);
+ }
Review Comment:
Recommend swapping the logic to use `==` instead of `!=`:
```suggestion
if (flowfile == null) {
session.getProvenanceReporter().receive(responseFlowFile, uri.toString(), elapsedTime);
} else {
session.getProvenanceReporter().fetch(responseFlowFile, uri.toString(), elapsedTime);
}
```
##########
nifi-nar-bundles/nifi-workday-bundle/nifi-workday-processors/src/main/java/org/apache/nifi/processors/workday/GetWorkdayReport.java:
##########
@@ -0,0 +1,432 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.processors.workday;
+
+import static org.apache.nifi.expression.ExpressionLanguageScope.FLOWFILE_ATTRIBUTES;
+import static org.apache.nifi.expression.ExpressionLanguageScope.NONE;
+import static org.apache.nifi.processor.util.StandardValidators.NON_BLANK_VALIDATOR;
+import static org.apache.nifi.processor.util.StandardValidators.URL_VALIDATOR;
+
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Base64;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import org.apache.nifi.annotation.behavior.EventDriven;
+import org.apache.nifi.annotation.behavior.InputRequirement;
+import org.apache.nifi.annotation.behavior.InputRequirement.Requirement;
+import org.apache.nifi.annotation.behavior.SideEffectFree;
+import org.apache.nifi.annotation.behavior.SupportsBatching;
+import org.apache.nifi.annotation.behavior.WritesAttribute;
+import org.apache.nifi.annotation.behavior.WritesAttributes;
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.annotation.lifecycle.OnScheduled;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.ValidationContext;
+import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.flowfile.attributes.CoreAttributes;
+import org.apache.nifi.logging.ComponentLog;
+import org.apache.nifi.processor.AbstractProcessor;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.processor.util.StandardValidators;
+import org.apache.nifi.record.path.RecordPath;
+import org.apache.nifi.record.path.RecordPathResult;
+import org.apache.nifi.schema.access.SchemaNotFoundException;
+import org.apache.nifi.security.util.crypto.HashAlgorithm;
+import org.apache.nifi.security.util.crypto.HashService;
+import org.apache.nifi.serialization.MalformedRecordException;
+import org.apache.nifi.serialization.RecordReader;
+import org.apache.nifi.serialization.RecordReaderFactory;
+import org.apache.nifi.serialization.RecordSetWriter;
+import org.apache.nifi.serialization.RecordSetWriterFactory;
+import org.apache.nifi.serialization.record.Record;
+import org.apache.nifi.serialization.record.RecordFieldType;
+import org.apache.nifi.serialization.record.RecordSchema;
+import org.apache.nifi.web.client.api.HttpResponseEntity;
+import org.apache.nifi.web.client.api.WebClientService;
+import org.apache.nifi.web.client.provider.api.WebClientServiceProvider;
+
+@Tags({"Workday", "report", "get"})
+@InputRequirement(Requirement.INPUT_ALLOWED)
+@CapabilityDescription("A processor which can interact with a configurable Workday Report. The processor can forward the content without modification, or you can transform it by"
+ + " providing the specific Record Reader and Record Writer services based on your needs. You can also hash, or remove fields using the input parameters and schemes in the Record writer. "
+ + "Supported Workday report formats are: csv, simplexml, json")
+@EventDriven
+@SideEffectFree
+@SupportsBatching
+@WritesAttributes({
+ @WritesAttribute(attribute = GetWorkdayReport.GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS, description = "The Java exception class raised when the processor fails"),
+ @WritesAttribute(attribute = GetWorkdayReport.GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE, description = "The Java exception message raised when the processor fails"),
+ @WritesAttribute(attribute = "mime.type", description = "Sets the mime.type attribute to the MIME Type specified by the Source / Record Writer"),
+ @WritesAttribute(attribute= GetWorkdayReport.RECORD_COUNT, description = "The number of records in an outgoing FlowFile. This is only populated on the 'success' relationship "
+ + "when Record Reader and Writer is set.")})
+public class GetWorkdayReport extends AbstractProcessor {
+
+ protected static final String STATUS_CODE = "getworkdayreport.status.code";
+ protected static final String REQUEST_URL = "getworkdayreport.request.url";
+ protected static final String REQUEST_DURATION = "getworkdayreport.request.duration";
+ protected static final String TRANSACTION_ID = "getworkdayreport.tx.id";
+ protected static final String GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS = "getworkdayreport.java.exception.class";
+ protected static final String GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE = "getworkdayreport.java.exception.message";
+ protected static final String RECORD_COUNT = "record.count";
+ protected static final String BASIC_PREFIX = "Basic ";
+ protected static final String COLUMNS_TO_HASH_SEPARATOR = ",";
+ protected static final String HEADER_AUTHORIZATION = "Authorization";
+ protected static final String HEADER_CONTENT_TYPE = "Content-Type";
+ protected static final String USERNAME_PASSWORD_SEPARATOR = ":";
+
+ protected static final PropertyDescriptor REPORT_URL = new PropertyDescriptor.Builder()
+ .name("Workday report URL")
+ .displayName("Workday report URL")
+ .description("HTTP remote URL of Workday report including a scheme of http or https, as well as a hostname or IP address with optional port and path elements.")
+ .required(true)
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .addValidator(URL_VALIDATOR)
+ .build();
+
+ protected static final PropertyDescriptor WORKDAY_USERNAME = new PropertyDescriptor.Builder()
+ .name("Workday Username")
+ .displayName("Workday Username")
+ .description("The username provided for authentication of Workday requests. Encoded using Base64 for HTTP Basic Authentication as described in RFC 7617.")
+ .required(true)
+ .addValidator(StandardValidators.createRegexMatchingValidator(Pattern.compile("^[\\x20-\\x39\\x3b-\\x7e\\x80-\\xff]+$")))
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .build();
+
+ protected static final PropertyDescriptor WORKDAY_PASSWORD = new PropertyDescriptor.Builder()
+ .name("Workday Password")
+ .displayName("Workday Password")
+ .description("The password provided for authentication of Workday requests. Encoded using Base64 for HTTP Basic Authentication as described in RFC 7617.")
+ .required(true)
+ .sensitive(true)
+ .addValidator(StandardValidators.createRegexMatchingValidator(Pattern.compile("^[\\x20-\\x7e\\x80-\\xff]+$")))
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .build();
+
+ protected static final PropertyDescriptor WEB_CLIENT_SERVICE = new PropertyDescriptor.Builder()
+ .name("Standard Web Client Service")
+ .description("Web client which is used to communicate with the Workday API.")
+ .required(true)
+ .identifiesControllerService(WebClientServiceProvider.class)
+ .build();
+
+ protected static final PropertyDescriptor FIELDS_TO_HASH = new PropertyDescriptor.Builder()
+ .name("Fields to hash")
+ .displayName("Fields to hash")
+ .description("Comma separated record paths to replace with hash.")
+ .required(false)
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .addValidator(NON_BLANK_VALIDATOR)
+ .build();
+
+ protected static final PropertyDescriptor HASHING_ALGORITHM = new PropertyDescriptor.Builder()
+ .name("Hashing algorithm")
+ .displayName("Hashing algorithm")
+ .description("Determines what hashing algorithm should be used to perform the hashing function.")
+ .required(true)
+ .allowableValues(HashService.buildHashAlgorithmAllowableValues())
+ .defaultValue(HashAlgorithm.SHA256.getName())
+ .addValidator(NON_BLANK_VALIDATOR)
+ .expressionLanguageSupported(NONE)
+ .dependsOn(FIELDS_TO_HASH)
+ .build();
+
+ protected static final PropertyDescriptor RECORD_READER_FACTORY = new PropertyDescriptor.Builder()
+ .name("record-reader")
+ .displayName("Record Reader")
+ .description("Specifies the Controller Service to use for parsing incoming data and determining the data's schema.")
+ .identifiesControllerService(RecordReaderFactory.class)
+ .required(false)
+ .build();
+
+ protected static final PropertyDescriptor RECORD_WRITER_FACTORY = new PropertyDescriptor.Builder()
+ .name("record-writer")
+ .displayName("Record Writer")
+ .description("The Record Writer to use for serializing Records to an output FlowFile.")
+ .identifiesControllerService(RecordSetWriterFactory.class)
+ .dependsOn(RECORD_READER_FACTORY)
+ .required(true)
+ .build();
+
+ protected static final Relationship ORIGINAL = new Relationship.Builder()
+ .name("Original")
+ .description("Request FlowFiles transferred when receiving HTTP responses with a status code between 200 and 299.")
+ .build();
+
+ protected static final Relationship FAILURE = new Relationship.Builder()
+ .name("Failure")
+ .description("Request FlowFiles transferred when receiving socket communication errors.")
+ .build();
+
+ protected static final Relationship RESPONSE = new Relationship.Builder()
+ .name("Response")
+ .description("Response FlowFiles transferred when receiving HTTP responses with a status code between 200 and 299.")
+ .build();
+
+ protected static final Set<Relationship> RELATIONSHIPS = Collections.unmodifiableSet(new HashSet<>(Arrays.asList(ORIGINAL, RESPONSE, FAILURE)));
+ protected static final List<PropertyDescriptor> PROPERTIES = Collections.unmodifiableList(Arrays.asList(REPORT_URL, WORKDAY_USERNAME, WORKDAY_PASSWORD, WEB_CLIENT_SERVICE,
+ FIELDS_TO_HASH, HASHING_ALGORITHM, RECORD_READER_FACTORY, RECORD_WRITER_FACTORY));
+
+ private final AtomicReference<WebClientService> webClientAtomicReference = new AtomicReference<>();
+ private final AtomicReference<RecordReaderFactory> recordReaderFactoryAtomicReference = new AtomicReference<>();
+ private final AtomicReference<RecordSetWriterFactory> recordSetWriterFactoryAtomicReference = new AtomicReference<>();
+
+ @Override
+ protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
+ return PROPERTIES;
+ }
+
+ @Override
+ public Set<Relationship> getRelationships() {
+ return RELATIONSHIPS;
+ }
+
+ @OnScheduled
+ public void setUpClient(final ProcessContext context) {
+ WebClientServiceProvider standardWebClientServiceProvider = context.getProperty(WEB_CLIENT_SERVICE).asControllerService(WebClientServiceProvider.class);
+ RecordReaderFactory recordReaderFactory = context.getProperty(RECORD_READER_FACTORY).asControllerService(RecordReaderFactory.class);
+ RecordSetWriterFactory recordSetWriterFactory = context.getProperty(RECORD_WRITER_FACTORY).asControllerService(RecordSetWriterFactory.class);
+ WebClientService webClientService = standardWebClientServiceProvider.getWebClientService();
+ webClientAtomicReference.set(webClientService);
+ recordReaderFactoryAtomicReference.set(recordReaderFactory);
+ recordSetWriterFactoryAtomicReference.set(recordSetWriterFactory);
+ }
+
+ @Override
+ public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException {
+ FlowFile flowfile = session.get();
+
+ if (skipExecution(context, flowfile)) {
+ return;
+ }
+
+ ComponentLog logger = getLogger();
+ FlowFile responseFlowFile = null;
+
+ try {
+ WebClientService webClientService = webClientAtomicReference.get();
+ URI uri = new URI(context.getProperty(REPORT_URL).evaluateAttributeExpressions(flowfile).getValue().trim());
+ long startNanos = System.nanoTime();
+ String authorization = createAuthorizationHeader(context, flowfile);
+
+ try(HttpResponseEntity httpResponseEntity = webClientService.get().uri(uri).header(HEADER_AUTHORIZATION, authorization).retrieve()) {
+ responseFlowFile = createResponseFlowFile(flowfile, session, context, httpResponseEntity);
+ long elapsedTime = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos);
+ Map<String, String> commonAttributes = createCommonAttributes(uri, httpResponseEntity, elapsedTime);
+
+ if (flowfile != null) {
+ flowfile = session.putAllAttributes(flowfile, decorateWithMimeAttribute(commonAttributes, httpResponseEntity));
+ }
+ if (responseFlowFile != null) {
+ responseFlowFile = session.putAllAttributes(responseFlowFile, commonAttributes);
+ if (flowfile != null) {
+ session.getProvenanceReporter().fetch(responseFlowFile, uri.toString(), elapsedTime);
+ } else {
+ session.getProvenanceReporter().receive(responseFlowFile, uri.toString(), elapsedTime);
+ }
+ }
+
+ route(flowfile, responseFlowFile, session, context, httpResponseEntity.statusCode());
+ }
+ } catch (Exception e) {
+ if (flowfile == null) {
+ logger.error("Request Processing failed", e);
+ context.yield();
+ } else {
+ logger.error("Request Processing failed: {}", flowfile, e);
+ session.penalize(flowfile);
+ flowfile = session.putAttribute(flowfile, GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS, e.getClass().getName());
+ flowfile = session.putAttribute(flowfile, GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE, e.getMessage());
+ session.transfer(flowfile, FAILURE);
+ }
+
+ if (responseFlowFile != null) {
+ session.remove(responseFlowFile);
+ }
+ }
+ }
+
+ @Override
+ protected Collection<ValidationResult> customValidate(ValidationContext validationContext) {
+ List<ValidationResult> results = new ArrayList<>(super.customValidate(validationContext));
+ if (validationContext.getProperty(FIELDS_TO_HASH).isSet() && !validationContext.getProperty(RECORD_READER_FACTORY).isSet()) {
+ results.add(new ValidationResult.Builder()
+ .valid(false)
+ .explanation("Record-Reader and Record-Writer must be set if you would like to hash specific fields")
+ .subject("Workday report configuration")
+ .build());
+ }
+ return results;
+ }
+
+ /*
+ * If we have no FlowFile, and all incoming connections are self-loops then we can continue on.
+ * However, if we have no FlowFile and we have connections coming from other Processors, then
+ * we know that we should run only if we have a FlowFile.
+ */
+ private boolean skipExecution(ProcessContext context, FlowFile flowfile) {
+ return context.hasIncomingConnection() && flowfile == null && context.hasNonLoopConnection();
+ }
+
+ private FlowFile createResponseFlowFile(FlowFile flowfile, ProcessSession session, ProcessContext context, HttpResponseEntity httpResponseEntity)
+ throws IOException, SchemaNotFoundException, MalformedRecordException {
+ FlowFile responseFlowFile = null;
+ String hashingAlgorithm = context.getProperty(HASHING_ALGORITHM).getValue();
+ Set<String> columnsToHash = Optional.ofNullable(context.getProperty(FIELDS_TO_HASH).evaluateAttributeExpressions(flowfile).getValue()).map(String::trim)
+ .map(columns -> columns.split(COLUMNS_TO_HASH_SEPARATOR)).map(Arrays::stream).map(columns -> columns.collect(Collectors.toSet())).orElse(Collections.emptySet());
+ try {
+ if (isSuccess(httpResponseEntity.statusCode())) {
+ responseFlowFile = flowfile != null ? session.create(flowfile) : session.create();
Review Comment:
Recommend adjusting the logic as follows:
```suggestion
responseFlowFile = flowfile == null ? session.create() : session.create(flowfile);
```
##########
nifi-nar-bundles/nifi-workday-bundle/nifi-workday-processors/src/main/java/org/apache/nifi/processors/workday/GetWorkdayReport.java:
##########
@@ -0,0 +1,432 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.processors.workday;
+
+import static org.apache.nifi.expression.ExpressionLanguageScope.FLOWFILE_ATTRIBUTES;
+import static org.apache.nifi.expression.ExpressionLanguageScope.NONE;
+import static org.apache.nifi.processor.util.StandardValidators.NON_BLANK_VALIDATOR;
+import static org.apache.nifi.processor.util.StandardValidators.URL_VALIDATOR;
+
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Base64;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import org.apache.nifi.annotation.behavior.EventDriven;
+import org.apache.nifi.annotation.behavior.InputRequirement;
+import org.apache.nifi.annotation.behavior.InputRequirement.Requirement;
+import org.apache.nifi.annotation.behavior.SideEffectFree;
+import org.apache.nifi.annotation.behavior.SupportsBatching;
+import org.apache.nifi.annotation.behavior.WritesAttribute;
+import org.apache.nifi.annotation.behavior.WritesAttributes;
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.annotation.lifecycle.OnScheduled;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.ValidationContext;
+import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.flowfile.attributes.CoreAttributes;
+import org.apache.nifi.logging.ComponentLog;
+import org.apache.nifi.processor.AbstractProcessor;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.processor.util.StandardValidators;
+import org.apache.nifi.record.path.RecordPath;
+import org.apache.nifi.record.path.RecordPathResult;
+import org.apache.nifi.schema.access.SchemaNotFoundException;
+import org.apache.nifi.security.util.crypto.HashAlgorithm;
+import org.apache.nifi.security.util.crypto.HashService;
+import org.apache.nifi.serialization.MalformedRecordException;
+import org.apache.nifi.serialization.RecordReader;
+import org.apache.nifi.serialization.RecordReaderFactory;
+import org.apache.nifi.serialization.RecordSetWriter;
+import org.apache.nifi.serialization.RecordSetWriterFactory;
+import org.apache.nifi.serialization.record.Record;
+import org.apache.nifi.serialization.record.RecordFieldType;
+import org.apache.nifi.serialization.record.RecordSchema;
+import org.apache.nifi.web.client.api.HttpResponseEntity;
+import org.apache.nifi.web.client.api.WebClientService;
+import org.apache.nifi.web.client.provider.api.WebClientServiceProvider;
+
+@Tags({"Workday", "report", "get"})
+@InputRequirement(Requirement.INPUT_ALLOWED)
+@CapabilityDescription("A processor which can interact with a configurable Workday Report. The processor can forward the content without modification, or you can transform it by"
+ + " providing the specific Record Reader and Record Writer services based on your needs. You can also hash, or remove fields using the input parameters and schemes in the Record writer. "
+ + "Supported Workday report formats are: csv, simplexml, json")
+@EventDriven
+@SideEffectFree
+@SupportsBatching
+@WritesAttributes({
+ @WritesAttribute(attribute = GetWorkdayReport.GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS, description = "The Java exception class raised when the processor fails"),
+ @WritesAttribute(attribute = GetWorkdayReport.GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE, description = "The Java exception message raised when the processor fails"),
+ @WritesAttribute(attribute = "mime.type", description = "Sets the mime.type attribute to the MIME Type specified by the Source / Record Writer"),
+ @WritesAttribute(attribute= GetWorkdayReport.RECORD_COUNT, description = "The number of records in an outgoing FlowFile. This is only populated on the 'success' relationship "
+ + "when Record Reader and Writer is set.")})
+public class GetWorkdayReport extends AbstractProcessor {
+
+ protected static final String STATUS_CODE = "getworkdayreport.status.code";
+ protected static final String REQUEST_URL = "getworkdayreport.request.url";
+ protected static final String REQUEST_DURATION = "getworkdayreport.request.duration";
+ protected static final String TRANSACTION_ID = "getworkdayreport.tx.id";
+ protected static final String GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS = "getworkdayreport.java.exception.class";
+ protected static final String GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE = "getworkdayreport.java.exception.message";
+ protected static final String RECORD_COUNT = "record.count";
+ protected static final String BASIC_PREFIX = "Basic ";
+ protected static final String COLUMNS_TO_HASH_SEPARATOR = ",";
+ protected static final String HEADER_AUTHORIZATION = "Authorization";
+ protected static final String HEADER_CONTENT_TYPE = "Content-Type";
+ protected static final String USERNAME_PASSWORD_SEPARATOR = ":";
+
+ protected static final PropertyDescriptor REPORT_URL = new PropertyDescriptor.Builder()
+ .name("Workday report URL")
+ .displayName("Workday report URL")
+ .description("HTTP remote URL of Workday report including a scheme of http or https, as well as a hostname or IP address with optional port and path elements.")
+ .required(true)
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .addValidator(URL_VALIDATOR)
+ .build();
+
+ protected static final PropertyDescriptor WORKDAY_USERNAME = new PropertyDescriptor.Builder()
+ .name("Workday Username")
+ .displayName("Workday Username")
+ .description("The username provided for authentication of Workday requests. Encoded using Base64 for HTTP Basic Authentication as described in RFC 7617.")
+ .required(true)
+ .addValidator(StandardValidators.createRegexMatchingValidator(Pattern.compile("^[\\x20-\\x39\\x3b-\\x7e\\x80-\\xff]+$")))
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .build();
+
+ protected static final PropertyDescriptor WORKDAY_PASSWORD = new PropertyDescriptor.Builder()
+ .name("Workday Password")
+ .displayName("Workday Password")
+ .description("The password provided for authentication of Workday requests. Encoded using Base64 for HTTP Basic Authentication as described in RFC 7617.")
+ .required(true)
+ .sensitive(true)
+ .addValidator(StandardValidators.createRegexMatchingValidator(Pattern.compile("^[\\x20-\\x7e\\x80-\\xff]+$")))
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .build();
+
+ protected static final PropertyDescriptor WEB_CLIENT_SERVICE = new PropertyDescriptor.Builder()
+ .name("Standard Web Client Service")
+ .description("Web client which is used to communicate with the Workday API.")
+ .required(true)
+ .identifiesControllerService(WebClientServiceProvider.class)
+ .build();
+
+ protected static final PropertyDescriptor FIELDS_TO_HASH = new PropertyDescriptor.Builder()
+ .name("Fields to hash")
+ .displayName("Fields to hash")
+ .description("Comma separated record paths to replace with hash.")
+ .required(false)
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .addValidator(NON_BLANK_VALIDATOR)
+ .build();
+
+ protected static final PropertyDescriptor HASHING_ALGORITHM = new PropertyDescriptor.Builder()
+ .name("Hashing algorithm")
+ .displayName("Hashing algorithm")
+ .description("Determines what hashing algorithm should be used to perform the hashing function.")
+ .required(true)
+ .allowableValues(HashService.buildHashAlgorithmAllowableValues())
+ .defaultValue(HashAlgorithm.SHA256.getName())
+ .addValidator(NON_BLANK_VALIDATOR)
+ .expressionLanguageSupported(NONE)
+ .dependsOn(FIELDS_TO_HASH)
+ .build();
+
+ protected static final PropertyDescriptor RECORD_READER_FACTORY = new PropertyDescriptor.Builder()
+ .name("record-reader")
+ .displayName("Record Reader")
+ .description("Specifies the Controller Service to use for parsing incoming data and determining the data's schema.")
+ .identifiesControllerService(RecordReaderFactory.class)
+ .required(false)
+ .build();
+
+ protected static final PropertyDescriptor RECORD_WRITER_FACTORY = new PropertyDescriptor.Builder()
+ .name("record-writer")
+ .displayName("Record Writer")
+ .description("The Record Writer to use for serializing Records to an output FlowFile.")
+ .identifiesControllerService(RecordSetWriterFactory.class)
+ .dependsOn(RECORD_READER_FACTORY)
+ .required(true)
+ .build();
+
+ protected static final Relationship ORIGINAL = new Relationship.Builder()
+ .name("Original")
+ .description("Request FlowFiles transferred when receiving HTTP responses with a status code between 200 and 299.")
+ .build();
+
+ protected static final Relationship FAILURE = new Relationship.Builder()
+ .name("Failure")
+ .description("Request FlowFiles transferred when receiving socket communication errors.")
+ .build();
+
+ protected static final Relationship RESPONSE = new Relationship.Builder()
+ .name("Response")
+ .description("Response FlowFiles transferred when receiving HTTP responses with a status code between 200 and 299.")
+ .build();
+
+ protected static final Set<Relationship> RELATIONSHIPS = Collections.unmodifiableSet(new HashSet<>(Arrays.asList(ORIGINAL, RESPONSE, FAILURE)));
+ protected static final List<PropertyDescriptor> PROPERTIES = Collections.unmodifiableList(Arrays.asList(REPORT_URL, WORKDAY_USERNAME, WORKDAY_PASSWORD, WEB_CLIENT_SERVICE,
+ FIELDS_TO_HASH, HASHING_ALGORITHM, RECORD_READER_FACTORY, RECORD_WRITER_FACTORY));
+
+ private final AtomicReference<WebClientService> webClientAtomicReference = new AtomicReference<>();
+ private final AtomicReference<RecordReaderFactory> recordReaderFactoryAtomicReference = new AtomicReference<>();
+ private final AtomicReference<RecordSetWriterFactory> recordSetWriterFactoryAtomicReference = new AtomicReference<>();
+
+ @Override
+ protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
+ return PROPERTIES;
+ }
+
+ @Override
+ public Set<Relationship> getRelationships() {
+ return RELATIONSHIPS;
+ }
+
+ @OnScheduled
+ public void setUpClient(final ProcessContext context) {
+ WebClientServiceProvider standardWebClientServiceProvider = context.getProperty(WEB_CLIENT_SERVICE).asControllerService(WebClientServiceProvider.class);
+ RecordReaderFactory recordReaderFactory = context.getProperty(RECORD_READER_FACTORY).asControllerService(RecordReaderFactory.class);
+ RecordSetWriterFactory recordSetWriterFactory = context.getProperty(RECORD_WRITER_FACTORY).asControllerService(RecordSetWriterFactory.class);
+ WebClientService webClientService = standardWebClientServiceProvider.getWebClientService();
+ webClientAtomicReference.set(webClientService);
+ recordReaderFactoryAtomicReference.set(recordReaderFactory);
+ recordSetWriterFactoryAtomicReference.set(recordSetWriterFactory);
+ }
+
+ @Override
+ public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException {
+ FlowFile flowfile = session.get();
+
+ if (skipExecution(context, flowfile)) {
+ return;
+ }
+
+ ComponentLog logger = getLogger();
+ FlowFile responseFlowFile = null;
+
+ try {
+ WebClientService webClientService = webClientAtomicReference.get();
+ URI uri = new URI(context.getProperty(REPORT_URL).evaluateAttributeExpressions(flowfile).getValue().trim());
+ long startNanos = System.nanoTime();
+ String authorization = createAuthorizationHeader(context, flowfile);
+
+ try(HttpResponseEntity httpResponseEntity = webClientService.get().uri(uri).header(HEADER_AUTHORIZATION, authorization).retrieve()) {
+ responseFlowFile = createResponseFlowFile(flowfile, session, context, httpResponseEntity);
+ long elapsedTime = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos);
+ Map<String, String> commonAttributes = createCommonAttributes(uri, httpResponseEntity, elapsedTime);
+
+ if (flowfile != null) {
+ flowfile = session.putAllAttributes(flowfile, decorateWithMimeAttribute(commonAttributes, httpResponseEntity));
+ }
+ if (responseFlowFile != null) {
+ responseFlowFile = session.putAllAttributes(responseFlowFile, commonAttributes);
+ if (flowfile != null) {
+ session.getProvenanceReporter().fetch(responseFlowFile, uri.toString(), elapsedTime);
+ } else {
+ session.getProvenanceReporter().receive(responseFlowFile, uri.toString(), elapsedTime);
+ }
+ }
+
+ route(flowfile, responseFlowFile, session, context, httpResponseEntity.statusCode());
+ }
+ } catch (Exception e) {
+ if (flowfile == null) {
+ logger.error("Request Processing failed", e);
+ context.yield();
+ } else {
+ logger.error("Request Processing failed: {}", flowfile, e);
+ session.penalize(flowfile);
+ flowfile = session.putAttribute(flowfile, GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS, e.getClass().getName());
+ flowfile = session.putAttribute(flowfile, GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE, e.getMessage());
+ session.transfer(flowfile, FAILURE);
+ }
+
+ if (responseFlowFile != null) {
+ session.remove(responseFlowFile);
+ }
+ }
+ }
+
+ @Override
+ protected Collection<ValidationResult> customValidate(ValidationContext validationContext) {
+ List<ValidationResult> results = new ArrayList<>(super.customValidate(validationContext));
+ if (validationContext.getProperty(FIELDS_TO_HASH).isSet() && !validationContext.getProperty(RECORD_READER_FACTORY).isSet()) {
+ results.add(new ValidationResult.Builder()
+ .valid(false)
+ .explanation("Record-Reader and Record-Writer must be set if you would like to hash specific fields")
+ .subject("Workday report configuration")
+ .build());
+ }
+ return results;
+ }
+
+ /*
+ * If we have no FlowFile, and all incoming connections are self-loops then we can continue on.
+ * However, if we have no FlowFile and we have connections coming from other Processors, then
+ * we know that we should run only if we have a FlowFile.
+ */
+ private boolean skipExecution(ProcessContext context, FlowFile flowfile) {
+ return context.hasIncomingConnection() && flowfile == null && context.hasNonLoopConnection();
+ }
+
+ private FlowFile createResponseFlowFile(FlowFile flowfile, ProcessSession session, ProcessContext context, HttpResponseEntity httpResponseEntity)
+ throws IOException, SchemaNotFoundException, MalformedRecordException {
+ FlowFile responseFlowFile = null;
+ String hashingAlgorithm = context.getProperty(HASHING_ALGORITHM).getValue();
+ Set<String> columnsToHash = Optional.ofNullable(context.getProperty(FIELDS_TO_HASH).evaluateAttributeExpressions(flowfile).getValue()).map(String::trim)
+ .map(columns -> columns.split(COLUMNS_TO_HASH_SEPARATOR)).map(Arrays::stream).map(columns -> columns.collect(Collectors.toSet())).orElse(Collections.emptySet());
+ try {
+ if (isSuccess(httpResponseEntity.statusCode())) {
+ responseFlowFile = flowfile != null ? session.create(flowfile) : session.create();
+ InputStream responseBodyStream = httpResponseEntity.body();
+ if (recordReaderFactoryAtomicReference.get() != null) {
+ TransformResult transformResult = transformRecords(session, flowfile, responseFlowFile, hashingAlgorithm, columnsToHash, responseBodyStream);
+ Map<String, String> attributes = new HashMap<>();
+ attributes.put(RECORD_COUNT, String.valueOf(transformResult.getNumberOfRecords()));
+ attributes.put(CoreAttributes.MIME_TYPE.key(), transformResult.getMimeType());
+ responseFlowFile = session.putAllAttributes(responseFlowFile, attributes);
+ } else {
+ responseFlowFile = session.importFrom(responseBodyStream, responseFlowFile);
+ Optional<String> mimeType = httpResponseEntity.headers().getFirstHeader(HEADER_CONTENT_TYPE);
+ if (mimeType.isPresent()) {
+ responseFlowFile = session.putAttribute(responseFlowFile, CoreAttributes.MIME_TYPE.key(), mimeType.get());
+ }
+ }
+ }
+ } catch (Exception e) {
+ session.remove(responseFlowFile);
+ throw e;
+ }
+ return responseFlowFile;
+ }
+
+ private String createAuthorizationHeader(ProcessContext context, FlowFile flowfile) {
+ String userName = context.getProperty(WORKDAY_USERNAME).evaluateAttributeExpressions(flowfile).getValue();
+ String password = context.getProperty(WORKDAY_PASSWORD).evaluateAttributeExpressions(flowfile).getValue();
+ String base64Credential = Base64.getEncoder().encodeToString((userName + USERNAME_PASSWORD_SEPARATOR + password).getBytes(StandardCharsets.ISO_8859_1));
Review Comment:
What do you think about using `StandardCharsets.UTF_8` instead of ISO-8859-1? Standards appear to be different, but using UTF-8 should provide general compatibility.
##########
nifi-nar-bundles/nifi-workday-bundle/nifi-workday-processors/src/main/java/org/apache/nifi/processors/workday/GetWorkdayReport.java:
##########
@@ -0,0 +1,432 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.processors.workday;
+
+import static org.apache.nifi.expression.ExpressionLanguageScope.FLOWFILE_ATTRIBUTES;
+import static org.apache.nifi.expression.ExpressionLanguageScope.NONE;
+import static org.apache.nifi.processor.util.StandardValidators.NON_BLANK_VALIDATOR;
+import static org.apache.nifi.processor.util.StandardValidators.URL_VALIDATOR;
+
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Base64;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import org.apache.nifi.annotation.behavior.EventDriven;
+import org.apache.nifi.annotation.behavior.InputRequirement;
+import org.apache.nifi.annotation.behavior.InputRequirement.Requirement;
+import org.apache.nifi.annotation.behavior.SideEffectFree;
+import org.apache.nifi.annotation.behavior.SupportsBatching;
+import org.apache.nifi.annotation.behavior.WritesAttribute;
+import org.apache.nifi.annotation.behavior.WritesAttributes;
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.annotation.lifecycle.OnScheduled;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.ValidationContext;
+import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.flowfile.attributes.CoreAttributes;
+import org.apache.nifi.logging.ComponentLog;
+import org.apache.nifi.processor.AbstractProcessor;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.processor.util.StandardValidators;
+import org.apache.nifi.record.path.RecordPath;
+import org.apache.nifi.record.path.RecordPathResult;
+import org.apache.nifi.schema.access.SchemaNotFoundException;
+import org.apache.nifi.security.util.crypto.HashAlgorithm;
+import org.apache.nifi.security.util.crypto.HashService;
+import org.apache.nifi.serialization.MalformedRecordException;
+import org.apache.nifi.serialization.RecordReader;
+import org.apache.nifi.serialization.RecordReaderFactory;
+import org.apache.nifi.serialization.RecordSetWriter;
+import org.apache.nifi.serialization.RecordSetWriterFactory;
+import org.apache.nifi.serialization.record.Record;
+import org.apache.nifi.serialization.record.RecordFieldType;
+import org.apache.nifi.serialization.record.RecordSchema;
+import org.apache.nifi.web.client.api.HttpResponseEntity;
+import org.apache.nifi.web.client.api.WebClientService;
+import org.apache.nifi.web.client.provider.api.WebClientServiceProvider;
+
+@Tags({"Workday", "report", "get"})
+@InputRequirement(Requirement.INPUT_ALLOWED)
+@CapabilityDescription("A processor which can interact with a configurable Workday Report. The processor can forward the content without modification, or you can transform it by"
+ + " providing the specific Record Reader and Record Writer services based on your needs. You can also hash, or remove fields using the input parameters and schemes in the Record writer. "
+ + "Supported Workday report formats are: csv, simplexml, json")
+@EventDriven
+@SideEffectFree
+@SupportsBatching
+@WritesAttributes({
+ @WritesAttribute(attribute = GetWorkdayReport.GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS, description = "The Java exception class raised when the processor fails"),
+ @WritesAttribute(attribute = GetWorkdayReport.GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE, description = "The Java exception message raised when the processor fails"),
+ @WritesAttribute(attribute = "mime.type", description = "Sets the mime.type attribute to the MIME Type specified by the Source / Record Writer"),
+ @WritesAttribute(attribute= GetWorkdayReport.RECORD_COUNT, description = "The number of records in an outgoing FlowFile. This is only populated on the 'success' relationship "
+ + "when Record Reader and Writer is set.")})
+public class GetWorkdayReport extends AbstractProcessor {
+
+ protected static final String STATUS_CODE = "getworkdayreport.status.code";
+ protected static final String REQUEST_URL = "getworkdayreport.request.url";
+ protected static final String REQUEST_DURATION = "getworkdayreport.request.duration";
+ protected static final String TRANSACTION_ID = "getworkdayreport.tx.id";
+ protected static final String GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS = "getworkdayreport.java.exception.class";
+ protected static final String GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE = "getworkdayreport.java.exception.message";
+ protected static final String RECORD_COUNT = "record.count";
+ protected static final String BASIC_PREFIX = "Basic ";
+ protected static final String COLUMNS_TO_HASH_SEPARATOR = ",";
+ protected static final String HEADER_AUTHORIZATION = "Authorization";
+ protected static final String HEADER_CONTENT_TYPE = "Content-Type";
+ protected static final String USERNAME_PASSWORD_SEPARATOR = ":";
+
+ protected static final PropertyDescriptor REPORT_URL = new PropertyDescriptor.Builder()
+ .name("Workday report URL")
+ .displayName("Workday report URL")
+ .description("HTTP remote URL of Workday report including a scheme of http or https, as well as a hostname or IP address with optional port and path elements.")
+ .required(true)
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .addValidator(URL_VALIDATOR)
+ .build();
+
+ protected static final PropertyDescriptor WORKDAY_USERNAME = new PropertyDescriptor.Builder()
+ .name("Workday Username")
+ .displayName("Workday Username")
+ .description("The username provided for authentication of Workday requests. Encoded using Base64 for HTTP Basic Authentication as described in RFC 7617.")
+ .required(true)
+ .addValidator(StandardValidators.createRegexMatchingValidator(Pattern.compile("^[\\x20-\\x39\\x3b-\\x7e\\x80-\\xff]+$")))
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .build();
+
+ protected static final PropertyDescriptor WORKDAY_PASSWORD = new PropertyDescriptor.Builder()
+ .name("Workday Password")
+ .displayName("Workday Password")
+ .description("The password provided for authentication of Workday requests. Encoded using Base64 for HTTP Basic Authentication as described in RFC 7617.")
+ .required(true)
+ .sensitive(true)
+ .addValidator(StandardValidators.createRegexMatchingValidator(Pattern.compile("^[\\x20-\\x7e\\x80-\\xff]+$")))
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .build();
+
+ protected static final PropertyDescriptor WEB_CLIENT_SERVICE = new PropertyDescriptor.Builder()
+ .name("Standard Web Client Service")
+ .description("Web client which is used to communicate with the Workday API.")
+ .required(true)
+ .identifiesControllerService(WebClientServiceProvider.class)
+ .build();
+
+ protected static final PropertyDescriptor FIELDS_TO_HASH = new PropertyDescriptor.Builder()
+ .name("Fields to hash")
+ .displayName("Fields to hash")
+ .description("Comma separated record paths to replace with hash.")
+ .required(false)
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .addValidator(NON_BLANK_VALIDATOR)
+ .build();
+
+ protected static final PropertyDescriptor HASHING_ALGORITHM = new PropertyDescriptor.Builder()
+ .name("Hashing algorithm")
+ .displayName("Hashing algorithm")
+ .description("Determines what hashing algorithm should be used to perform the hashing function.")
+ .required(true)
+ .allowableValues(HashService.buildHashAlgorithmAllowableValues())
+ .defaultValue(HashAlgorithm.SHA256.getName())
+ .addValidator(NON_BLANK_VALIDATOR)
+ .expressionLanguageSupported(NONE)
+ .dependsOn(FIELDS_TO_HASH)
+ .build();
+
+ protected static final PropertyDescriptor RECORD_READER_FACTORY = new PropertyDescriptor.Builder()
+ .name("record-reader")
+ .displayName("Record Reader")
+ .description("Specifies the Controller Service to use for parsing incoming data and determining the data's schema.")
+ .identifiesControllerService(RecordReaderFactory.class)
+ .required(false)
+ .build();
+
+ protected static final PropertyDescriptor RECORD_WRITER_FACTORY = new PropertyDescriptor.Builder()
+ .name("record-writer")
+ .displayName("Record Writer")
+ .description("The Record Writer to use for serializing Records to an output FlowFile.")
+ .identifiesControllerService(RecordSetWriterFactory.class)
+ .dependsOn(RECORD_READER_FACTORY)
+ .required(true)
+ .build();
+
+ protected static final Relationship ORIGINAL = new Relationship.Builder()
+ .name("Original")
+ .description("Request FlowFiles transferred when receiving HTTP responses with a status code between 200 and 299.")
+ .build();
+
+ protected static final Relationship FAILURE = new Relationship.Builder()
+ .name("Failure")
+ .description("Request FlowFiles transferred when receiving socket communication errors.")
+ .build();
+
+ protected static final Relationship RESPONSE = new Relationship.Builder()
+ .name("Response")
+ .description("Response FlowFiles transferred when receiving HTTP responses with a status code between 200 and 299.")
+ .build();
+
+ protected static final Set<Relationship> RELATIONSHIPS = Collections.unmodifiableSet(new HashSet<>(Arrays.asList(ORIGINAL, RESPONSE, FAILURE)));
+ protected static final List<PropertyDescriptor> PROPERTIES = Collections.unmodifiableList(Arrays.asList(REPORT_URL, WORKDAY_USERNAME, WORKDAY_PASSWORD, WEB_CLIENT_SERVICE,
+ FIELDS_TO_HASH, HASHING_ALGORITHM, RECORD_READER_FACTORY, RECORD_WRITER_FACTORY));
+
+ private final AtomicReference<WebClientService> webClientAtomicReference = new AtomicReference<>();
+ private final AtomicReference<RecordReaderFactory> recordReaderFactoryAtomicReference = new AtomicReference<>();
+ private final AtomicReference<RecordSetWriterFactory> recordSetWriterFactoryAtomicReference = new AtomicReference<>();
+
+ @Override
+ protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
+ return PROPERTIES;
+ }
+
+ @Override
+ public Set<Relationship> getRelationships() {
+ return RELATIONSHIPS;
+ }
+
+ @OnScheduled
+ public void setUpClient(final ProcessContext context) {
+ WebClientServiceProvider standardWebClientServiceProvider = context.getProperty(WEB_CLIENT_SERVICE).asControllerService(WebClientServiceProvider.class);
+ RecordReaderFactory recordReaderFactory = context.getProperty(RECORD_READER_FACTORY).asControllerService(RecordReaderFactory.class);
+ RecordSetWriterFactory recordSetWriterFactory = context.getProperty(RECORD_WRITER_FACTORY).asControllerService(RecordSetWriterFactory.class);
+ WebClientService webClientService = standardWebClientServiceProvider.getWebClientService();
+ webClientAtomicReference.set(webClientService);
+ recordReaderFactoryAtomicReference.set(recordReaderFactory);
+ recordSetWriterFactoryAtomicReference.set(recordSetWriterFactory);
+ }
+
+ @Override
+ public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException {
+ FlowFile flowfile = session.get();
+
+ if (skipExecution(context, flowfile)) {
+ return;
+ }
+
+ ComponentLog logger = getLogger();
+ FlowFile responseFlowFile = null;
+
+ try {
+ WebClientService webClientService = webClientAtomicReference.get();
+ URI uri = new URI(context.getProperty(REPORT_URL).evaluateAttributeExpressions(flowfile).getValue().trim());
+ long startNanos = System.nanoTime();
+ String authorization = createAuthorizationHeader(context, flowfile);
+
+ try(HttpResponseEntity httpResponseEntity = webClientService.get().uri(uri).header(HEADER_AUTHORIZATION, authorization).retrieve()) {
+ responseFlowFile = createResponseFlowFile(flowfile, session, context, httpResponseEntity);
+ long elapsedTime = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos);
+ Map<String, String> commonAttributes = createCommonAttributes(uri, httpResponseEntity, elapsedTime);
+
+ if (flowfile != null) {
+ flowfile = session.putAllAttributes(flowfile, decorateWithMimeAttribute(commonAttributes, httpResponseEntity));
+ }
+ if (responseFlowFile != null) {
+ responseFlowFile = session.putAllAttributes(responseFlowFile, commonAttributes);
+ if (flowfile != null) {
+ session.getProvenanceReporter().fetch(responseFlowFile, uri.toString(), elapsedTime);
+ } else {
+ session.getProvenanceReporter().receive(responseFlowFile, uri.toString(), elapsedTime);
+ }
+ }
+
+ route(flowfile, responseFlowFile, session, context, httpResponseEntity.statusCode());
+ }
+ } catch (Exception e) {
+ if (flowfile == null) {
+ logger.error("Request Processing failed", e);
+ context.yield();
+ } else {
+ logger.error("Request Processing failed: {}", flowfile, e);
+ session.penalize(flowfile);
+ flowfile = session.putAttribute(flowfile, GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS, e.getClass().getName());
+ flowfile = session.putAttribute(flowfile, GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE, e.getMessage());
+ session.transfer(flowfile, FAILURE);
+ }
+
+ if (responseFlowFile != null) {
+ session.remove(responseFlowFile);
+ }
+ }
+ }
+
+ @Override
+ protected Collection<ValidationResult> customValidate(ValidationContext validationContext) {
+ List<ValidationResult> results = new ArrayList<>(super.customValidate(validationContext));
+ if (validationContext.getProperty(FIELDS_TO_HASH).isSet() && !validationContext.getProperty(RECORD_READER_FACTORY).isSet()) {
+ results.add(new ValidationResult.Builder()
+ .valid(false)
+ .explanation("Record-Reader and Record-Writer must be set if you would like to hash specific fields")
+ .subject("Workday report configuration")
+ .build());
+ }
+ return results;
+ }
+
+ /*
+ * If we have no FlowFile, and all incoming connections are self-loops then we can continue on.
+ * However, if we have no FlowFile and we have connections coming from other Processors, then
+ * we know that we should run only if we have a FlowFile.
+ */
+ private boolean skipExecution(ProcessContext context, FlowFile flowfile) {
+ return context.hasIncomingConnection() && flowfile == null && context.hasNonLoopConnection();
+ }
+
+ private FlowFile createResponseFlowFile(FlowFile flowfile, ProcessSession session, ProcessContext context, HttpResponseEntity httpResponseEntity)
+ throws IOException, SchemaNotFoundException, MalformedRecordException {
+ FlowFile responseFlowFile = null;
+ String hashingAlgorithm = context.getProperty(HASHING_ALGORITHM).getValue();
+ Set<String> columnsToHash = Optional.ofNullable(context.getProperty(FIELDS_TO_HASH).evaluateAttributeExpressions(flowfile).getValue()).map(String::trim)
+ .map(columns -> columns.split(COLUMNS_TO_HASH_SEPARATOR)).map(Arrays::stream).map(columns -> columns.collect(Collectors.toSet())).orElse(Collections.emptySet());
Review Comment:
Recommend breaking these calls into separate lines per method to make it easier to read.
##########
nifi-nar-bundles/nifi-workday-bundle/nifi-workday-processors/src/main/java/org/apache/nifi/processors/workday/GetWorkdayReport.java:
##########
@@ -0,0 +1,432 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.processors.workday;
+
+import static org.apache.nifi.expression.ExpressionLanguageScope.FLOWFILE_ATTRIBUTES;
+import static org.apache.nifi.expression.ExpressionLanguageScope.NONE;
+import static org.apache.nifi.processor.util.StandardValidators.NON_BLANK_VALIDATOR;
+import static org.apache.nifi.processor.util.StandardValidators.URL_VALIDATOR;
+
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Base64;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import org.apache.nifi.annotation.behavior.EventDriven;
+import org.apache.nifi.annotation.behavior.InputRequirement;
+import org.apache.nifi.annotation.behavior.InputRequirement.Requirement;
+import org.apache.nifi.annotation.behavior.SideEffectFree;
+import org.apache.nifi.annotation.behavior.SupportsBatching;
+import org.apache.nifi.annotation.behavior.WritesAttribute;
+import org.apache.nifi.annotation.behavior.WritesAttributes;
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.annotation.lifecycle.OnScheduled;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.ValidationContext;
+import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.flowfile.attributes.CoreAttributes;
+import org.apache.nifi.logging.ComponentLog;
+import org.apache.nifi.processor.AbstractProcessor;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.processor.util.StandardValidators;
+import org.apache.nifi.record.path.RecordPath;
+import org.apache.nifi.record.path.RecordPathResult;
+import org.apache.nifi.schema.access.SchemaNotFoundException;
+import org.apache.nifi.security.util.crypto.HashAlgorithm;
+import org.apache.nifi.security.util.crypto.HashService;
+import org.apache.nifi.serialization.MalformedRecordException;
+import org.apache.nifi.serialization.RecordReader;
+import org.apache.nifi.serialization.RecordReaderFactory;
+import org.apache.nifi.serialization.RecordSetWriter;
+import org.apache.nifi.serialization.RecordSetWriterFactory;
+import org.apache.nifi.serialization.record.Record;
+import org.apache.nifi.serialization.record.RecordFieldType;
+import org.apache.nifi.serialization.record.RecordSchema;
+import org.apache.nifi.web.client.api.HttpResponseEntity;
+import org.apache.nifi.web.client.api.WebClientService;
+import org.apache.nifi.web.client.provider.api.WebClientServiceProvider;
+
+@Tags({"Workday", "report", "get"})
+@InputRequirement(Requirement.INPUT_ALLOWED)
+@CapabilityDescription("A processor which can interact with a configurable Workday Report. The processor can forward the content without modification, or you can transform it by"
+ + " providing the specific Record Reader and Record Writer services based on your needs. You can also hash, or remove fields using the input parameters and schemes in the Record writer. "
+ + "Supported Workday report formats are: csv, simplexml, json")
+@EventDriven
+@SideEffectFree
+@SupportsBatching
+@WritesAttributes({
+ @WritesAttribute(attribute = GetWorkdayReport.GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS, description = "The Java exception class raised when the processor fails"),
+ @WritesAttribute(attribute = GetWorkdayReport.GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE, description = "The Java exception message raised when the processor fails"),
+ @WritesAttribute(attribute = "mime.type", description = "Sets the mime.type attribute to the MIME Type specified by the Source / Record Writer"),
+ @WritesAttribute(attribute= GetWorkdayReport.RECORD_COUNT, description = "The number of records in an outgoing FlowFile. This is only populated on the 'success' relationship "
+ + "when Record Reader and Writer is set.")})
+public class GetWorkdayReport extends AbstractProcessor {
+
+ protected static final String STATUS_CODE = "getworkdayreport.status.code";
+ protected static final String REQUEST_URL = "getworkdayreport.request.url";
+ protected static final String REQUEST_DURATION = "getworkdayreport.request.duration";
+ protected static final String TRANSACTION_ID = "getworkdayreport.tx.id";
+ protected static final String GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS = "getworkdayreport.java.exception.class";
+ protected static final String GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE = "getworkdayreport.java.exception.message";
+ protected static final String RECORD_COUNT = "record.count";
+ protected static final String BASIC_PREFIX = "Basic ";
+ protected static final String COLUMNS_TO_HASH_SEPARATOR = ",";
+ protected static final String HEADER_AUTHORIZATION = "Authorization";
+ protected static final String HEADER_CONTENT_TYPE = "Content-Type";
+ protected static final String USERNAME_PASSWORD_SEPARATOR = ":";
+
+ protected static final PropertyDescriptor REPORT_URL = new PropertyDescriptor.Builder()
+ .name("Workday report URL")
+ .displayName("Workday report URL")
+ .description("HTTP remote URL of Workday report including a scheme of http or https, as well as a hostname or IP address with optional port and path elements.")
+ .required(true)
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .addValidator(URL_VALIDATOR)
+ .build();
+
+ protected static final PropertyDescriptor WORKDAY_USERNAME = new PropertyDescriptor.Builder()
+ .name("Workday Username")
+ .displayName("Workday Username")
+ .description("The username provided for authentication of Workday requests. Encoded using Base64 for HTTP Basic Authentication as described in RFC 7617.")
+ .required(true)
+ .addValidator(StandardValidators.createRegexMatchingValidator(Pattern.compile("^[\\x20-\\x39\\x3b-\\x7e\\x80-\\xff]+$")))
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .build();
+
+ protected static final PropertyDescriptor WORKDAY_PASSWORD = new PropertyDescriptor.Builder()
+ .name("Workday Password")
+ .displayName("Workday Password")
+ .description("The password provided for authentication of Workday requests. Encoded using Base64 for HTTP Basic Authentication as described in RFC 7617.")
+ .required(true)
+ .sensitive(true)
+ .addValidator(StandardValidators.createRegexMatchingValidator(Pattern.compile("^[\\x20-\\x7e\\x80-\\xff]+$")))
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .build();
+
+ protected static final PropertyDescriptor WEB_CLIENT_SERVICE = new PropertyDescriptor.Builder()
+ .name("Standard Web Client Service")
+ .description("Web client which is used to communicate with the Workday API.")
+ .required(true)
+ .identifiesControllerService(WebClientServiceProvider.class)
+ .build();
+
+ protected static final PropertyDescriptor FIELDS_TO_HASH = new PropertyDescriptor.Builder()
+ .name("Fields to hash")
+ .displayName("Fields to hash")
+ .description("Comma separated record paths to replace with hash.")
+ .required(false)
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .addValidator(NON_BLANK_VALIDATOR)
+ .build();
+
+ protected static final PropertyDescriptor HASHING_ALGORITHM = new PropertyDescriptor.Builder()
+ .name("Hashing algorithm")
+ .displayName("Hashing algorithm")
+ .description("Determines what hashing algorithm should be used to perform the hashing function.")
+ .required(true)
+ .allowableValues(HashService.buildHashAlgorithmAllowableValues())
Review Comment:
Although this method is convenient, in general it should not be necessary to support such a wide ranger of algorithms. Recommend limiting this to SHA-256 and SHA-512.
##########
nifi-nar-bundles/nifi-workday-bundle/nifi-workday-processors/src/main/java/org/apache/nifi/processors/workday/GetWorkdayReport.java:
##########
@@ -0,0 +1,432 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.processors.workday;
+
+import static org.apache.nifi.expression.ExpressionLanguageScope.FLOWFILE_ATTRIBUTES;
+import static org.apache.nifi.expression.ExpressionLanguageScope.NONE;
+import static org.apache.nifi.processor.util.StandardValidators.NON_BLANK_VALIDATOR;
+import static org.apache.nifi.processor.util.StandardValidators.URL_VALIDATOR;
+
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Base64;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import org.apache.nifi.annotation.behavior.EventDriven;
+import org.apache.nifi.annotation.behavior.InputRequirement;
+import org.apache.nifi.annotation.behavior.InputRequirement.Requirement;
+import org.apache.nifi.annotation.behavior.SideEffectFree;
+import org.apache.nifi.annotation.behavior.SupportsBatching;
+import org.apache.nifi.annotation.behavior.WritesAttribute;
+import org.apache.nifi.annotation.behavior.WritesAttributes;
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.annotation.lifecycle.OnScheduled;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.ValidationContext;
+import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.flowfile.attributes.CoreAttributes;
+import org.apache.nifi.logging.ComponentLog;
+import org.apache.nifi.processor.AbstractProcessor;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.processor.util.StandardValidators;
+import org.apache.nifi.record.path.RecordPath;
+import org.apache.nifi.record.path.RecordPathResult;
+import org.apache.nifi.schema.access.SchemaNotFoundException;
+import org.apache.nifi.security.util.crypto.HashAlgorithm;
+import org.apache.nifi.security.util.crypto.HashService;
+import org.apache.nifi.serialization.MalformedRecordException;
+import org.apache.nifi.serialization.RecordReader;
+import org.apache.nifi.serialization.RecordReaderFactory;
+import org.apache.nifi.serialization.RecordSetWriter;
+import org.apache.nifi.serialization.RecordSetWriterFactory;
+import org.apache.nifi.serialization.record.Record;
+import org.apache.nifi.serialization.record.RecordFieldType;
+import org.apache.nifi.serialization.record.RecordSchema;
+import org.apache.nifi.web.client.api.HttpResponseEntity;
+import org.apache.nifi.web.client.api.WebClientService;
+import org.apache.nifi.web.client.provider.api.WebClientServiceProvider;
+
+@Tags({"Workday", "report", "get"})
+@InputRequirement(Requirement.INPUT_ALLOWED)
+@CapabilityDescription("A processor which can interact with a configurable Workday Report. The processor can forward the content without modification, or you can transform it by"
+ + " providing the specific Record Reader and Record Writer services based on your needs. You can also hash, or remove fields using the input parameters and schemes in the Record writer. "
+ + "Supported Workday report formats are: csv, simplexml, json")
+@EventDriven
+@SideEffectFree
+@SupportsBatching
+@WritesAttributes({
+ @WritesAttribute(attribute = GetWorkdayReport.GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS, description = "The Java exception class raised when the processor fails"),
+ @WritesAttribute(attribute = GetWorkdayReport.GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE, description = "The Java exception message raised when the processor fails"),
+ @WritesAttribute(attribute = "mime.type", description = "Sets the mime.type attribute to the MIME Type specified by the Source / Record Writer"),
+ @WritesAttribute(attribute= GetWorkdayReport.RECORD_COUNT, description = "The number of records in an outgoing FlowFile. This is only populated on the 'success' relationship "
+ + "when Record Reader and Writer is set.")})
+public class GetWorkdayReport extends AbstractProcessor {
+
+ protected static final String STATUS_CODE = "getworkdayreport.status.code";
+ protected static final String REQUEST_URL = "getworkdayreport.request.url";
+ protected static final String REQUEST_DURATION = "getworkdayreport.request.duration";
+ protected static final String TRANSACTION_ID = "getworkdayreport.tx.id";
+ protected static final String GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS = "getworkdayreport.java.exception.class";
+ protected static final String GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE = "getworkdayreport.java.exception.message";
+ protected static final String RECORD_COUNT = "record.count";
+ protected static final String BASIC_PREFIX = "Basic ";
+ protected static final String COLUMNS_TO_HASH_SEPARATOR = ",";
+ protected static final String HEADER_AUTHORIZATION = "Authorization";
+ protected static final String HEADER_CONTENT_TYPE = "Content-Type";
+ protected static final String USERNAME_PASSWORD_SEPARATOR = ":";
+
+ protected static final PropertyDescriptor REPORT_URL = new PropertyDescriptor.Builder()
+ .name("Workday report URL")
+ .displayName("Workday report URL")
+ .description("HTTP remote URL of Workday report including a scheme of http or https, as well as a hostname or IP address with optional port and path elements.")
+ .required(true)
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .addValidator(URL_VALIDATOR)
+ .build();
+
+ protected static final PropertyDescriptor WORKDAY_USERNAME = new PropertyDescriptor.Builder()
+ .name("Workday Username")
+ .displayName("Workday Username")
+ .description("The username provided for authentication of Workday requests. Encoded using Base64 for HTTP Basic Authentication as described in RFC 7617.")
+ .required(true)
+ .addValidator(StandardValidators.createRegexMatchingValidator(Pattern.compile("^[\\x20-\\x39\\x3b-\\x7e\\x80-\\xff]+$")))
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .build();
+
+ protected static final PropertyDescriptor WORKDAY_PASSWORD = new PropertyDescriptor.Builder()
+ .name("Workday Password")
+ .displayName("Workday Password")
+ .description("The password provided for authentication of Workday requests. Encoded using Base64 for HTTP Basic Authentication as described in RFC 7617.")
+ .required(true)
+ .sensitive(true)
+ .addValidator(StandardValidators.createRegexMatchingValidator(Pattern.compile("^[\\x20-\\x7e\\x80-\\xff]+$")))
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .build();
+
+ protected static final PropertyDescriptor WEB_CLIENT_SERVICE = new PropertyDescriptor.Builder()
+ .name("Standard Web Client Service")
+ .description("Web client which is used to communicate with the Workday API.")
+ .required(true)
+ .identifiesControllerService(WebClientServiceProvider.class)
+ .build();
+
+ protected static final PropertyDescriptor FIELDS_TO_HASH = new PropertyDescriptor.Builder()
+ .name("Fields to hash")
+ .displayName("Fields to hash")
+ .description("Comma separated record paths to replace with hash.")
+ .required(false)
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .addValidator(NON_BLANK_VALIDATOR)
+ .build();
+
+ protected static final PropertyDescriptor HASHING_ALGORITHM = new PropertyDescriptor.Builder()
+ .name("Hashing algorithm")
+ .displayName("Hashing algorithm")
+ .description("Determines what hashing algorithm should be used to perform the hashing function.")
+ .required(true)
+ .allowableValues(HashService.buildHashAlgorithmAllowableValues())
+ .defaultValue(HashAlgorithm.SHA256.getName())
+ .addValidator(NON_BLANK_VALIDATOR)
+ .expressionLanguageSupported(NONE)
+ .dependsOn(FIELDS_TO_HASH)
+ .build();
+
+ protected static final PropertyDescriptor RECORD_READER_FACTORY = new PropertyDescriptor.Builder()
+ .name("record-reader")
+ .displayName("Record Reader")
+ .description("Specifies the Controller Service to use for parsing incoming data and determining the data's schema.")
+ .identifiesControllerService(RecordReaderFactory.class)
+ .required(false)
+ .build();
+
+ protected static final PropertyDescriptor RECORD_WRITER_FACTORY = new PropertyDescriptor.Builder()
+ .name("record-writer")
+ .displayName("Record Writer")
+ .description("The Record Writer to use for serializing Records to an output FlowFile.")
+ .identifiesControllerService(RecordSetWriterFactory.class)
+ .dependsOn(RECORD_READER_FACTORY)
+ .required(true)
+ .build();
+
+ protected static final Relationship ORIGINAL = new Relationship.Builder()
+ .name("Original")
+ .description("Request FlowFiles transferred when receiving HTTP responses with a status code between 200 and 299.")
+ .build();
+
+ protected static final Relationship FAILURE = new Relationship.Builder()
+ .name("Failure")
+ .description("Request FlowFiles transferred when receiving socket communication errors.")
+ .build();
+
+ protected static final Relationship RESPONSE = new Relationship.Builder()
+ .name("Response")
+ .description("Response FlowFiles transferred when receiving HTTP responses with a status code between 200 and 299.")
+ .build();
+
+ protected static final Set<Relationship> RELATIONSHIPS = Collections.unmodifiableSet(new HashSet<>(Arrays.asList(ORIGINAL, RESPONSE, FAILURE)));
+ protected static final List<PropertyDescriptor> PROPERTIES = Collections.unmodifiableList(Arrays.asList(REPORT_URL, WORKDAY_USERNAME, WORKDAY_PASSWORD, WEB_CLIENT_SERVICE,
+ FIELDS_TO_HASH, HASHING_ALGORITHM, RECORD_READER_FACTORY, RECORD_WRITER_FACTORY));
+
+ private final AtomicReference<WebClientService> webClientAtomicReference = new AtomicReference<>();
+ private final AtomicReference<RecordReaderFactory> recordReaderFactoryAtomicReference = new AtomicReference<>();
+ private final AtomicReference<RecordSetWriterFactory> recordSetWriterFactoryAtomicReference = new AtomicReference<>();
+
+ @Override
+ protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
+ return PROPERTIES;
+ }
+
+ @Override
+ public Set<Relationship> getRelationships() {
+ return RELATIONSHIPS;
+ }
+
+ @OnScheduled
+ public void setUpClient(final ProcessContext context) {
+ WebClientServiceProvider standardWebClientServiceProvider = context.getProperty(WEB_CLIENT_SERVICE).asControllerService(WebClientServiceProvider.class);
+ RecordReaderFactory recordReaderFactory = context.getProperty(RECORD_READER_FACTORY).asControllerService(RecordReaderFactory.class);
+ RecordSetWriterFactory recordSetWriterFactory = context.getProperty(RECORD_WRITER_FACTORY).asControllerService(RecordSetWriterFactory.class);
+ WebClientService webClientService = standardWebClientServiceProvider.getWebClientService();
+ webClientAtomicReference.set(webClientService);
+ recordReaderFactoryAtomicReference.set(recordReaderFactory);
+ recordSetWriterFactoryAtomicReference.set(recordSetWriterFactory);
+ }
+
+ @Override
+ public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException {
+ FlowFile flowfile = session.get();
Review Comment:
```suggestion
FlowFile flowFile = session.get();
```
##########
nifi-nar-bundles/nifi-workday-bundle/nifi-workday-processors/src/main/java/org/apache/nifi/processors/workday/GetWorkdayReport.java:
##########
@@ -0,0 +1,432 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.processors.workday;
+
+import static org.apache.nifi.expression.ExpressionLanguageScope.FLOWFILE_ATTRIBUTES;
+import static org.apache.nifi.expression.ExpressionLanguageScope.NONE;
+import static org.apache.nifi.processor.util.StandardValidators.NON_BLANK_VALIDATOR;
+import static org.apache.nifi.processor.util.StandardValidators.URL_VALIDATOR;
+
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Base64;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import org.apache.nifi.annotation.behavior.EventDriven;
+import org.apache.nifi.annotation.behavior.InputRequirement;
+import org.apache.nifi.annotation.behavior.InputRequirement.Requirement;
+import org.apache.nifi.annotation.behavior.SideEffectFree;
+import org.apache.nifi.annotation.behavior.SupportsBatching;
+import org.apache.nifi.annotation.behavior.WritesAttribute;
+import org.apache.nifi.annotation.behavior.WritesAttributes;
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.annotation.lifecycle.OnScheduled;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.ValidationContext;
+import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.flowfile.attributes.CoreAttributes;
+import org.apache.nifi.logging.ComponentLog;
+import org.apache.nifi.processor.AbstractProcessor;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.processor.util.StandardValidators;
+import org.apache.nifi.record.path.RecordPath;
+import org.apache.nifi.record.path.RecordPathResult;
+import org.apache.nifi.schema.access.SchemaNotFoundException;
+import org.apache.nifi.security.util.crypto.HashAlgorithm;
+import org.apache.nifi.security.util.crypto.HashService;
+import org.apache.nifi.serialization.MalformedRecordException;
+import org.apache.nifi.serialization.RecordReader;
+import org.apache.nifi.serialization.RecordReaderFactory;
+import org.apache.nifi.serialization.RecordSetWriter;
+import org.apache.nifi.serialization.RecordSetWriterFactory;
+import org.apache.nifi.serialization.record.Record;
+import org.apache.nifi.serialization.record.RecordFieldType;
+import org.apache.nifi.serialization.record.RecordSchema;
+import org.apache.nifi.web.client.api.HttpResponseEntity;
+import org.apache.nifi.web.client.api.WebClientService;
+import org.apache.nifi.web.client.provider.api.WebClientServiceProvider;
+
+@Tags({"Workday", "report", "get"})
+@InputRequirement(Requirement.INPUT_ALLOWED)
+@CapabilityDescription("A processor which can interact with a configurable Workday Report. The processor can forward the content without modification, or you can transform it by"
+ + " providing the specific Record Reader and Record Writer services based on your needs. You can also hash, or remove fields using the input parameters and schemes in the Record writer. "
+ + "Supported Workday report formats are: csv, simplexml, json")
+@EventDriven
+@SideEffectFree
+@SupportsBatching
+@WritesAttributes({
+ @WritesAttribute(attribute = GetWorkdayReport.GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS, description = "The Java exception class raised when the processor fails"),
+ @WritesAttribute(attribute = GetWorkdayReport.GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE, description = "The Java exception message raised when the processor fails"),
+ @WritesAttribute(attribute = "mime.type", description = "Sets the mime.type attribute to the MIME Type specified by the Source / Record Writer"),
+ @WritesAttribute(attribute= GetWorkdayReport.RECORD_COUNT, description = "The number of records in an outgoing FlowFile. This is only populated on the 'success' relationship "
+ + "when Record Reader and Writer is set.")})
+public class GetWorkdayReport extends AbstractProcessor {
+
+ protected static final String STATUS_CODE = "getworkdayreport.status.code";
+ protected static final String REQUEST_URL = "getworkdayreport.request.url";
+ protected static final String REQUEST_DURATION = "getworkdayreport.request.duration";
+ protected static final String TRANSACTION_ID = "getworkdayreport.tx.id";
+ protected static final String GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS = "getworkdayreport.java.exception.class";
+ protected static final String GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE = "getworkdayreport.java.exception.message";
+ protected static final String RECORD_COUNT = "record.count";
+ protected static final String BASIC_PREFIX = "Basic ";
+ protected static final String COLUMNS_TO_HASH_SEPARATOR = ",";
+ protected static final String HEADER_AUTHORIZATION = "Authorization";
+ protected static final String HEADER_CONTENT_TYPE = "Content-Type";
+ protected static final String USERNAME_PASSWORD_SEPARATOR = ":";
+
+ protected static final PropertyDescriptor REPORT_URL = new PropertyDescriptor.Builder()
+ .name("Workday report URL")
+ .displayName("Workday report URL")
+ .description("HTTP remote URL of Workday report including a scheme of http or https, as well as a hostname or IP address with optional port and path elements.")
+ .required(true)
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .addValidator(URL_VALIDATOR)
+ .build();
+
+ protected static final PropertyDescriptor WORKDAY_USERNAME = new PropertyDescriptor.Builder()
+ .name("Workday Username")
+ .displayName("Workday Username")
+ .description("The username provided for authentication of Workday requests. Encoded using Base64 for HTTP Basic Authentication as described in RFC 7617.")
+ .required(true)
+ .addValidator(StandardValidators.createRegexMatchingValidator(Pattern.compile("^[\\x20-\\x39\\x3b-\\x7e\\x80-\\xff]+$")))
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .build();
+
+ protected static final PropertyDescriptor WORKDAY_PASSWORD = new PropertyDescriptor.Builder()
+ .name("Workday Password")
+ .displayName("Workday Password")
+ .description("The password provided for authentication of Workday requests. Encoded using Base64 for HTTP Basic Authentication as described in RFC 7617.")
+ .required(true)
+ .sensitive(true)
+ .addValidator(StandardValidators.createRegexMatchingValidator(Pattern.compile("^[\\x20-\\x7e\\x80-\\xff]+$")))
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .build();
+
+ protected static final PropertyDescriptor WEB_CLIENT_SERVICE = new PropertyDescriptor.Builder()
+ .name("Standard Web Client Service")
+ .description("Web client which is used to communicate with the Workday API.")
+ .required(true)
+ .identifiesControllerService(WebClientServiceProvider.class)
+ .build();
+
+ protected static final PropertyDescriptor FIELDS_TO_HASH = new PropertyDescriptor.Builder()
+ .name("Fields to hash")
+ .displayName("Fields to hash")
+ .description("Comma separated record paths to replace with hash.")
+ .required(false)
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .addValidator(NON_BLANK_VALIDATOR)
+ .build();
+
+ protected static final PropertyDescriptor HASHING_ALGORITHM = new PropertyDescriptor.Builder()
+ .name("Hashing algorithm")
+ .displayName("Hashing algorithm")
+ .description("Determines what hashing algorithm should be used to perform the hashing function.")
+ .required(true)
+ .allowableValues(HashService.buildHashAlgorithmAllowableValues())
+ .defaultValue(HashAlgorithm.SHA256.getName())
+ .addValidator(NON_BLANK_VALIDATOR)
+ .expressionLanguageSupported(NONE)
+ .dependsOn(FIELDS_TO_HASH)
+ .build();
+
+ protected static final PropertyDescriptor RECORD_READER_FACTORY = new PropertyDescriptor.Builder()
+ .name("record-reader")
+ .displayName("Record Reader")
+ .description("Specifies the Controller Service to use for parsing incoming data and determining the data's schema.")
+ .identifiesControllerService(RecordReaderFactory.class)
+ .required(false)
+ .build();
+
+ protected static final PropertyDescriptor RECORD_WRITER_FACTORY = new PropertyDescriptor.Builder()
+ .name("record-writer")
+ .displayName("Record Writer")
+ .description("The Record Writer to use for serializing Records to an output FlowFile.")
+ .identifiesControllerService(RecordSetWriterFactory.class)
+ .dependsOn(RECORD_READER_FACTORY)
+ .required(true)
+ .build();
+
+ protected static final Relationship ORIGINAL = new Relationship.Builder()
+ .name("Original")
+ .description("Request FlowFiles transferred when receiving HTTP responses with a status code between 200 and 299.")
+ .build();
+
+ protected static final Relationship FAILURE = new Relationship.Builder()
+ .name("Failure")
+ .description("Request FlowFiles transferred when receiving socket communication errors.")
+ .build();
+
+ protected static final Relationship RESPONSE = new Relationship.Builder()
+ .name("Response")
+ .description("Response FlowFiles transferred when receiving HTTP responses with a status code between 200 and 299.")
+ .build();
+
+ protected static final Set<Relationship> RELATIONSHIPS = Collections.unmodifiableSet(new HashSet<>(Arrays.asList(ORIGINAL, RESPONSE, FAILURE)));
+ protected static final List<PropertyDescriptor> PROPERTIES = Collections.unmodifiableList(Arrays.asList(REPORT_URL, WORKDAY_USERNAME, WORKDAY_PASSWORD, WEB_CLIENT_SERVICE,
+ FIELDS_TO_HASH, HASHING_ALGORITHM, RECORD_READER_FACTORY, RECORD_WRITER_FACTORY));
+
+ private final AtomicReference<WebClientService> webClientAtomicReference = new AtomicReference<>();
+ private final AtomicReference<RecordReaderFactory> recordReaderFactoryAtomicReference = new AtomicReference<>();
+ private final AtomicReference<RecordSetWriterFactory> recordSetWriterFactoryAtomicReference = new AtomicReference<>();
+
+ @Override
+ protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
+ return PROPERTIES;
+ }
+
+ @Override
+ public Set<Relationship> getRelationships() {
+ return RELATIONSHIPS;
+ }
+
+ @OnScheduled
+ public void setUpClient(final ProcessContext context) {
+ WebClientServiceProvider standardWebClientServiceProvider = context.getProperty(WEB_CLIENT_SERVICE).asControllerService(WebClientServiceProvider.class);
+ RecordReaderFactory recordReaderFactory = context.getProperty(RECORD_READER_FACTORY).asControllerService(RecordReaderFactory.class);
+ RecordSetWriterFactory recordSetWriterFactory = context.getProperty(RECORD_WRITER_FACTORY).asControllerService(RecordSetWriterFactory.class);
+ WebClientService webClientService = standardWebClientServiceProvider.getWebClientService();
+ webClientAtomicReference.set(webClientService);
+ recordReaderFactoryAtomicReference.set(recordReaderFactory);
+ recordSetWriterFactoryAtomicReference.set(recordSetWriterFactory);
+ }
+
+ @Override
+ public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException {
+ FlowFile flowfile = session.get();
+
+ if (skipExecution(context, flowfile)) {
+ return;
+ }
+
+ ComponentLog logger = getLogger();
+ FlowFile responseFlowFile = null;
+
+ try {
+ WebClientService webClientService = webClientAtomicReference.get();
+ URI uri = new URI(context.getProperty(REPORT_URL).evaluateAttributeExpressions(flowfile).getValue().trim());
+ long startNanos = System.nanoTime();
+ String authorization = createAuthorizationHeader(context, flowfile);
+
+ try(HttpResponseEntity httpResponseEntity = webClientService.get().uri(uri).header(HEADER_AUTHORIZATION, authorization).retrieve()) {
+ responseFlowFile = createResponseFlowFile(flowfile, session, context, httpResponseEntity);
+ long elapsedTime = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos);
+ Map<String, String> commonAttributes = createCommonAttributes(uri, httpResponseEntity, elapsedTime);
+
+ if (flowfile != null) {
+ flowfile = session.putAllAttributes(flowfile, decorateWithMimeAttribute(commonAttributes, httpResponseEntity));
+ }
+ if (responseFlowFile != null) {
+ responseFlowFile = session.putAllAttributes(responseFlowFile, commonAttributes);
+ if (flowfile != null) {
+ session.getProvenanceReporter().fetch(responseFlowFile, uri.toString(), elapsedTime);
+ } else {
+ session.getProvenanceReporter().receive(responseFlowFile, uri.toString(), elapsedTime);
+ }
+ }
+
+ route(flowfile, responseFlowFile, session, context, httpResponseEntity.statusCode());
+ }
+ } catch (Exception e) {
+ if (flowfile == null) {
+ logger.error("Request Processing failed", e);
+ context.yield();
+ } else {
+ logger.error("Request Processing failed: {}", flowfile, e);
+ session.penalize(flowfile);
+ flowfile = session.putAttribute(flowfile, GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS, e.getClass().getName());
+ flowfile = session.putAttribute(flowfile, GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE, e.getMessage());
+ session.transfer(flowfile, FAILURE);
+ }
+
+ if (responseFlowFile != null) {
+ session.remove(responseFlowFile);
+ }
+ }
+ }
+
+ @Override
+ protected Collection<ValidationResult> customValidate(ValidationContext validationContext) {
+ List<ValidationResult> results = new ArrayList<>(super.customValidate(validationContext));
+ if (validationContext.getProperty(FIELDS_TO_HASH).isSet() && !validationContext.getProperty(RECORD_READER_FACTORY).isSet()) {
+ results.add(new ValidationResult.Builder()
+ .valid(false)
+ .explanation("Record-Reader and Record-Writer must be set if you would like to hash specific fields")
+ .subject("Workday report configuration")
+ .build());
+ }
+ return results;
+ }
+
+ /*
+ * If we have no FlowFile, and all incoming connections are self-loops then we can continue on.
+ * However, if we have no FlowFile and we have connections coming from other Processors, then
+ * we know that we should run only if we have a FlowFile.
+ */
+ private boolean skipExecution(ProcessContext context, FlowFile flowfile) {
+ return context.hasIncomingConnection() && flowfile == null && context.hasNonLoopConnection();
+ }
+
+ private FlowFile createResponseFlowFile(FlowFile flowfile, ProcessSession session, ProcessContext context, HttpResponseEntity httpResponseEntity)
+ throws IOException, SchemaNotFoundException, MalformedRecordException {
+ FlowFile responseFlowFile = null;
+ String hashingAlgorithm = context.getProperty(HASHING_ALGORITHM).getValue();
+ Set<String> columnsToHash = Optional.ofNullable(context.getProperty(FIELDS_TO_HASH).evaluateAttributeExpressions(flowfile).getValue()).map(String::trim)
+ .map(columns -> columns.split(COLUMNS_TO_HASH_SEPARATOR)).map(Arrays::stream).map(columns -> columns.collect(Collectors.toSet())).orElse(Collections.emptySet());
+ try {
+ if (isSuccess(httpResponseEntity.statusCode())) {
+ responseFlowFile = flowfile != null ? session.create(flowfile) : session.create();
+ InputStream responseBodyStream = httpResponseEntity.body();
+ if (recordReaderFactoryAtomicReference.get() != null) {
+ TransformResult transformResult = transformRecords(session, flowfile, responseFlowFile, hashingAlgorithm, columnsToHash, responseBodyStream);
+ Map<String, String> attributes = new HashMap<>();
+ attributes.put(RECORD_COUNT, String.valueOf(transformResult.getNumberOfRecords()));
+ attributes.put(CoreAttributes.MIME_TYPE.key(), transformResult.getMimeType());
+ responseFlowFile = session.putAllAttributes(responseFlowFile, attributes);
+ } else {
+ responseFlowFile = session.importFrom(responseBodyStream, responseFlowFile);
+ Optional<String> mimeType = httpResponseEntity.headers().getFirstHeader(HEADER_CONTENT_TYPE);
+ if (mimeType.isPresent()) {
+ responseFlowFile = session.putAttribute(responseFlowFile, CoreAttributes.MIME_TYPE.key(), mimeType.get());
+ }
+ }
+ }
+ } catch (Exception e) {
+ session.remove(responseFlowFile);
+ throw e;
+ }
+ return responseFlowFile;
+ }
+
+ private String createAuthorizationHeader(ProcessContext context, FlowFile flowfile) {
+ String userName = context.getProperty(WORKDAY_USERNAME).evaluateAttributeExpressions(flowfile).getValue();
+ String password = context.getProperty(WORKDAY_PASSWORD).evaluateAttributeExpressions(flowfile).getValue();
+ String base64Credential = Base64.getEncoder().encodeToString((userName + USERNAME_PASSWORD_SEPARATOR + password).getBytes(StandardCharsets.ISO_8859_1));
+ return BASIC_PREFIX + base64Credential;
+ }
+
+ private TransformResult transformRecords(ProcessSession session, FlowFile flowfile, FlowFile responseFlowFile, String hashingAlgorithm, Set<String> columnsToHash,
+ InputStream responseBodyStream) throws IOException, SchemaNotFoundException, MalformedRecordException {
+ int numberOfRecords = 0;
+ String mimeType = null;
+ try (RecordReader reader = recordReaderFactoryAtomicReference.get().createRecordReader(flowfile,
+ new BufferedInputStream(responseBodyStream), getLogger())) {
+ RecordSchema schema = recordSetWriterFactoryAtomicReference.get()
+ .getSchema(flowfile == null ? Collections.emptyMap() : flowfile.getAttributes(), reader.getSchema());
+ try (OutputStream responseStream = session.write(responseFlowFile);
+ RecordSetWriter recordSetWriter = recordSetWriterFactoryAtomicReference.get().createWriter(getLogger(), schema, responseStream, responseFlowFile)) {
+ mimeType = recordSetWriter.getMimeType();
+ recordSetWriter.beginRecordSet();
+ Record currentRecord;
+ while ((currentRecord = reader.nextRecord(false, true)) != null) {
+ for (String recordPath : columnsToHash) {
+ RecordPathResult evaluate = RecordPath.compile("hash(" + recordPath + ", '" + hashingAlgorithm + "')").evaluate(currentRecord);
+ evaluate.getSelectedFields().forEach(fieldVal -> fieldVal.updateValue(fieldVal.getValue(), RecordFieldType.STRING.getDataType()));
+ }
+ currentRecord.incorporateInactiveFields();
+ recordSetWriter.write(currentRecord);
+ numberOfRecords++;
+ }
+ }
+ }
+ return new TransformResult(numberOfRecords, mimeType);
+ }
+
+ private void route(FlowFile request, FlowFile response, ProcessSession session, ProcessContext context, int statusCode) {
+ if (!isSuccess(statusCode) && request == null) {
+ context.yield();
+ }
+
+ if (isSuccess(statusCode)) {
+ if (request != null) {
+ session.transfer(request, ORIGINAL);
+ }
+ if (response != null) {
+ session.transfer(response, RESPONSE);
+ }
+ } else {
+ if (request != null) {
+ session.transfer(request, FAILURE);
+ }
+ }
+ }
+
+ private boolean isSuccess(int statusCode) {
+ return statusCode / 100 == 2;
+ }
+
+ private Map<String, String> createCommonAttributes(URI uri, HttpResponseEntity httpResponseEntity, long elapsedTime) {
+ Map<String, String> attributes = new HashMap<>();
+ attributes.put(STATUS_CODE, String.valueOf(httpResponseEntity.statusCode()));
+ attributes.put(REQUEST_URL, uri.toString());
+ attributes.put(REQUEST_DURATION, Long.toString(elapsedTime));
+ attributes.put(TRANSACTION_ID, UUID.randomUUID().toString());
+ return attributes;
+ }
+
+ private Map<String, String> decorateWithMimeAttribute(Map<String, String> commonAttributes, HttpResponseEntity httpResponseEntity) {
Review Comment:
Recommend renaming this method:
```suggestion
private Map<String, String> setMimeType(Map<String, String> commonAttributes, HttpResponseEntity httpResponseEntity) {
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@nifi.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [nifi] exceptionfactory commented on a diff in pull request #6376: NIFI-10455 Get workday report processor
Posted by GitBox <gi...@apache.org>.
exceptionfactory commented on code in PR #6376:
URL: https://github.com/apache/nifi/pull/6376#discussion_r968423220
##########
nifi-nar-bundles/nifi-workday-bundle/nifi-workday-processors/src/main/java/org/apache/nifi/processors/workday/GetWorkdayReport.java:
##########
@@ -0,0 +1,442 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.processors.workday;
+
+import static org.apache.nifi.expression.ExpressionLanguageScope.FLOWFILE_ATTRIBUTES;
+import static org.apache.nifi.expression.ExpressionLanguageScope.NONE;
+import static org.apache.nifi.processor.util.StandardValidators.NON_BLANK_VALIDATOR;
+import static org.apache.nifi.processor.util.StandardValidators.URL_VALIDATOR;
+
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Base64;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import org.apache.nifi.annotation.behavior.EventDriven;
+import org.apache.nifi.annotation.behavior.InputRequirement;
+import org.apache.nifi.annotation.behavior.InputRequirement.Requirement;
+import org.apache.nifi.annotation.behavior.SideEffectFree;
+import org.apache.nifi.annotation.behavior.SupportsBatching;
+import org.apache.nifi.annotation.behavior.WritesAttribute;
+import org.apache.nifi.annotation.behavior.WritesAttributes;
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.annotation.lifecycle.OnScheduled;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.ValidationContext;
+import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.flowfile.attributes.CoreAttributes;
+import org.apache.nifi.logging.ComponentLog;
+import org.apache.nifi.processor.AbstractProcessor;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.processor.util.StandardValidators;
+import org.apache.nifi.record.path.RecordPath;
+import org.apache.nifi.record.path.RecordPathResult;
+import org.apache.nifi.schema.access.SchemaNotFoundException;
+import org.apache.nifi.security.util.crypto.HashAlgorithm;
+import org.apache.nifi.serialization.MalformedRecordException;
+import org.apache.nifi.serialization.RecordReader;
+import org.apache.nifi.serialization.RecordReaderFactory;
+import org.apache.nifi.serialization.RecordSetWriter;
+import org.apache.nifi.serialization.RecordSetWriterFactory;
+import org.apache.nifi.serialization.record.Record;
+import org.apache.nifi.serialization.record.RecordFieldType;
+import org.apache.nifi.serialization.record.RecordSchema;
+import org.apache.nifi.web.client.api.HttpResponseEntity;
+import org.apache.nifi.web.client.api.WebClientService;
+import org.apache.nifi.web.client.provider.api.WebClientServiceProvider;
+
+@Tags({"Workday", "report"})
+@InputRequirement(Requirement.INPUT_ALLOWED)
+@CapabilityDescription("A processor which can interact with a configurable Workday Report. The processor can forward the content without modification, or you can transform it by"
+ + " providing the specific Record Reader and Record Writer services based on your needs. You can also hash, or remove fields using the input parameters and schemes in the Record Writer. "
+ + "Supported Workday report formats are: csv, simplexml, json")
+@EventDriven
+@SideEffectFree
+@SupportsBatching
+@WritesAttributes({
+ @WritesAttribute(attribute = GetWorkdayReport.GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS, description = "The Java exception class raised when the processor fails"),
+ @WritesAttribute(attribute = GetWorkdayReport.GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE, description = "The Java exception message raised when the processor fails"),
+ @WritesAttribute(attribute = "mime.type", description = "Sets the mime.type attribute to the MIME Type specified by the Source / Record Writer"),
+ @WritesAttribute(attribute= GetWorkdayReport.RECORD_COUNT, description = "The number of records in an outgoing FlowFile. This is only populated on the 'success' relationship "
+ + "when Record Reader and Writer is set.")})
+public class GetWorkdayReport extends AbstractProcessor {
+
+ protected static final String STATUS_CODE = "getworkdayreport.status.code";
+ protected static final String REQUEST_URL = "getworkdayreport.request.url";
+ protected static final String REQUEST_DURATION = "getworkdayreport.request.duration";
+ protected static final String TRANSACTION_ID = "getworkdayreport.tx.id";
+ protected static final String GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS = "getworkdayreport.java.exception.class";
+ protected static final String GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE = "getworkdayreport.java.exception.message";
+ protected static final String RECORD_COUNT = "record.count";
+ protected static final String BASIC_PREFIX = "Basic ";
+ protected static final String COLUMNS_TO_HASH_SEPARATOR = ",";
+ protected static final String HEADER_AUTHORIZATION = "Authorization";
+ protected static final String HEADER_CONTENT_TYPE = "Content-Type";
+ protected static final String USERNAME_PASSWORD_SEPARATOR = ":";
+
+ protected static final PropertyDescriptor REPORT_URL = new PropertyDescriptor.Builder()
+ .name("Workday Report URL")
+ .displayName("Workday Report URL")
+ .description("HTTP remote URL of Workday report including a scheme of http or https, as well as a hostname or IP address with optional port and path elements.")
+ .required(true)
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .addValidator(URL_VALIDATOR)
+ .build();
+
+ protected static final PropertyDescriptor WORKDAY_USERNAME = new PropertyDescriptor.Builder()
+ .name("Workday Username")
+ .displayName("Workday Username")
+ .description("The username provided for authentication of Workday requests. Encoded using Base64 for HTTP Basic Authentication as described in RFC 7617.")
+ .required(true)
+ .addValidator(StandardValidators.createRegexMatchingValidator(Pattern.compile("^[\\x20-\\x39\\x3b-\\x7e\\x80-\\xff]+$")))
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .build();
+
+ protected static final PropertyDescriptor WORKDAY_PASSWORD = new PropertyDescriptor.Builder()
+ .name("Workday Password")
+ .displayName("Workday Password")
+ .description("The password provided for authentication of Workday requests. Encoded using Base64 for HTTP Basic Authentication as described in RFC 7617.")
+ .required(true)
+ .sensitive(true)
+ .addValidator(StandardValidators.createRegexMatchingValidator(Pattern.compile("^[\\x20-\\x7e\\x80-\\xff]+$")))
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .build();
+
+ protected static final PropertyDescriptor WEB_CLIENT_SERVICE = new PropertyDescriptor.Builder()
+ .name("Web Client Service Provider")
+ .description("Web client which is used to communicate with the Workday API.")
+ .required(true)
+ .identifiesControllerService(WebClientServiceProvider.class)
+ .build();
+
+ protected static final PropertyDescriptor FIELDS_TO_HASH = new PropertyDescriptor.Builder()
+ .name("Hashed Fields")
+ .displayName("Hashed Fields")
+ .description("Comma separated record paths to be replaced with their corresponding hash.")
+ .required(false)
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .addValidator(NON_BLANK_VALIDATOR)
+ .build();
+
+ protected static final PropertyDescriptor HASHING_ALGORITHM = new PropertyDescriptor.Builder()
+ .name("Hashing Algorithm")
+ .displayName("Hashing Algorithm")
+ .description("Determines what hashing algorithm should be used to perform the hashing function.")
+ .required(true)
+ .allowableValues(HashAlgorithm.SHA256.getName(), HashAlgorithm.SHA512.getName())
+ .defaultValue(HashAlgorithm.SHA256.getName())
+ .addValidator(NON_BLANK_VALIDATOR)
+ .expressionLanguageSupported(NONE)
+ .dependsOn(FIELDS_TO_HASH)
+ .build();
+
+ protected static final PropertyDescriptor RECORD_READER_FACTORY = new PropertyDescriptor.Builder()
+ .name("record-reader")
+ .displayName("Record Reader")
+ .description("Specifies the Controller Service to use for parsing incoming data and determining the data's schema.")
+ .identifiesControllerService(RecordReaderFactory.class)
+ .required(false)
+ .build();
+
+ protected static final PropertyDescriptor RECORD_WRITER_FACTORY = new PropertyDescriptor.Builder()
+ .name("record-writer")
+ .displayName("Record Writer")
+ .description("The Record Writer to use for serializing Records to an output FlowFile.")
+ .identifiesControllerService(RecordSetWriterFactory.class)
+ .dependsOn(RECORD_READER_FACTORY)
+ .required(true)
+ .build();
+
+ protected static final Relationship ORIGINAL = new Relationship.Builder()
+ .name("Original")
+ .description("Request FlowFiles transferred when receiving HTTP responses with a status code between 200 and 299.")
+ .build();
+
+ protected static final Relationship FAILURE = new Relationship.Builder()
+ .name("Failure")
+ .description("Request FlowFiles transferred when receiving socket communication errors.")
+ .build();
+
+ protected static final Relationship RESPONSE = new Relationship.Builder()
+ .name("Response")
+ .description("Response FlowFiles transferred when receiving HTTP responses with a status code between 200 and 299.")
+ .build();
Review Comment:
Although not completely consistent, the general pattern for relationships is all lowercase. `InvokeHTTP` is an exception to this pattern, and although these names are similar, recommend renaming to `original`, `failure`, and `success`.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@nifi.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [nifi] ferencerdei commented on a diff in pull request #6376: NIFI-10455 Get workday report processor
Posted by GitBox <gi...@apache.org>.
ferencerdei commented on code in PR #6376:
URL: https://github.com/apache/nifi/pull/6376#discussion_r969404482
##########
nifi-nar-bundles/nifi-workday-bundle/nifi-workday-processors/pom.xml:
##########
@@ -0,0 +1,114 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-workday-bundle</artifactId>
+ <version>1.18.0-SNAPSHOT</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>nifi-workday-processors</artifactId>
+ <packaging>jar</packaging>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-api</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-web-client-provider-api</artifactId>
+ <version>1.18.0-SNAPSHOT</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-utils</artifactId>
+ <version>1.18.0-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-record</artifactId>
+ <version>1.18.0-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-record-path</artifactId>
+ <version>1.18.0-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-security-utils</artifactId>
+ <version>1.18.0-SNAPSHOT</version>
+ </dependency>
Review Comment:
I've removed the dependency and introduced the enum, but still see the bouncycastle dependencies for example in the bundled-dependencies folder in the nar. I tried to check the same with maven dependency tree but according to it the scope is test there. I'm not sure if this is an issue, or if there is some magic behind the nar creation and these dependencies are required.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@nifi.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [nifi] exceptionfactory commented on a diff in pull request #6376: NIFI-10455 Get workday report processor
Posted by GitBox <gi...@apache.org>.
exceptionfactory commented on code in PR #6376:
URL: https://github.com/apache/nifi/pull/6376#discussion_r971938982
##########
nifi-nar-bundles/nifi-workday-bundle/nifi-workday-processors/src/main/java/org/apache/nifi/processors/workday/GetWorkdayReport.java:
##########
@@ -0,0 +1,466 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.processors.workday;
+
+import static org.apache.nifi.expression.ExpressionLanguageScope.FLOWFILE_ATTRIBUTES;
+import static org.apache.nifi.expression.ExpressionLanguageScope.NONE;
+import static org.apache.nifi.processor.util.StandardValidators.NON_BLANK_VALIDATOR;
+import static org.apache.nifi.processor.util.StandardValidators.URL_VALIDATOR;
+
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Base64;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import org.apache.nifi.annotation.behavior.EventDriven;
+import org.apache.nifi.annotation.behavior.InputRequirement;
+import org.apache.nifi.annotation.behavior.InputRequirement.Requirement;
+import org.apache.nifi.annotation.behavior.SideEffectFree;
+import org.apache.nifi.annotation.behavior.SupportsBatching;
+import org.apache.nifi.annotation.behavior.WritesAttribute;
+import org.apache.nifi.annotation.behavior.WritesAttributes;
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.annotation.lifecycle.OnScheduled;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.ValidationContext;
+import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.flowfile.attributes.CoreAttributes;
+import org.apache.nifi.processor.AbstractProcessor;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.processor.util.StandardValidators;
+import org.apache.nifi.record.path.RecordPath;
+import org.apache.nifi.record.path.RecordPathResult;
+import org.apache.nifi.schema.access.SchemaNotFoundException;
+import org.apache.nifi.serialization.MalformedRecordException;
+import org.apache.nifi.serialization.RecordReader;
+import org.apache.nifi.serialization.RecordReaderFactory;
+import org.apache.nifi.serialization.RecordSetWriter;
+import org.apache.nifi.serialization.RecordSetWriterFactory;
+import org.apache.nifi.serialization.record.Record;
+import org.apache.nifi.serialization.record.RecordFieldType;
+import org.apache.nifi.serialization.record.RecordSchema;
+import org.apache.nifi.web.client.api.HttpResponseEntity;
+import org.apache.nifi.web.client.api.WebClientService;
+import org.apache.nifi.web.client.provider.api.WebClientServiceProvider;
+
+@Tags({"Workday", "report"})
+@InputRequirement(Requirement.INPUT_ALLOWED)
+@CapabilityDescription("A processor which can interact with a configurable Workday Report. The processor can forward the content without modification, or you can transform it by"
+ + " providing the specific Record Reader and Record Writer services based on your needs. You can also hash, or remove fields using the input parameters and schemes in the Record Writer. "
+ + "Supported Workday report formats are: csv, simplexml, json")
+@EventDriven
+@SideEffectFree
+@SupportsBatching
+@WritesAttributes({
+ @WritesAttribute(attribute = GetWorkdayReport.GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS, description = "The Java exception class raised when the processor fails"),
+ @WritesAttribute(attribute = GetWorkdayReport.GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE, description = "The Java exception message raised when the processor fails"),
+ @WritesAttribute(attribute = "mime.type", description = "Sets the mime.type attribute to the MIME Type specified by the Source / Record Writer"),
+ @WritesAttribute(attribute= GetWorkdayReport.RECORD_COUNT, description = "The number of records in an outgoing FlowFile. This is only populated on the 'success' relationship "
+ + "when Record Reader and Writer is set.")})
+public class GetWorkdayReport extends AbstractProcessor {
+
+ protected static final String STATUS_CODE = "getworkdayreport.status.code";
+ protected static final String REQUEST_URL = "getworkdayreport.request.url";
+ protected static final String REQUEST_DURATION = "getworkdayreport.request.duration";
+ protected static final String TRANSACTION_ID = "getworkdayreport.tx.id";
+ protected static final String GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS = "getworkdayreport.java.exception.class";
+ protected static final String GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE = "getworkdayreport.java.exception.message";
+ protected static final String RECORD_COUNT = "record.count";
+ protected static final String BASIC_PREFIX = "Basic ";
+ protected static final String COLUMNS_TO_HASH_SEPARATOR = ",";
+ protected static final String HEADER_AUTHORIZATION = "Authorization";
+ protected static final String HEADER_CONTENT_TYPE = "Content-Type";
+ protected static final String USERNAME_PASSWORD_SEPARATOR = ":";
+
+ protected static final PropertyDescriptor REPORT_URL = new PropertyDescriptor.Builder()
+ .name("Workday Report URL")
+ .displayName("Workday Report URL")
+ .description("HTTP remote URL of Workday report including a scheme of http or https, as well as a hostname or IP address with optional port and path elements.")
+ .required(true)
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .addValidator(URL_VALIDATOR)
+ .build();
+
+ protected static final PropertyDescriptor WORKDAY_USERNAME = new PropertyDescriptor.Builder()
+ .name("Workday Username")
+ .displayName("Workday Username")
+ .description("The username provided for authentication of Workday requests. Encoded using Base64 for HTTP Basic Authentication as described in RFC 7617.")
+ .required(true)
+ .addValidator(StandardValidators.createRegexMatchingValidator(Pattern.compile("^[\\x20-\\x39\\x3b-\\x7e\\x80-\\xff]+$")))
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .build();
+
+ protected static final PropertyDescriptor WORKDAY_PASSWORD = new PropertyDescriptor.Builder()
+ .name("Workday Password")
+ .displayName("Workday Password")
+ .description("The password provided for authentication of Workday requests. Encoded using Base64 for HTTP Basic Authentication as described in RFC 7617.")
+ .required(true)
+ .sensitive(true)
+ .addValidator(StandardValidators.createRegexMatchingValidator(Pattern.compile("^[\\x20-\\x7e\\x80-\\xff]+$")))
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .build();
+
+ protected static final PropertyDescriptor WEB_CLIENT_SERVICE = new PropertyDescriptor.Builder()
+ .name("Web Client Service Provider")
+ .description("Web client which is used to communicate with the Workday API.")
+ .required(true)
+ .identifiesControllerService(WebClientServiceProvider.class)
+ .build();
+
+ protected static final PropertyDescriptor FIELDS_TO_HASH = new PropertyDescriptor.Builder()
+ .name("Hashed Fields")
+ .displayName("Hashed Fields")
+ .description("Comma separated record paths to be replaced with their corresponding hash.")
+ .required(false)
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .addValidator(NON_BLANK_VALIDATOR)
+ .build();
+
+ protected static final PropertyDescriptor HASHING_ALGORITHM = new PropertyDescriptor.Builder()
+ .name("Hashing Algorithm")
+ .displayName("Hashing Algorithm")
+ .description("Determines what hashing algorithm should be used to perform the hashing function.")
+ .required(true)
+ .allowableValues(HashAlgorithm.getNames())
+ .defaultValue(HashAlgorithm.SHA256.getName())
+ .addValidator(NON_BLANK_VALIDATOR)
+ .expressionLanguageSupported(NONE)
+ .dependsOn(FIELDS_TO_HASH)
+ .build();
+
+ protected static final PropertyDescriptor RECORD_READER_FACTORY = new PropertyDescriptor.Builder()
+ .name("record-reader")
+ .displayName("Record Reader")
+ .description("Specifies the Controller Service to use for parsing incoming data and determining the data's schema.")
+ .identifiesControllerService(RecordReaderFactory.class)
+ .required(false)
+ .build();
+
+ protected static final PropertyDescriptor RECORD_WRITER_FACTORY = new PropertyDescriptor.Builder()
+ .name("record-writer")
+ .displayName("Record Writer")
+ .description("The Record Writer to use for serializing Records to an output FlowFile.")
+ .identifiesControllerService(RecordSetWriterFactory.class)
+ .dependsOn(RECORD_READER_FACTORY)
+ .required(true)
+ .build();
+
+ protected static final Relationship ORIGINAL = new Relationship.Builder()
+ .name("original")
+ .description("Request FlowFiles transferred when receiving HTTP responses with a status code between 200 and 299.")
+ .build();
+
+ protected static final Relationship FAILURE = new Relationship.Builder()
+ .name("failure")
+ .description("Request FlowFiles transferred when receiving socket communication errors.")
+ .build();
+
+ protected static final Relationship SUCCESS = new Relationship.Builder()
+ .name("success")
+ .description("Response FlowFiles transferred when receiving HTTP responses with a status code between 200 and 299.")
+ .build();
+
+ protected static final Set<Relationship> RELATIONSHIPS = Collections.unmodifiableSet(new HashSet<>(Arrays.asList(ORIGINAL, SUCCESS, FAILURE)));
+ protected static final List<PropertyDescriptor> PROPERTIES = Collections.unmodifiableList(Arrays.asList(
+ REPORT_URL,
+ WORKDAY_USERNAME,
+ WORKDAY_PASSWORD,
+ WEB_CLIENT_SERVICE,
+ FIELDS_TO_HASH,
+ HASHING_ALGORITHM,
+ RECORD_READER_FACTORY,
+ RECORD_WRITER_FACTORY
+ ));
+
+ private final AtomicReference<WebClientService> webClientReference = new AtomicReference<>();
+ private final AtomicReference<RecordReaderFactory> recordReaderFactoryReference = new AtomicReference<>();
+ private final AtomicReference<RecordSetWriterFactory> recordSetWriterFactoryReference = new AtomicReference<>();
+
+ @Override
+ protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
+ return PROPERTIES;
+ }
+
+ @Override
+ public Set<Relationship> getRelationships() {
+ return RELATIONSHIPS;
+ }
+
+ @OnScheduled
+ public void setUpClient(final ProcessContext context) {
+ WebClientServiceProvider standardWebClientServiceProvider = context.getProperty(WEB_CLIENT_SERVICE).asControllerService(WebClientServiceProvider.class);
+ RecordReaderFactory recordReaderFactory = context.getProperty(RECORD_READER_FACTORY).asControllerService(RecordReaderFactory.class);
+ RecordSetWriterFactory recordSetWriterFactory = context.getProperty(RECORD_WRITER_FACTORY).asControllerService(RecordSetWriterFactory.class);
+ WebClientService webClientService = standardWebClientServiceProvider.getWebClientService();
+ webClientReference.set(webClientService);
+ recordReaderFactoryReference.set(recordReaderFactory);
+ recordSetWriterFactoryReference.set(recordSetWriterFactory);
+ }
+
+ @Override
+ public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException {
+ FlowFile flowFile = session.get();
+
+ if (skipExecution(context, flowFile)) {
+ return;
+ }
+
+ FlowFile responseFlowFile = null;
+
+ try {
+ WebClientService webClientService = webClientReference.get();
+ URI uri = new URI(context.getProperty(REPORT_URL).evaluateAttributeExpressions(flowFile).getValue().trim());
+ long startNanos = System.nanoTime();
+ String authorization = createAuthorizationHeader(context, flowFile);
+
+ try(HttpResponseEntity httpResponseEntity = webClientService.get()
+ .uri(uri)
+ .header(HEADER_AUTHORIZATION, authorization)
+ .retrieve()) {
+ responseFlowFile = createResponseFlowFile(flowFile, session, context, httpResponseEntity);
+ long elapsedTime = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos);
+ Map<String, String> commonAttributes = createCommonAttributes(uri, httpResponseEntity, elapsedTime);
+
+ if (flowFile != null) {
+ flowFile = session.putAllAttributes(flowFile, setMimeType(commonAttributes, httpResponseEntity));
+ }
+ if (responseFlowFile != null) {
+ responseFlowFile = session.putAllAttributes(responseFlowFile, commonAttributes);
+ if (flowFile == null) {
+ session.getProvenanceReporter().receive(responseFlowFile, uri.toString(), elapsedTime);
+ } else {
+ session.getProvenanceReporter().fetch(responseFlowFile, uri.toString(), elapsedTime);
+ }
+ }
+
+ route(flowFile, responseFlowFile, session, context, httpResponseEntity.statusCode());
+ }
+ } catch (Exception e) {
+ if (flowFile == null) {
+ getLogger().error("Request Processing failed", e);
+ context.yield();
+ } else {
+ getLogger().error("Request Processing failed: {}", flowFile, e);
+ session.penalize(flowFile);
+ flowFile = session.putAttribute(flowFile, GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS, e.getClass().getSimpleName());
+ flowFile = session.putAttribute(flowFile, GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE, e.getMessage());
+ session.transfer(flowFile, FAILURE);
+ }
+
+ if (responseFlowFile != null) {
+ session.remove(responseFlowFile);
+ }
+ }
+ }
+
+ @Override
+ protected Collection<ValidationResult> customValidate(ValidationContext validationContext) {
+ List<ValidationResult> results = new ArrayList<>(super.customValidate(validationContext));
+ if (validationContext.getProperty(FIELDS_TO_HASH).isSet() && !validationContext.getProperty(RECORD_READER_FACTORY).isSet()) {
+ results.add(new ValidationResult.Builder()
+ .valid(false)
+ .explanation("Record-Reader and Record-Writer must be set if you would like to hash specific fields")
+ .subject("Workday report configuration")
+ .build());
+ }
+ return results;
+ }
+
+ /*
+ * If we have no FlowFile, and all incoming connections are self-loops then we can continue on.
+ * However, if we have no FlowFile and we have connections coming from other Processors, then
+ * we know that we should run only if we have a FlowFile.
+ */
+ private boolean skipExecution(ProcessContext context, FlowFile flowfile) {
+ return context.hasIncomingConnection() && flowfile == null && context.hasNonLoopConnection();
+ }
+
+ private FlowFile createResponseFlowFile(FlowFile flowfile, ProcessSession session, ProcessContext context, HttpResponseEntity httpResponseEntity)
+ throws IOException, SchemaNotFoundException, MalformedRecordException {
+ FlowFile responseFlowFile = null;
+ String hashingAlgorithm = context.getProperty(HASHING_ALGORITHM).getValue();
+ Set<String> columnsToHash = Optional.ofNullable(
+ context.getProperty(FIELDS_TO_HASH)
+ .evaluateAttributeExpressions(flowfile)
+ .getValue())
+ .map(String::trim)
+ .map(columns -> columns.split(COLUMNS_TO_HASH_SEPARATOR))
+ .map(Arrays::stream)
+ .map(columns -> columns.collect(Collectors.toSet()))
+ .orElse(Collections.emptySet());
+ try {
+ if (isSuccess(httpResponseEntity.statusCode())) {
+ responseFlowFile = flowfile == null ? session.create() : session.create(flowfile);
+ InputStream responseBodyStream = httpResponseEntity.body();
+ if (recordReaderFactoryReference.get() != null) {
+ TransformResult transformResult = transformRecords(session, flowfile, responseFlowFile, hashingAlgorithm, columnsToHash, responseBodyStream);
+ Map<String, String> attributes = new HashMap<>();
+ attributes.put(RECORD_COUNT, String.valueOf(transformResult.getNumberOfRecords()));
+ attributes.put(CoreAttributes.MIME_TYPE.key(), transformResult.getMimeType());
+ responseFlowFile = session.putAllAttributes(responseFlowFile, attributes);
+ } else {
+ responseFlowFile = session.importFrom(responseBodyStream, responseFlowFile);
+ Optional<String> mimeType = httpResponseEntity.headers().getFirstHeader(HEADER_CONTENT_TYPE);
+ if (mimeType.isPresent()) {
+ responseFlowFile = session.putAttribute(responseFlowFile, CoreAttributes.MIME_TYPE.key(), mimeType.get());
+ }
+ }
+ }
+ } catch (Exception e) {
+ session.remove(responseFlowFile);
+ throw e;
+ }
+ return responseFlowFile;
+ }
+
+ private String createAuthorizationHeader(ProcessContext context, FlowFile flowfile) {
+ String userName = context.getProperty(WORKDAY_USERNAME).evaluateAttributeExpressions(flowfile).getValue();
+ String password = context.getProperty(WORKDAY_PASSWORD).evaluateAttributeExpressions(flowfile).getValue();
+ String base64Credential = Base64.getEncoder().encodeToString((userName + USERNAME_PASSWORD_SEPARATOR + password).getBytes(StandardCharsets.UTF_8));
+ return BASIC_PREFIX + base64Credential;
+ }
+
+ private TransformResult transformRecords(ProcessSession session, FlowFile flowfile, FlowFile responseFlowFile, String hashingAlgorithm, Set<String> columnsToHash,
+ InputStream responseBodyStream) throws IOException, SchemaNotFoundException, MalformedRecordException {
+ int numberOfRecords = 0;
+ String mimeType = null;
+ try (RecordReader reader = recordReaderFactoryReference.get().createRecordReader(flowfile,
+ new BufferedInputStream(responseBodyStream), getLogger())) {
+ RecordSchema schema = recordSetWriterFactoryReference.get()
+ .getSchema(flowfile == null ? Collections.emptyMap() : flowfile.getAttributes(), reader.getSchema());
+ try (OutputStream responseStream = session.write(responseFlowFile);
+ RecordSetWriter recordSetWriter = recordSetWriterFactoryReference.get().createWriter(getLogger(), schema, responseStream, responseFlowFile)) {
+ mimeType = recordSetWriter.getMimeType();
+ recordSetWriter.beginRecordSet();
+ Record currentRecord;
+ // as the report can be changed independently from the flow, it's safer to ignore field types and unknown fields in the Record Reading process
+ while ((currentRecord = reader.nextRecord(false, true)) != null) {
+ for (String recordPath : columnsToHash) {
+ RecordPathResult evaluate = RecordPath.compile("hash(" + recordPath + ", '" + hashingAlgorithm + "')").evaluate(currentRecord);
+ evaluate.getSelectedFields().forEach(fieldVal -> fieldVal.updateValue(fieldVal.getValue(), RecordFieldType.STRING.getDataType()));
+ }
Review Comment:
Precompiling the paths might help, but it would still be much better to remove the functionality, and the associated dependency, since `nifi-record-path` brings in a number of transitive dependencies that are otherwise unnecessary for this processor. Removing this functionality would keep the capabilities of this processor more focused on retrieval and formatting.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@nifi.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [nifi] ferencerdei commented on a diff in pull request #6376: NIFI-10455 Get workday report processor
Posted by GitBox <gi...@apache.org>.
ferencerdei commented on code in PR #6376:
URL: https://github.com/apache/nifi/pull/6376#discussion_r973021578
##########
nifi-nar-bundles/nifi-workday-bundle/nifi-workday-processors/src/main/java/org/apache/nifi/processors/workday/GetWorkdayReport.java:
##########
@@ -0,0 +1,466 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.processors.workday;
+
+import static org.apache.nifi.expression.ExpressionLanguageScope.FLOWFILE_ATTRIBUTES;
+import static org.apache.nifi.expression.ExpressionLanguageScope.NONE;
+import static org.apache.nifi.processor.util.StandardValidators.NON_BLANK_VALIDATOR;
+import static org.apache.nifi.processor.util.StandardValidators.URL_VALIDATOR;
+
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Base64;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import org.apache.nifi.annotation.behavior.EventDriven;
+import org.apache.nifi.annotation.behavior.InputRequirement;
+import org.apache.nifi.annotation.behavior.InputRequirement.Requirement;
+import org.apache.nifi.annotation.behavior.SideEffectFree;
+import org.apache.nifi.annotation.behavior.SupportsBatching;
+import org.apache.nifi.annotation.behavior.WritesAttribute;
+import org.apache.nifi.annotation.behavior.WritesAttributes;
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.annotation.lifecycle.OnScheduled;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.ValidationContext;
+import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.flowfile.attributes.CoreAttributes;
+import org.apache.nifi.processor.AbstractProcessor;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.processor.util.StandardValidators;
+import org.apache.nifi.record.path.RecordPath;
+import org.apache.nifi.record.path.RecordPathResult;
+import org.apache.nifi.schema.access.SchemaNotFoundException;
+import org.apache.nifi.serialization.MalformedRecordException;
+import org.apache.nifi.serialization.RecordReader;
+import org.apache.nifi.serialization.RecordReaderFactory;
+import org.apache.nifi.serialization.RecordSetWriter;
+import org.apache.nifi.serialization.RecordSetWriterFactory;
+import org.apache.nifi.serialization.record.Record;
+import org.apache.nifi.serialization.record.RecordFieldType;
+import org.apache.nifi.serialization.record.RecordSchema;
+import org.apache.nifi.web.client.api.HttpResponseEntity;
+import org.apache.nifi.web.client.api.WebClientService;
+import org.apache.nifi.web.client.provider.api.WebClientServiceProvider;
+
+@Tags({"Workday", "report"})
+@InputRequirement(Requirement.INPUT_ALLOWED)
+@CapabilityDescription("A processor which can interact with a configurable Workday Report. The processor can forward the content without modification, or you can transform it by"
+ + " providing the specific Record Reader and Record Writer services based on your needs. You can also hash, or remove fields using the input parameters and schemes in the Record Writer. "
+ + "Supported Workday report formats are: csv, simplexml, json")
+@EventDriven
+@SideEffectFree
+@SupportsBatching
+@WritesAttributes({
+ @WritesAttribute(attribute = GetWorkdayReport.GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS, description = "The Java exception class raised when the processor fails"),
+ @WritesAttribute(attribute = GetWorkdayReport.GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE, description = "The Java exception message raised when the processor fails"),
+ @WritesAttribute(attribute = "mime.type", description = "Sets the mime.type attribute to the MIME Type specified by the Source / Record Writer"),
+ @WritesAttribute(attribute= GetWorkdayReport.RECORD_COUNT, description = "The number of records in an outgoing FlowFile. This is only populated on the 'success' relationship "
+ + "when Record Reader and Writer is set.")})
+public class GetWorkdayReport extends AbstractProcessor {
+
+ protected static final String STATUS_CODE = "getworkdayreport.status.code";
+ protected static final String REQUEST_URL = "getworkdayreport.request.url";
+ protected static final String REQUEST_DURATION = "getworkdayreport.request.duration";
+ protected static final String TRANSACTION_ID = "getworkdayreport.tx.id";
+ protected static final String GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS = "getworkdayreport.java.exception.class";
+ protected static final String GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE = "getworkdayreport.java.exception.message";
+ protected static final String RECORD_COUNT = "record.count";
+ protected static final String BASIC_PREFIX = "Basic ";
+ protected static final String COLUMNS_TO_HASH_SEPARATOR = ",";
+ protected static final String HEADER_AUTHORIZATION = "Authorization";
+ protected static final String HEADER_CONTENT_TYPE = "Content-Type";
+ protected static final String USERNAME_PASSWORD_SEPARATOR = ":";
+
+ protected static final PropertyDescriptor REPORT_URL = new PropertyDescriptor.Builder()
+ .name("Workday Report URL")
+ .displayName("Workday Report URL")
+ .description("HTTP remote URL of Workday report including a scheme of http or https, as well as a hostname or IP address with optional port and path elements.")
+ .required(true)
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .addValidator(URL_VALIDATOR)
+ .build();
+
+ protected static final PropertyDescriptor WORKDAY_USERNAME = new PropertyDescriptor.Builder()
+ .name("Workday Username")
+ .displayName("Workday Username")
+ .description("The username provided for authentication of Workday requests. Encoded using Base64 for HTTP Basic Authentication as described in RFC 7617.")
+ .required(true)
+ .addValidator(StandardValidators.createRegexMatchingValidator(Pattern.compile("^[\\x20-\\x39\\x3b-\\x7e\\x80-\\xff]+$")))
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .build();
+
+ protected static final PropertyDescriptor WORKDAY_PASSWORD = new PropertyDescriptor.Builder()
+ .name("Workday Password")
+ .displayName("Workday Password")
+ .description("The password provided for authentication of Workday requests. Encoded using Base64 for HTTP Basic Authentication as described in RFC 7617.")
+ .required(true)
+ .sensitive(true)
+ .addValidator(StandardValidators.createRegexMatchingValidator(Pattern.compile("^[\\x20-\\x7e\\x80-\\xff]+$")))
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .build();
+
+ protected static final PropertyDescriptor WEB_CLIENT_SERVICE = new PropertyDescriptor.Builder()
+ .name("Web Client Service Provider")
+ .description("Web client which is used to communicate with the Workday API.")
+ .required(true)
+ .identifiesControllerService(WebClientServiceProvider.class)
+ .build();
+
+ protected static final PropertyDescriptor FIELDS_TO_HASH = new PropertyDescriptor.Builder()
+ .name("Hashed Fields")
+ .displayName("Hashed Fields")
+ .description("Comma separated record paths to be replaced with their corresponding hash.")
+ .required(false)
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .addValidator(NON_BLANK_VALIDATOR)
+ .build();
+
+ protected static final PropertyDescriptor HASHING_ALGORITHM = new PropertyDescriptor.Builder()
+ .name("Hashing Algorithm")
+ .displayName("Hashing Algorithm")
+ .description("Determines what hashing algorithm should be used to perform the hashing function.")
+ .required(true)
+ .allowableValues(HashAlgorithm.getNames())
+ .defaultValue(HashAlgorithm.SHA256.getName())
+ .addValidator(NON_BLANK_VALIDATOR)
+ .expressionLanguageSupported(NONE)
+ .dependsOn(FIELDS_TO_HASH)
+ .build();
+
+ protected static final PropertyDescriptor RECORD_READER_FACTORY = new PropertyDescriptor.Builder()
+ .name("record-reader")
+ .displayName("Record Reader")
+ .description("Specifies the Controller Service to use for parsing incoming data and determining the data's schema.")
+ .identifiesControllerService(RecordReaderFactory.class)
+ .required(false)
+ .build();
+
+ protected static final PropertyDescriptor RECORD_WRITER_FACTORY = new PropertyDescriptor.Builder()
+ .name("record-writer")
+ .displayName("Record Writer")
+ .description("The Record Writer to use for serializing Records to an output FlowFile.")
+ .identifiesControllerService(RecordSetWriterFactory.class)
+ .dependsOn(RECORD_READER_FACTORY)
+ .required(true)
+ .build();
+
+ protected static final Relationship ORIGINAL = new Relationship.Builder()
+ .name("original")
+ .description("Request FlowFiles transferred when receiving HTTP responses with a status code between 200 and 299.")
+ .build();
+
+ protected static final Relationship FAILURE = new Relationship.Builder()
+ .name("failure")
+ .description("Request FlowFiles transferred when receiving socket communication errors.")
+ .build();
+
+ protected static final Relationship SUCCESS = new Relationship.Builder()
+ .name("success")
+ .description("Response FlowFiles transferred when receiving HTTP responses with a status code between 200 and 299.")
+ .build();
+
+ protected static final Set<Relationship> RELATIONSHIPS = Collections.unmodifiableSet(new HashSet<>(Arrays.asList(ORIGINAL, SUCCESS, FAILURE)));
+ protected static final List<PropertyDescriptor> PROPERTIES = Collections.unmodifiableList(Arrays.asList(
+ REPORT_URL,
+ WORKDAY_USERNAME,
+ WORKDAY_PASSWORD,
+ WEB_CLIENT_SERVICE,
+ FIELDS_TO_HASH,
+ HASHING_ALGORITHM,
+ RECORD_READER_FACTORY,
+ RECORD_WRITER_FACTORY
+ ));
+
+ private final AtomicReference<WebClientService> webClientReference = new AtomicReference<>();
+ private final AtomicReference<RecordReaderFactory> recordReaderFactoryReference = new AtomicReference<>();
+ private final AtomicReference<RecordSetWriterFactory> recordSetWriterFactoryReference = new AtomicReference<>();
+
+ @Override
+ protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
+ return PROPERTIES;
+ }
+
+ @Override
+ public Set<Relationship> getRelationships() {
+ return RELATIONSHIPS;
+ }
+
+ @OnScheduled
+ public void setUpClient(final ProcessContext context) {
+ WebClientServiceProvider standardWebClientServiceProvider = context.getProperty(WEB_CLIENT_SERVICE).asControllerService(WebClientServiceProvider.class);
+ RecordReaderFactory recordReaderFactory = context.getProperty(RECORD_READER_FACTORY).asControllerService(RecordReaderFactory.class);
+ RecordSetWriterFactory recordSetWriterFactory = context.getProperty(RECORD_WRITER_FACTORY).asControllerService(RecordSetWriterFactory.class);
+ WebClientService webClientService = standardWebClientServiceProvider.getWebClientService();
+ webClientReference.set(webClientService);
+ recordReaderFactoryReference.set(recordReaderFactory);
+ recordSetWriterFactoryReference.set(recordSetWriterFactory);
+ }
+
+ @Override
+ public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException {
+ FlowFile flowFile = session.get();
+
+ if (skipExecution(context, flowFile)) {
+ return;
+ }
+
+ FlowFile responseFlowFile = null;
+
+ try {
+ WebClientService webClientService = webClientReference.get();
+ URI uri = new URI(context.getProperty(REPORT_URL).evaluateAttributeExpressions(flowFile).getValue().trim());
+ long startNanos = System.nanoTime();
+ String authorization = createAuthorizationHeader(context, flowFile);
+
+ try(HttpResponseEntity httpResponseEntity = webClientService.get()
+ .uri(uri)
+ .header(HEADER_AUTHORIZATION, authorization)
+ .retrieve()) {
+ responseFlowFile = createResponseFlowFile(flowFile, session, context, httpResponseEntity);
+ long elapsedTime = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos);
+ Map<String, String> commonAttributes = createCommonAttributes(uri, httpResponseEntity, elapsedTime);
+
+ if (flowFile != null) {
+ flowFile = session.putAllAttributes(flowFile, setMimeType(commonAttributes, httpResponseEntity));
+ }
+ if (responseFlowFile != null) {
+ responseFlowFile = session.putAllAttributes(responseFlowFile, commonAttributes);
+ if (flowFile == null) {
+ session.getProvenanceReporter().receive(responseFlowFile, uri.toString(), elapsedTime);
+ } else {
+ session.getProvenanceReporter().fetch(responseFlowFile, uri.toString(), elapsedTime);
+ }
+ }
+
+ route(flowFile, responseFlowFile, session, context, httpResponseEntity.statusCode());
+ }
+ } catch (Exception e) {
+ if (flowFile == null) {
+ getLogger().error("Request Processing failed", e);
+ context.yield();
+ } else {
+ getLogger().error("Request Processing failed: {}", flowFile, e);
+ session.penalize(flowFile);
+ flowFile = session.putAttribute(flowFile, GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS, e.getClass().getSimpleName());
+ flowFile = session.putAttribute(flowFile, GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE, e.getMessage());
+ session.transfer(flowFile, FAILURE);
+ }
+
+ if (responseFlowFile != null) {
+ session.remove(responseFlowFile);
+ }
+ }
+ }
+
+ @Override
+ protected Collection<ValidationResult> customValidate(ValidationContext validationContext) {
+ List<ValidationResult> results = new ArrayList<>(super.customValidate(validationContext));
+ if (validationContext.getProperty(FIELDS_TO_HASH).isSet() && !validationContext.getProperty(RECORD_READER_FACTORY).isSet()) {
+ results.add(new ValidationResult.Builder()
+ .valid(false)
+ .explanation("Record-Reader and Record-Writer must be set if you would like to hash specific fields")
+ .subject("Workday report configuration")
+ .build());
+ }
+ return results;
+ }
+
+ /*
+ * If we have no FlowFile, and all incoming connections are self-loops then we can continue on.
+ * However, if we have no FlowFile and we have connections coming from other Processors, then
+ * we know that we should run only if we have a FlowFile.
+ */
+ private boolean skipExecution(ProcessContext context, FlowFile flowfile) {
+ return context.hasIncomingConnection() && flowfile == null && context.hasNonLoopConnection();
+ }
+
+ private FlowFile createResponseFlowFile(FlowFile flowfile, ProcessSession session, ProcessContext context, HttpResponseEntity httpResponseEntity)
+ throws IOException, SchemaNotFoundException, MalformedRecordException {
+ FlowFile responseFlowFile = null;
+ String hashingAlgorithm = context.getProperty(HASHING_ALGORITHM).getValue();
+ Set<String> columnsToHash = Optional.ofNullable(
+ context.getProperty(FIELDS_TO_HASH)
+ .evaluateAttributeExpressions(flowfile)
+ .getValue())
+ .map(String::trim)
+ .map(columns -> columns.split(COLUMNS_TO_HASH_SEPARATOR))
+ .map(Arrays::stream)
+ .map(columns -> columns.collect(Collectors.toSet()))
+ .orElse(Collections.emptySet());
+ try {
+ if (isSuccess(httpResponseEntity.statusCode())) {
+ responseFlowFile = flowfile == null ? session.create() : session.create(flowfile);
+ InputStream responseBodyStream = httpResponseEntity.body();
+ if (recordReaderFactoryReference.get() != null) {
+ TransformResult transformResult = transformRecords(session, flowfile, responseFlowFile, hashingAlgorithm, columnsToHash, responseBodyStream);
+ Map<String, String> attributes = new HashMap<>();
+ attributes.put(RECORD_COUNT, String.valueOf(transformResult.getNumberOfRecords()));
+ attributes.put(CoreAttributes.MIME_TYPE.key(), transformResult.getMimeType());
+ responseFlowFile = session.putAllAttributes(responseFlowFile, attributes);
+ } else {
+ responseFlowFile = session.importFrom(responseBodyStream, responseFlowFile);
+ Optional<String> mimeType = httpResponseEntity.headers().getFirstHeader(HEADER_CONTENT_TYPE);
+ if (mimeType.isPresent()) {
+ responseFlowFile = session.putAttribute(responseFlowFile, CoreAttributes.MIME_TYPE.key(), mimeType.get());
+ }
+ }
+ }
+ } catch (Exception e) {
+ session.remove(responseFlowFile);
+ throw e;
+ }
+ return responseFlowFile;
+ }
+
+ private String createAuthorizationHeader(ProcessContext context, FlowFile flowfile) {
+ String userName = context.getProperty(WORKDAY_USERNAME).evaluateAttributeExpressions(flowfile).getValue();
+ String password = context.getProperty(WORKDAY_PASSWORD).evaluateAttributeExpressions(flowfile).getValue();
+ String base64Credential = Base64.getEncoder().encodeToString((userName + USERNAME_PASSWORD_SEPARATOR + password).getBytes(StandardCharsets.UTF_8));
+ return BASIC_PREFIX + base64Credential;
+ }
+
+ private TransformResult transformRecords(ProcessSession session, FlowFile flowfile, FlowFile responseFlowFile, String hashingAlgorithm, Set<String> columnsToHash,
+ InputStream responseBodyStream) throws IOException, SchemaNotFoundException, MalformedRecordException {
+ int numberOfRecords = 0;
+ String mimeType = null;
+ try (RecordReader reader = recordReaderFactoryReference.get().createRecordReader(flowfile,
+ new BufferedInputStream(responseBodyStream), getLogger())) {
+ RecordSchema schema = recordSetWriterFactoryReference.get()
+ .getSchema(flowfile == null ? Collections.emptyMap() : flowfile.getAttributes(), reader.getSchema());
+ try (OutputStream responseStream = session.write(responseFlowFile);
+ RecordSetWriter recordSetWriter = recordSetWriterFactoryReference.get().createWriter(getLogger(), schema, responseStream, responseFlowFile)) {
+ mimeType = recordSetWriter.getMimeType();
+ recordSetWriter.beginRecordSet();
+ Record currentRecord;
+ // as the report can be changed independently from the flow, it's safer to ignore field types and unknown fields in the Record Reading process
+ while ((currentRecord = reader.nextRecord(false, true)) != null) {
+ for (String recordPath : columnsToHash) {
+ RecordPathResult evaluate = RecordPath.compile("hash(" + recordPath + ", '" + hashingAlgorithm + "')").evaluate(currentRecord);
+ evaluate.getSelectedFields().forEach(fieldVal -> fieldVal.updateValue(fieldVal.getValue(), RecordFieldType.STRING.getDataType()));
+ }
+ currentRecord.incorporateInactiveFields();
+ recordSetWriter.write(currentRecord);
+ numberOfRecords++;
+ }
+ }
+ }
+ return new TransformResult(numberOfRecords, mimeType);
+ }
+
+ private void route(FlowFile request, FlowFile response, ProcessSession session, ProcessContext context, int statusCode) {
+ if (!isSuccess(statusCode) && request == null) {
+ context.yield();
+ }
+
+ if (isSuccess(statusCode)) {
+ if (request != null) {
+ session.transfer(request, ORIGINAL);
+ }
+ if (response != null) {
+ session.transfer(response, SUCCESS);
+ }
+ } else {
+ if (request != null) {
+ session.transfer(request, FAILURE);
+ }
+ }
+ }
+
+ private boolean isSuccess(int statusCode) {
+ return statusCode >= 200 && statusCode < 300;
+ }
+
+ private Map<String, String> createCommonAttributes(URI uri, HttpResponseEntity httpResponseEntity, long elapsedTime) {
+ Map<String, String> attributes = new HashMap<>();
+ attributes.put(STATUS_CODE, String.valueOf(httpResponseEntity.statusCode()));
+ attributes.put(REQUEST_URL, uri.toString());
+ attributes.put(REQUEST_DURATION, Long.toString(elapsedTime));
+ attributes.put(TRANSACTION_ID, UUID.randomUUID().toString());
+ return attributes;
+ }
+
+ private Map<String, String> setMimeType(Map<String, String> commonAttributes, HttpResponseEntity httpResponseEntity) {
+ Map<String, String> attributes = commonAttributes;
+ Optional<String> contentType = httpResponseEntity.headers().getFirstHeader(HEADER_CONTENT_TYPE);
+ if (contentType.isPresent()) {
+ attributes = new HashMap<>(commonAttributes);
+ attributes.put(CoreAttributes.MIME_TYPE.key(), contentType.get());
+ }
+ return attributes;
+ }
+
+ protected enum HashAlgorithm {
+ SHA256("SHA-256"),
+ SHA512("SHA-512");
+
+ private final String name;
+
+ HashAlgorithm(String name) {
+ this.name = name;
+ }
+
+ private String getName() {
+ return name;
+ }
+
+ private static String[] getNames() {
+ return Arrays.stream(HashAlgorithm.values()).map(HashAlgorithm::getName).toArray(String[]::new);
+ }
+ }
+
+ private class TransformResult {
Review Comment:
done
##########
nifi-nar-bundles/nifi-workday-bundle/nifi-workday-processors/src/main/java/org/apache/nifi/processors/workday/GetWorkdayReport.java:
##########
@@ -0,0 +1,466 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.processors.workday;
+
+import static org.apache.nifi.expression.ExpressionLanguageScope.FLOWFILE_ATTRIBUTES;
+import static org.apache.nifi.expression.ExpressionLanguageScope.NONE;
+import static org.apache.nifi.processor.util.StandardValidators.NON_BLANK_VALIDATOR;
+import static org.apache.nifi.processor.util.StandardValidators.URL_VALIDATOR;
+
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Base64;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import org.apache.nifi.annotation.behavior.EventDriven;
+import org.apache.nifi.annotation.behavior.InputRequirement;
+import org.apache.nifi.annotation.behavior.InputRequirement.Requirement;
+import org.apache.nifi.annotation.behavior.SideEffectFree;
+import org.apache.nifi.annotation.behavior.SupportsBatching;
+import org.apache.nifi.annotation.behavior.WritesAttribute;
+import org.apache.nifi.annotation.behavior.WritesAttributes;
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.annotation.lifecycle.OnScheduled;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.ValidationContext;
+import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.flowfile.attributes.CoreAttributes;
+import org.apache.nifi.processor.AbstractProcessor;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.processor.util.StandardValidators;
+import org.apache.nifi.record.path.RecordPath;
+import org.apache.nifi.record.path.RecordPathResult;
+import org.apache.nifi.schema.access.SchemaNotFoundException;
+import org.apache.nifi.serialization.MalformedRecordException;
+import org.apache.nifi.serialization.RecordReader;
+import org.apache.nifi.serialization.RecordReaderFactory;
+import org.apache.nifi.serialization.RecordSetWriter;
+import org.apache.nifi.serialization.RecordSetWriterFactory;
+import org.apache.nifi.serialization.record.Record;
+import org.apache.nifi.serialization.record.RecordFieldType;
+import org.apache.nifi.serialization.record.RecordSchema;
+import org.apache.nifi.web.client.api.HttpResponseEntity;
+import org.apache.nifi.web.client.api.WebClientService;
+import org.apache.nifi.web.client.provider.api.WebClientServiceProvider;
+
+@Tags({"Workday", "report"})
+@InputRequirement(Requirement.INPUT_ALLOWED)
+@CapabilityDescription("A processor which can interact with a configurable Workday Report. The processor can forward the content without modification, or you can transform it by"
+ + " providing the specific Record Reader and Record Writer services based on your needs. You can also hash, or remove fields using the input parameters and schemes in the Record Writer. "
+ + "Supported Workday report formats are: csv, simplexml, json")
+@EventDriven
+@SideEffectFree
+@SupportsBatching
+@WritesAttributes({
+ @WritesAttribute(attribute = GetWorkdayReport.GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS, description = "The Java exception class raised when the processor fails"),
+ @WritesAttribute(attribute = GetWorkdayReport.GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE, description = "The Java exception message raised when the processor fails"),
+ @WritesAttribute(attribute = "mime.type", description = "Sets the mime.type attribute to the MIME Type specified by the Source / Record Writer"),
+ @WritesAttribute(attribute= GetWorkdayReport.RECORD_COUNT, description = "The number of records in an outgoing FlowFile. This is only populated on the 'success' relationship "
+ + "when Record Reader and Writer is set.")})
+public class GetWorkdayReport extends AbstractProcessor {
+
+ protected static final String STATUS_CODE = "getworkdayreport.status.code";
+ protected static final String REQUEST_URL = "getworkdayreport.request.url";
+ protected static final String REQUEST_DURATION = "getworkdayreport.request.duration";
+ protected static final String TRANSACTION_ID = "getworkdayreport.tx.id";
+ protected static final String GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS = "getworkdayreport.java.exception.class";
+ protected static final String GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE = "getworkdayreport.java.exception.message";
+ protected static final String RECORD_COUNT = "record.count";
+ protected static final String BASIC_PREFIX = "Basic ";
+ protected static final String COLUMNS_TO_HASH_SEPARATOR = ",";
+ protected static final String HEADER_AUTHORIZATION = "Authorization";
+ protected static final String HEADER_CONTENT_TYPE = "Content-Type";
+ protected static final String USERNAME_PASSWORD_SEPARATOR = ":";
+
+ protected static final PropertyDescriptor REPORT_URL = new PropertyDescriptor.Builder()
+ .name("Workday Report URL")
+ .displayName("Workday Report URL")
+ .description("HTTP remote URL of Workday report including a scheme of http or https, as well as a hostname or IP address with optional port and path elements.")
+ .required(true)
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .addValidator(URL_VALIDATOR)
+ .build();
+
+ protected static final PropertyDescriptor WORKDAY_USERNAME = new PropertyDescriptor.Builder()
+ .name("Workday Username")
+ .displayName("Workday Username")
+ .description("The username provided for authentication of Workday requests. Encoded using Base64 for HTTP Basic Authentication as described in RFC 7617.")
+ .required(true)
+ .addValidator(StandardValidators.createRegexMatchingValidator(Pattern.compile("^[\\x20-\\x39\\x3b-\\x7e\\x80-\\xff]+$")))
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .build();
+
+ protected static final PropertyDescriptor WORKDAY_PASSWORD = new PropertyDescriptor.Builder()
+ .name("Workday Password")
+ .displayName("Workday Password")
+ .description("The password provided for authentication of Workday requests. Encoded using Base64 for HTTP Basic Authentication as described in RFC 7617.")
+ .required(true)
+ .sensitive(true)
+ .addValidator(StandardValidators.createRegexMatchingValidator(Pattern.compile("^[\\x20-\\x7e\\x80-\\xff]+$")))
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .build();
+
+ protected static final PropertyDescriptor WEB_CLIENT_SERVICE = new PropertyDescriptor.Builder()
+ .name("Web Client Service Provider")
+ .description("Web client which is used to communicate with the Workday API.")
+ .required(true)
+ .identifiesControllerService(WebClientServiceProvider.class)
+ .build();
+
+ protected static final PropertyDescriptor FIELDS_TO_HASH = new PropertyDescriptor.Builder()
+ .name("Hashed Fields")
+ .displayName("Hashed Fields")
+ .description("Comma separated record paths to be replaced with their corresponding hash.")
+ .required(false)
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .addValidator(NON_BLANK_VALIDATOR)
+ .build();
+
+ protected static final PropertyDescriptor HASHING_ALGORITHM = new PropertyDescriptor.Builder()
+ .name("Hashing Algorithm")
+ .displayName("Hashing Algorithm")
+ .description("Determines what hashing algorithm should be used to perform the hashing function.")
+ .required(true)
+ .allowableValues(HashAlgorithm.getNames())
+ .defaultValue(HashAlgorithm.SHA256.getName())
+ .addValidator(NON_BLANK_VALIDATOR)
+ .expressionLanguageSupported(NONE)
+ .dependsOn(FIELDS_TO_HASH)
+ .build();
+
+ protected static final PropertyDescriptor RECORD_READER_FACTORY = new PropertyDescriptor.Builder()
+ .name("record-reader")
+ .displayName("Record Reader")
+ .description("Specifies the Controller Service to use for parsing incoming data and determining the data's schema.")
+ .identifiesControllerService(RecordReaderFactory.class)
+ .required(false)
+ .build();
+
+ protected static final PropertyDescriptor RECORD_WRITER_FACTORY = new PropertyDescriptor.Builder()
+ .name("record-writer")
+ .displayName("Record Writer")
+ .description("The Record Writer to use for serializing Records to an output FlowFile.")
+ .identifiesControllerService(RecordSetWriterFactory.class)
+ .dependsOn(RECORD_READER_FACTORY)
+ .required(true)
+ .build();
+
+ protected static final Relationship ORIGINAL = new Relationship.Builder()
+ .name("original")
+ .description("Request FlowFiles transferred when receiving HTTP responses with a status code between 200 and 299.")
+ .build();
+
+ protected static final Relationship FAILURE = new Relationship.Builder()
+ .name("failure")
+ .description("Request FlowFiles transferred when receiving socket communication errors.")
+ .build();
+
+ protected static final Relationship SUCCESS = new Relationship.Builder()
+ .name("success")
+ .description("Response FlowFiles transferred when receiving HTTP responses with a status code between 200 and 299.")
+ .build();
+
+ protected static final Set<Relationship> RELATIONSHIPS = Collections.unmodifiableSet(new HashSet<>(Arrays.asList(ORIGINAL, SUCCESS, FAILURE)));
+ protected static final List<PropertyDescriptor> PROPERTIES = Collections.unmodifiableList(Arrays.asList(
+ REPORT_URL,
+ WORKDAY_USERNAME,
+ WORKDAY_PASSWORD,
+ WEB_CLIENT_SERVICE,
+ FIELDS_TO_HASH,
+ HASHING_ALGORITHM,
+ RECORD_READER_FACTORY,
+ RECORD_WRITER_FACTORY
+ ));
+
+ private final AtomicReference<WebClientService> webClientReference = new AtomicReference<>();
+ private final AtomicReference<RecordReaderFactory> recordReaderFactoryReference = new AtomicReference<>();
+ private final AtomicReference<RecordSetWriterFactory> recordSetWriterFactoryReference = new AtomicReference<>();
+
+ @Override
+ protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
+ return PROPERTIES;
+ }
+
+ @Override
+ public Set<Relationship> getRelationships() {
+ return RELATIONSHIPS;
+ }
+
+ @OnScheduled
+ public void setUpClient(final ProcessContext context) {
+ WebClientServiceProvider standardWebClientServiceProvider = context.getProperty(WEB_CLIENT_SERVICE).asControllerService(WebClientServiceProvider.class);
+ RecordReaderFactory recordReaderFactory = context.getProperty(RECORD_READER_FACTORY).asControllerService(RecordReaderFactory.class);
+ RecordSetWriterFactory recordSetWriterFactory = context.getProperty(RECORD_WRITER_FACTORY).asControllerService(RecordSetWriterFactory.class);
+ WebClientService webClientService = standardWebClientServiceProvider.getWebClientService();
+ webClientReference.set(webClientService);
+ recordReaderFactoryReference.set(recordReaderFactory);
+ recordSetWriterFactoryReference.set(recordSetWriterFactory);
+ }
+
+ @Override
+ public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException {
+ FlowFile flowFile = session.get();
+
+ if (skipExecution(context, flowFile)) {
+ return;
+ }
+
+ FlowFile responseFlowFile = null;
+
+ try {
+ WebClientService webClientService = webClientReference.get();
+ URI uri = new URI(context.getProperty(REPORT_URL).evaluateAttributeExpressions(flowFile).getValue().trim());
+ long startNanos = System.nanoTime();
+ String authorization = createAuthorizationHeader(context, flowFile);
+
+ try(HttpResponseEntity httpResponseEntity = webClientService.get()
+ .uri(uri)
+ .header(HEADER_AUTHORIZATION, authorization)
+ .retrieve()) {
+ responseFlowFile = createResponseFlowFile(flowFile, session, context, httpResponseEntity);
+ long elapsedTime = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos);
+ Map<String, String> commonAttributes = createCommonAttributes(uri, httpResponseEntity, elapsedTime);
+
+ if (flowFile != null) {
+ flowFile = session.putAllAttributes(flowFile, setMimeType(commonAttributes, httpResponseEntity));
+ }
+ if (responseFlowFile != null) {
+ responseFlowFile = session.putAllAttributes(responseFlowFile, commonAttributes);
+ if (flowFile == null) {
+ session.getProvenanceReporter().receive(responseFlowFile, uri.toString(), elapsedTime);
+ } else {
+ session.getProvenanceReporter().fetch(responseFlowFile, uri.toString(), elapsedTime);
+ }
+ }
+
+ route(flowFile, responseFlowFile, session, context, httpResponseEntity.statusCode());
+ }
+ } catch (Exception e) {
+ if (flowFile == null) {
+ getLogger().error("Request Processing failed", e);
+ context.yield();
+ } else {
+ getLogger().error("Request Processing failed: {}", flowFile, e);
+ session.penalize(flowFile);
+ flowFile = session.putAttribute(flowFile, GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS, e.getClass().getSimpleName());
+ flowFile = session.putAttribute(flowFile, GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE, e.getMessage());
+ session.transfer(flowFile, FAILURE);
+ }
+
+ if (responseFlowFile != null) {
+ session.remove(responseFlowFile);
+ }
+ }
+ }
+
+ @Override
+ protected Collection<ValidationResult> customValidate(ValidationContext validationContext) {
+ List<ValidationResult> results = new ArrayList<>(super.customValidate(validationContext));
+ if (validationContext.getProperty(FIELDS_TO_HASH).isSet() && !validationContext.getProperty(RECORD_READER_FACTORY).isSet()) {
+ results.add(new ValidationResult.Builder()
+ .valid(false)
+ .explanation("Record-Reader and Record-Writer must be set if you would like to hash specific fields")
+ .subject("Workday report configuration")
+ .build());
+ }
+ return results;
+ }
+
+ /*
+ * If we have no FlowFile, and all incoming connections are self-loops then we can continue on.
+ * However, if we have no FlowFile and we have connections coming from other Processors, then
+ * we know that we should run only if we have a FlowFile.
+ */
+ private boolean skipExecution(ProcessContext context, FlowFile flowfile) {
+ return context.hasIncomingConnection() && flowfile == null && context.hasNonLoopConnection();
+ }
+
+ private FlowFile createResponseFlowFile(FlowFile flowfile, ProcessSession session, ProcessContext context, HttpResponseEntity httpResponseEntity)
+ throws IOException, SchemaNotFoundException, MalformedRecordException {
+ FlowFile responseFlowFile = null;
+ String hashingAlgorithm = context.getProperty(HASHING_ALGORITHM).getValue();
+ Set<String> columnsToHash = Optional.ofNullable(
+ context.getProperty(FIELDS_TO_HASH)
+ .evaluateAttributeExpressions(flowfile)
+ .getValue())
+ .map(String::trim)
+ .map(columns -> columns.split(COLUMNS_TO_HASH_SEPARATOR))
+ .map(Arrays::stream)
+ .map(columns -> columns.collect(Collectors.toSet()))
+ .orElse(Collections.emptySet());
+ try {
+ if (isSuccess(httpResponseEntity.statusCode())) {
+ responseFlowFile = flowfile == null ? session.create() : session.create(flowfile);
+ InputStream responseBodyStream = httpResponseEntity.body();
+ if (recordReaderFactoryReference.get() != null) {
+ TransformResult transformResult = transformRecords(session, flowfile, responseFlowFile, hashingAlgorithm, columnsToHash, responseBodyStream);
+ Map<String, String> attributes = new HashMap<>();
+ attributes.put(RECORD_COUNT, String.valueOf(transformResult.getNumberOfRecords()));
+ attributes.put(CoreAttributes.MIME_TYPE.key(), transformResult.getMimeType());
+ responseFlowFile = session.putAllAttributes(responseFlowFile, attributes);
+ } else {
+ responseFlowFile = session.importFrom(responseBodyStream, responseFlowFile);
+ Optional<String> mimeType = httpResponseEntity.headers().getFirstHeader(HEADER_CONTENT_TYPE);
+ if (mimeType.isPresent()) {
+ responseFlowFile = session.putAttribute(responseFlowFile, CoreAttributes.MIME_TYPE.key(), mimeType.get());
+ }
+ }
+ }
+ } catch (Exception e) {
+ session.remove(responseFlowFile);
+ throw e;
+ }
+ return responseFlowFile;
+ }
+
+ private String createAuthorizationHeader(ProcessContext context, FlowFile flowfile) {
+ String userName = context.getProperty(WORKDAY_USERNAME).evaluateAttributeExpressions(flowfile).getValue();
+ String password = context.getProperty(WORKDAY_PASSWORD).evaluateAttributeExpressions(flowfile).getValue();
+ String base64Credential = Base64.getEncoder().encodeToString((userName + USERNAME_PASSWORD_SEPARATOR + password).getBytes(StandardCharsets.UTF_8));
+ return BASIC_PREFIX + base64Credential;
+ }
+
+ private TransformResult transformRecords(ProcessSession session, FlowFile flowfile, FlowFile responseFlowFile, String hashingAlgorithm, Set<String> columnsToHash,
+ InputStream responseBodyStream) throws IOException, SchemaNotFoundException, MalformedRecordException {
+ int numberOfRecords = 0;
+ String mimeType = null;
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@nifi.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [nifi] ferencerdei commented on a diff in pull request #6376: NIFI-10455 Get workday report processor
Posted by GitBox <gi...@apache.org>.
ferencerdei commented on code in PR #6376:
URL: https://github.com/apache/nifi/pull/6376#discussion_r968134509
##########
nifi-nar-bundles/nifi-workday-bundle/nifi-workday-processors/src/main/java/org/apache/nifi/processors/workday/GetWorkdayReport.java:
##########
@@ -0,0 +1,432 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.processors.workday;
+
+import static org.apache.nifi.expression.ExpressionLanguageScope.FLOWFILE_ATTRIBUTES;
+import static org.apache.nifi.expression.ExpressionLanguageScope.NONE;
+import static org.apache.nifi.processor.util.StandardValidators.NON_BLANK_VALIDATOR;
+import static org.apache.nifi.processor.util.StandardValidators.URL_VALIDATOR;
+
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Base64;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import org.apache.nifi.annotation.behavior.EventDriven;
+import org.apache.nifi.annotation.behavior.InputRequirement;
+import org.apache.nifi.annotation.behavior.InputRequirement.Requirement;
+import org.apache.nifi.annotation.behavior.SideEffectFree;
+import org.apache.nifi.annotation.behavior.SupportsBatching;
+import org.apache.nifi.annotation.behavior.WritesAttribute;
+import org.apache.nifi.annotation.behavior.WritesAttributes;
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.annotation.lifecycle.OnScheduled;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.ValidationContext;
+import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.flowfile.attributes.CoreAttributes;
+import org.apache.nifi.logging.ComponentLog;
+import org.apache.nifi.processor.AbstractProcessor;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.processor.util.StandardValidators;
+import org.apache.nifi.record.path.RecordPath;
+import org.apache.nifi.record.path.RecordPathResult;
+import org.apache.nifi.schema.access.SchemaNotFoundException;
+import org.apache.nifi.security.util.crypto.HashAlgorithm;
+import org.apache.nifi.security.util.crypto.HashService;
+import org.apache.nifi.serialization.MalformedRecordException;
+import org.apache.nifi.serialization.RecordReader;
+import org.apache.nifi.serialization.RecordReaderFactory;
+import org.apache.nifi.serialization.RecordSetWriter;
+import org.apache.nifi.serialization.RecordSetWriterFactory;
+import org.apache.nifi.serialization.record.Record;
+import org.apache.nifi.serialization.record.RecordFieldType;
+import org.apache.nifi.serialization.record.RecordSchema;
+import org.apache.nifi.web.client.api.HttpResponseEntity;
+import org.apache.nifi.web.client.api.WebClientService;
+import org.apache.nifi.web.client.provider.api.WebClientServiceProvider;
+
+@Tags({"Workday", "report", "get"})
+@InputRequirement(Requirement.INPUT_ALLOWED)
+@CapabilityDescription("A processor which can interact with a configurable Workday Report. The processor can forward the content without modification, or you can transform it by"
+ + " providing the specific Record Reader and Record Writer services based on your needs. You can also hash, or remove fields using the input parameters and schemes in the Record writer. "
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@nifi.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [nifi] ferencerdei commented on a diff in pull request #6376: NIFI-10455 Get workday report processor
Posted by GitBox <gi...@apache.org>.
ferencerdei commented on code in PR #6376:
URL: https://github.com/apache/nifi/pull/6376#discussion_r971712935
##########
nifi-nar-bundles/nifi-workday-bundle/nifi-workday-processors/src/main/java/org/apache/nifi/processors/workday/GetWorkdayReport.java:
##########
@@ -0,0 +1,466 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.processors.workday;
+
+import static org.apache.nifi.expression.ExpressionLanguageScope.FLOWFILE_ATTRIBUTES;
+import static org.apache.nifi.expression.ExpressionLanguageScope.NONE;
+import static org.apache.nifi.processor.util.StandardValidators.NON_BLANK_VALIDATOR;
+import static org.apache.nifi.processor.util.StandardValidators.URL_VALIDATOR;
+
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Base64;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import org.apache.nifi.annotation.behavior.EventDriven;
+import org.apache.nifi.annotation.behavior.InputRequirement;
+import org.apache.nifi.annotation.behavior.InputRequirement.Requirement;
+import org.apache.nifi.annotation.behavior.SideEffectFree;
+import org.apache.nifi.annotation.behavior.SupportsBatching;
+import org.apache.nifi.annotation.behavior.WritesAttribute;
+import org.apache.nifi.annotation.behavior.WritesAttributes;
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.annotation.lifecycle.OnScheduled;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.ValidationContext;
+import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.flowfile.attributes.CoreAttributes;
+import org.apache.nifi.processor.AbstractProcessor;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.processor.util.StandardValidators;
+import org.apache.nifi.record.path.RecordPath;
+import org.apache.nifi.record.path.RecordPathResult;
+import org.apache.nifi.schema.access.SchemaNotFoundException;
+import org.apache.nifi.serialization.MalformedRecordException;
+import org.apache.nifi.serialization.RecordReader;
+import org.apache.nifi.serialization.RecordReaderFactory;
+import org.apache.nifi.serialization.RecordSetWriter;
+import org.apache.nifi.serialization.RecordSetWriterFactory;
+import org.apache.nifi.serialization.record.Record;
+import org.apache.nifi.serialization.record.RecordFieldType;
+import org.apache.nifi.serialization.record.RecordSchema;
+import org.apache.nifi.web.client.api.HttpResponseEntity;
+import org.apache.nifi.web.client.api.WebClientService;
+import org.apache.nifi.web.client.provider.api.WebClientServiceProvider;
+
+@Tags({"Workday", "report"})
+@InputRequirement(Requirement.INPUT_ALLOWED)
+@CapabilityDescription("A processor which can interact with a configurable Workday Report. The processor can forward the content without modification, or you can transform it by"
+ + " providing the specific Record Reader and Record Writer services based on your needs. You can also hash, or remove fields using the input parameters and schemes in the Record Writer. "
+ + "Supported Workday report formats are: csv, simplexml, json")
+@EventDriven
+@SideEffectFree
+@SupportsBatching
+@WritesAttributes({
+ @WritesAttribute(attribute = GetWorkdayReport.GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS, description = "The Java exception class raised when the processor fails"),
+ @WritesAttribute(attribute = GetWorkdayReport.GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE, description = "The Java exception message raised when the processor fails"),
+ @WritesAttribute(attribute = "mime.type", description = "Sets the mime.type attribute to the MIME Type specified by the Source / Record Writer"),
+ @WritesAttribute(attribute= GetWorkdayReport.RECORD_COUNT, description = "The number of records in an outgoing FlowFile. This is only populated on the 'success' relationship "
+ + "when Record Reader and Writer is set.")})
+public class GetWorkdayReport extends AbstractProcessor {
+
+ protected static final String STATUS_CODE = "getworkdayreport.status.code";
+ protected static final String REQUEST_URL = "getworkdayreport.request.url";
+ protected static final String REQUEST_DURATION = "getworkdayreport.request.duration";
+ protected static final String TRANSACTION_ID = "getworkdayreport.tx.id";
+ protected static final String GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS = "getworkdayreport.java.exception.class";
+ protected static final String GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE = "getworkdayreport.java.exception.message";
+ protected static final String RECORD_COUNT = "record.count";
+ protected static final String BASIC_PREFIX = "Basic ";
+ protected static final String COLUMNS_TO_HASH_SEPARATOR = ",";
+ protected static final String HEADER_AUTHORIZATION = "Authorization";
+ protected static final String HEADER_CONTENT_TYPE = "Content-Type";
+ protected static final String USERNAME_PASSWORD_SEPARATOR = ":";
+
+ protected static final PropertyDescriptor REPORT_URL = new PropertyDescriptor.Builder()
+ .name("Workday Report URL")
+ .displayName("Workday Report URL")
+ .description("HTTP remote URL of Workday report including a scheme of http or https, as well as a hostname or IP address with optional port and path elements.")
+ .required(true)
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .addValidator(URL_VALIDATOR)
+ .build();
+
+ protected static final PropertyDescriptor WORKDAY_USERNAME = new PropertyDescriptor.Builder()
+ .name("Workday Username")
+ .displayName("Workday Username")
+ .description("The username provided for authentication of Workday requests. Encoded using Base64 for HTTP Basic Authentication as described in RFC 7617.")
+ .required(true)
+ .addValidator(StandardValidators.createRegexMatchingValidator(Pattern.compile("^[\\x20-\\x39\\x3b-\\x7e\\x80-\\xff]+$")))
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .build();
+
+ protected static final PropertyDescriptor WORKDAY_PASSWORD = new PropertyDescriptor.Builder()
+ .name("Workday Password")
+ .displayName("Workday Password")
+ .description("The password provided for authentication of Workday requests. Encoded using Base64 for HTTP Basic Authentication as described in RFC 7617.")
+ .required(true)
+ .sensitive(true)
+ .addValidator(StandardValidators.createRegexMatchingValidator(Pattern.compile("^[\\x20-\\x7e\\x80-\\xff]+$")))
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .build();
+
+ protected static final PropertyDescriptor WEB_CLIENT_SERVICE = new PropertyDescriptor.Builder()
+ .name("Web Client Service Provider")
+ .description("Web client which is used to communicate with the Workday API.")
+ .required(true)
+ .identifiesControllerService(WebClientServiceProvider.class)
+ .build();
+
+ protected static final PropertyDescriptor FIELDS_TO_HASH = new PropertyDescriptor.Builder()
+ .name("Hashed Fields")
+ .displayName("Hashed Fields")
+ .description("Comma separated record paths to be replaced with their corresponding hash.")
+ .required(false)
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .addValidator(NON_BLANK_VALIDATOR)
+ .build();
+
+ protected static final PropertyDescriptor HASHING_ALGORITHM = new PropertyDescriptor.Builder()
+ .name("Hashing Algorithm")
+ .displayName("Hashing Algorithm")
+ .description("Determines what hashing algorithm should be used to perform the hashing function.")
+ .required(true)
+ .allowableValues(HashAlgorithm.getNames())
+ .defaultValue(HashAlgorithm.SHA256.getName())
+ .addValidator(NON_BLANK_VALIDATOR)
+ .expressionLanguageSupported(NONE)
+ .dependsOn(FIELDS_TO_HASH)
+ .build();
+
+ protected static final PropertyDescriptor RECORD_READER_FACTORY = new PropertyDescriptor.Builder()
+ .name("record-reader")
+ .displayName("Record Reader")
+ .description("Specifies the Controller Service to use for parsing incoming data and determining the data's schema.")
+ .identifiesControllerService(RecordReaderFactory.class)
+ .required(false)
+ .build();
+
+ protected static final PropertyDescriptor RECORD_WRITER_FACTORY = new PropertyDescriptor.Builder()
+ .name("record-writer")
+ .displayName("Record Writer")
+ .description("The Record Writer to use for serializing Records to an output FlowFile.")
+ .identifiesControllerService(RecordSetWriterFactory.class)
+ .dependsOn(RECORD_READER_FACTORY)
+ .required(true)
+ .build();
+
+ protected static final Relationship ORIGINAL = new Relationship.Builder()
+ .name("original")
+ .description("Request FlowFiles transferred when receiving HTTP responses with a status code between 200 and 299.")
+ .build();
+
+ protected static final Relationship FAILURE = new Relationship.Builder()
+ .name("failure")
+ .description("Request FlowFiles transferred when receiving socket communication errors.")
+ .build();
+
+ protected static final Relationship SUCCESS = new Relationship.Builder()
+ .name("success")
+ .description("Response FlowFiles transferred when receiving HTTP responses with a status code between 200 and 299.")
+ .build();
+
+ protected static final Set<Relationship> RELATIONSHIPS = Collections.unmodifiableSet(new HashSet<>(Arrays.asList(ORIGINAL, SUCCESS, FAILURE)));
+ protected static final List<PropertyDescriptor> PROPERTIES = Collections.unmodifiableList(Arrays.asList(
+ REPORT_URL,
+ WORKDAY_USERNAME,
+ WORKDAY_PASSWORD,
+ WEB_CLIENT_SERVICE,
+ FIELDS_TO_HASH,
+ HASHING_ALGORITHM,
+ RECORD_READER_FACTORY,
+ RECORD_WRITER_FACTORY
+ ));
+
+ private final AtomicReference<WebClientService> webClientReference = new AtomicReference<>();
+ private final AtomicReference<RecordReaderFactory> recordReaderFactoryReference = new AtomicReference<>();
+ private final AtomicReference<RecordSetWriterFactory> recordSetWriterFactoryReference = new AtomicReference<>();
+
+ @Override
+ protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
+ return PROPERTIES;
+ }
+
+ @Override
+ public Set<Relationship> getRelationships() {
+ return RELATIONSHIPS;
+ }
+
+ @OnScheduled
+ public void setUpClient(final ProcessContext context) {
+ WebClientServiceProvider standardWebClientServiceProvider = context.getProperty(WEB_CLIENT_SERVICE).asControllerService(WebClientServiceProvider.class);
+ RecordReaderFactory recordReaderFactory = context.getProperty(RECORD_READER_FACTORY).asControllerService(RecordReaderFactory.class);
+ RecordSetWriterFactory recordSetWriterFactory = context.getProperty(RECORD_WRITER_FACTORY).asControllerService(RecordSetWriterFactory.class);
+ WebClientService webClientService = standardWebClientServiceProvider.getWebClientService();
+ webClientReference.set(webClientService);
+ recordReaderFactoryReference.set(recordReaderFactory);
+ recordSetWriterFactoryReference.set(recordSetWriterFactory);
+ }
+
+ @Override
+ public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException {
+ FlowFile flowFile = session.get();
+
+ if (skipExecution(context, flowFile)) {
+ return;
+ }
+
+ FlowFile responseFlowFile = null;
+
+ try {
+ WebClientService webClientService = webClientReference.get();
+ URI uri = new URI(context.getProperty(REPORT_URL).evaluateAttributeExpressions(flowFile).getValue().trim());
+ long startNanos = System.nanoTime();
+ String authorization = createAuthorizationHeader(context, flowFile);
+
+ try(HttpResponseEntity httpResponseEntity = webClientService.get()
+ .uri(uri)
+ .header(HEADER_AUTHORIZATION, authorization)
+ .retrieve()) {
+ responseFlowFile = createResponseFlowFile(flowFile, session, context, httpResponseEntity);
+ long elapsedTime = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos);
+ Map<String, String> commonAttributes = createCommonAttributes(uri, httpResponseEntity, elapsedTime);
+
+ if (flowFile != null) {
+ flowFile = session.putAllAttributes(flowFile, setMimeType(commonAttributes, httpResponseEntity));
+ }
+ if (responseFlowFile != null) {
+ responseFlowFile = session.putAllAttributes(responseFlowFile, commonAttributes);
+ if (flowFile == null) {
+ session.getProvenanceReporter().receive(responseFlowFile, uri.toString(), elapsedTime);
+ } else {
+ session.getProvenanceReporter().fetch(responseFlowFile, uri.toString(), elapsedTime);
+ }
+ }
+
+ route(flowFile, responseFlowFile, session, context, httpResponseEntity.statusCode());
+ }
+ } catch (Exception e) {
+ if (flowFile == null) {
+ getLogger().error("Request Processing failed", e);
+ context.yield();
+ } else {
+ getLogger().error("Request Processing failed: {}", flowFile, e);
+ session.penalize(flowFile);
+ flowFile = session.putAttribute(flowFile, GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS, e.getClass().getSimpleName());
+ flowFile = session.putAttribute(flowFile, GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE, e.getMessage());
+ session.transfer(flowFile, FAILURE);
+ }
+
+ if (responseFlowFile != null) {
+ session.remove(responseFlowFile);
+ }
+ }
+ }
+
+ @Override
+ protected Collection<ValidationResult> customValidate(ValidationContext validationContext) {
+ List<ValidationResult> results = new ArrayList<>(super.customValidate(validationContext));
+ if (validationContext.getProperty(FIELDS_TO_HASH).isSet() && !validationContext.getProperty(RECORD_READER_FACTORY).isSet()) {
+ results.add(new ValidationResult.Builder()
+ .valid(false)
+ .explanation("Record-Reader and Record-Writer must be set if you would like to hash specific fields")
+ .subject("Workday report configuration")
+ .build());
+ }
+ return results;
+ }
+
+ /*
+ * If we have no FlowFile, and all incoming connections are self-loops then we can continue on.
+ * However, if we have no FlowFile and we have connections coming from other Processors, then
+ * we know that we should run only if we have a FlowFile.
+ */
+ private boolean skipExecution(ProcessContext context, FlowFile flowfile) {
+ return context.hasIncomingConnection() && flowfile == null && context.hasNonLoopConnection();
+ }
+
+ private FlowFile createResponseFlowFile(FlowFile flowfile, ProcessSession session, ProcessContext context, HttpResponseEntity httpResponseEntity)
+ throws IOException, SchemaNotFoundException, MalformedRecordException {
+ FlowFile responseFlowFile = null;
+ String hashingAlgorithm = context.getProperty(HASHING_ALGORITHM).getValue();
+ Set<String> columnsToHash = Optional.ofNullable(
+ context.getProperty(FIELDS_TO_HASH)
+ .evaluateAttributeExpressions(flowfile)
+ .getValue())
+ .map(String::trim)
+ .map(columns -> columns.split(COLUMNS_TO_HASH_SEPARATOR))
+ .map(Arrays::stream)
+ .map(columns -> columns.collect(Collectors.toSet()))
+ .orElse(Collections.emptySet());
+ try {
+ if (isSuccess(httpResponseEntity.statusCode())) {
+ responseFlowFile = flowfile == null ? session.create() : session.create(flowfile);
+ InputStream responseBodyStream = httpResponseEntity.body();
+ if (recordReaderFactoryReference.get() != null) {
+ TransformResult transformResult = transformRecords(session, flowfile, responseFlowFile, hashingAlgorithm, columnsToHash, responseBodyStream);
+ Map<String, String> attributes = new HashMap<>();
+ attributes.put(RECORD_COUNT, String.valueOf(transformResult.getNumberOfRecords()));
+ attributes.put(CoreAttributes.MIME_TYPE.key(), transformResult.getMimeType());
+ responseFlowFile = session.putAllAttributes(responseFlowFile, attributes);
+ } else {
+ responseFlowFile = session.importFrom(responseBodyStream, responseFlowFile);
+ Optional<String> mimeType = httpResponseEntity.headers().getFirstHeader(HEADER_CONTENT_TYPE);
+ if (mimeType.isPresent()) {
+ responseFlowFile = session.putAttribute(responseFlowFile, CoreAttributes.MIME_TYPE.key(), mimeType.get());
+ }
+ }
+ }
+ } catch (Exception e) {
+ session.remove(responseFlowFile);
+ throw e;
+ }
+ return responseFlowFile;
+ }
+
+ private String createAuthorizationHeader(ProcessContext context, FlowFile flowfile) {
+ String userName = context.getProperty(WORKDAY_USERNAME).evaluateAttributeExpressions(flowfile).getValue();
+ String password = context.getProperty(WORKDAY_PASSWORD).evaluateAttributeExpressions(flowfile).getValue();
+ String base64Credential = Base64.getEncoder().encodeToString((userName + USERNAME_PASSWORD_SEPARATOR + password).getBytes(StandardCharsets.UTF_8));
+ return BASIC_PREFIX + base64Credential;
+ }
+
+ private TransformResult transformRecords(ProcessSession session, FlowFile flowfile, FlowFile responseFlowFile, String hashingAlgorithm, Set<String> columnsToHash,
+ InputStream responseBodyStream) throws IOException, SchemaNotFoundException, MalformedRecordException {
+ int numberOfRecords = 0;
+ String mimeType = null;
+ try (RecordReader reader = recordReaderFactoryReference.get().createRecordReader(flowfile,
+ new BufferedInputStream(responseBodyStream), getLogger())) {
+ RecordSchema schema = recordSetWriterFactoryReference.get()
+ .getSchema(flowfile == null ? Collections.emptyMap() : flowfile.getAttributes(), reader.getSchema());
+ try (OutputStream responseStream = session.write(responseFlowFile);
+ RecordSetWriter recordSetWriter = recordSetWriterFactoryReference.get().createWriter(getLogger(), schema, responseStream, responseFlowFile)) {
+ mimeType = recordSetWriter.getMimeType();
+ recordSetWriter.beginRecordSet();
+ Record currentRecord;
+ // as the report can be changed independently from the flow, it's safer to ignore field types and unknown fields in the Record Reading process
+ while ((currentRecord = reader.nextRecord(false, true)) != null) {
+ for (String recordPath : columnsToHash) {
+ RecordPathResult evaluate = RecordPath.compile("hash(" + recordPath + ", '" + hashingAlgorithm + "')").evaluate(currentRecord);
+ evaluate.getSelectedFields().forEach(fieldVal -> fieldVal.updateValue(fieldVal.getValue(), RecordFieldType.STRING.getDataType()));
+ }
Review Comment:
This is an optional functionality so I think it should be acceptable. Would it help if I create a Map with the compiled paths?
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@nifi.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [nifi] ferencerdei commented on a diff in pull request #6376: NIFI-10455 Get workday report processor
Posted by GitBox <gi...@apache.org>.
ferencerdei commented on code in PR #6376:
URL: https://github.com/apache/nifi/pull/6376#discussion_r968131712
##########
nifi-assembly/pom.xml:
##########
@@ -227,6 +227,12 @@ language governing permissions and limitations under the License. -->
<version>1.18.0-SNAPSHOT</version>
<type>nar</type>
</dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-web-client-provider-service-nar</artifactId>
+ <version>1.18.0-SNAPSHOT</version>
+ <type>nar</type>
+ </dependency>
Review Comment:
removed
##########
nifi-nar-bundles/nifi-workday-bundle/nifi-workday-processors/src/main/java/org/apache/nifi/processors/workday/GetWorkdayReport.java:
##########
@@ -0,0 +1,432 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.processors.workday;
+
+import static org.apache.nifi.expression.ExpressionLanguageScope.FLOWFILE_ATTRIBUTES;
+import static org.apache.nifi.expression.ExpressionLanguageScope.NONE;
+import static org.apache.nifi.processor.util.StandardValidators.NON_BLANK_VALIDATOR;
+import static org.apache.nifi.processor.util.StandardValidators.URL_VALIDATOR;
+
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Base64;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import org.apache.nifi.annotation.behavior.EventDriven;
+import org.apache.nifi.annotation.behavior.InputRequirement;
+import org.apache.nifi.annotation.behavior.InputRequirement.Requirement;
+import org.apache.nifi.annotation.behavior.SideEffectFree;
+import org.apache.nifi.annotation.behavior.SupportsBatching;
+import org.apache.nifi.annotation.behavior.WritesAttribute;
+import org.apache.nifi.annotation.behavior.WritesAttributes;
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.annotation.lifecycle.OnScheduled;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.ValidationContext;
+import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.flowfile.attributes.CoreAttributes;
+import org.apache.nifi.logging.ComponentLog;
+import org.apache.nifi.processor.AbstractProcessor;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.processor.util.StandardValidators;
+import org.apache.nifi.record.path.RecordPath;
+import org.apache.nifi.record.path.RecordPathResult;
+import org.apache.nifi.schema.access.SchemaNotFoundException;
+import org.apache.nifi.security.util.crypto.HashAlgorithm;
+import org.apache.nifi.security.util.crypto.HashService;
+import org.apache.nifi.serialization.MalformedRecordException;
+import org.apache.nifi.serialization.RecordReader;
+import org.apache.nifi.serialization.RecordReaderFactory;
+import org.apache.nifi.serialization.RecordSetWriter;
+import org.apache.nifi.serialization.RecordSetWriterFactory;
+import org.apache.nifi.serialization.record.Record;
+import org.apache.nifi.serialization.record.RecordFieldType;
+import org.apache.nifi.serialization.record.RecordSchema;
+import org.apache.nifi.web.client.api.HttpResponseEntity;
+import org.apache.nifi.web.client.api.WebClientService;
+import org.apache.nifi.web.client.provider.api.WebClientServiceProvider;
+
+@Tags({"Workday", "report", "get"})
+@InputRequirement(Requirement.INPUT_ALLOWED)
+@CapabilityDescription("A processor which can interact with a configurable Workday Report. The processor can forward the content without modification, or you can transform it by"
+ + " providing the specific Record Reader and Record Writer services based on your needs. You can also hash, or remove fields using the input parameters and schemes in the Record writer. "
+ + "Supported Workday report formats are: csv, simplexml, json")
+@EventDriven
+@SideEffectFree
+@SupportsBatching
+@WritesAttributes({
+ @WritesAttribute(attribute = GetWorkdayReport.GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS, description = "The Java exception class raised when the processor fails"),
+ @WritesAttribute(attribute = GetWorkdayReport.GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE, description = "The Java exception message raised when the processor fails"),
+ @WritesAttribute(attribute = "mime.type", description = "Sets the mime.type attribute to the MIME Type specified by the Source / Record Writer"),
+ @WritesAttribute(attribute= GetWorkdayReport.RECORD_COUNT, description = "The number of records in an outgoing FlowFile. This is only populated on the 'success' relationship "
+ + "when Record Reader and Writer is set.")})
+public class GetWorkdayReport extends AbstractProcessor {
+
+ protected static final String STATUS_CODE = "getworkdayreport.status.code";
+ protected static final String REQUEST_URL = "getworkdayreport.request.url";
+ protected static final String REQUEST_DURATION = "getworkdayreport.request.duration";
+ protected static final String TRANSACTION_ID = "getworkdayreport.tx.id";
+ protected static final String GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS = "getworkdayreport.java.exception.class";
+ protected static final String GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE = "getworkdayreport.java.exception.message";
+ protected static final String RECORD_COUNT = "record.count";
+ protected static final String BASIC_PREFIX = "Basic ";
+ protected static final String COLUMNS_TO_HASH_SEPARATOR = ",";
+ protected static final String HEADER_AUTHORIZATION = "Authorization";
+ protected static final String HEADER_CONTENT_TYPE = "Content-Type";
+ protected static final String USERNAME_PASSWORD_SEPARATOR = ":";
+
+ protected static final PropertyDescriptor REPORT_URL = new PropertyDescriptor.Builder()
+ .name("Workday report URL")
+ .displayName("Workday report URL")
Review Comment:
done
##########
nifi-nar-bundles/nifi-workday-bundle/nifi-workday-processors/src/main/java/org/apache/nifi/processors/workday/GetWorkdayReport.java:
##########
@@ -0,0 +1,432 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.processors.workday;
+
+import static org.apache.nifi.expression.ExpressionLanguageScope.FLOWFILE_ATTRIBUTES;
+import static org.apache.nifi.expression.ExpressionLanguageScope.NONE;
+import static org.apache.nifi.processor.util.StandardValidators.NON_BLANK_VALIDATOR;
+import static org.apache.nifi.processor.util.StandardValidators.URL_VALIDATOR;
+
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Base64;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import org.apache.nifi.annotation.behavior.EventDriven;
+import org.apache.nifi.annotation.behavior.InputRequirement;
+import org.apache.nifi.annotation.behavior.InputRequirement.Requirement;
+import org.apache.nifi.annotation.behavior.SideEffectFree;
+import org.apache.nifi.annotation.behavior.SupportsBatching;
+import org.apache.nifi.annotation.behavior.WritesAttribute;
+import org.apache.nifi.annotation.behavior.WritesAttributes;
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.annotation.lifecycle.OnScheduled;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.ValidationContext;
+import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.flowfile.attributes.CoreAttributes;
+import org.apache.nifi.logging.ComponentLog;
+import org.apache.nifi.processor.AbstractProcessor;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.processor.util.StandardValidators;
+import org.apache.nifi.record.path.RecordPath;
+import org.apache.nifi.record.path.RecordPathResult;
+import org.apache.nifi.schema.access.SchemaNotFoundException;
+import org.apache.nifi.security.util.crypto.HashAlgorithm;
+import org.apache.nifi.security.util.crypto.HashService;
+import org.apache.nifi.serialization.MalformedRecordException;
+import org.apache.nifi.serialization.RecordReader;
+import org.apache.nifi.serialization.RecordReaderFactory;
+import org.apache.nifi.serialization.RecordSetWriter;
+import org.apache.nifi.serialization.RecordSetWriterFactory;
+import org.apache.nifi.serialization.record.Record;
+import org.apache.nifi.serialization.record.RecordFieldType;
+import org.apache.nifi.serialization.record.RecordSchema;
+import org.apache.nifi.web.client.api.HttpResponseEntity;
+import org.apache.nifi.web.client.api.WebClientService;
+import org.apache.nifi.web.client.provider.api.WebClientServiceProvider;
+
+@Tags({"Workday", "report", "get"})
+@InputRequirement(Requirement.INPUT_ALLOWED)
+@CapabilityDescription("A processor which can interact with a configurable Workday Report. The processor can forward the content without modification, or you can transform it by"
+ + " providing the specific Record Reader and Record Writer services based on your needs. You can also hash, or remove fields using the input parameters and schemes in the Record writer. "
+ + "Supported Workday report formats are: csv, simplexml, json")
+@EventDriven
+@SideEffectFree
+@SupportsBatching
+@WritesAttributes({
+ @WritesAttribute(attribute = GetWorkdayReport.GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS, description = "The Java exception class raised when the processor fails"),
+ @WritesAttribute(attribute = GetWorkdayReport.GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE, description = "The Java exception message raised when the processor fails"),
+ @WritesAttribute(attribute = "mime.type", description = "Sets the mime.type attribute to the MIME Type specified by the Source / Record Writer"),
+ @WritesAttribute(attribute= GetWorkdayReport.RECORD_COUNT, description = "The number of records in an outgoing FlowFile. This is only populated on the 'success' relationship "
+ + "when Record Reader and Writer is set.")})
+public class GetWorkdayReport extends AbstractProcessor {
+
+ protected static final String STATUS_CODE = "getworkdayreport.status.code";
+ protected static final String REQUEST_URL = "getworkdayreport.request.url";
+ protected static final String REQUEST_DURATION = "getworkdayreport.request.duration";
+ protected static final String TRANSACTION_ID = "getworkdayreport.tx.id";
+ protected static final String GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS = "getworkdayreport.java.exception.class";
+ protected static final String GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE = "getworkdayreport.java.exception.message";
+ protected static final String RECORD_COUNT = "record.count";
+ protected static final String BASIC_PREFIX = "Basic ";
+ protected static final String COLUMNS_TO_HASH_SEPARATOR = ",";
+ protected static final String HEADER_AUTHORIZATION = "Authorization";
+ protected static final String HEADER_CONTENT_TYPE = "Content-Type";
+ protected static final String USERNAME_PASSWORD_SEPARATOR = ":";
+
+ protected static final PropertyDescriptor REPORT_URL = new PropertyDescriptor.Builder()
+ .name("Workday report URL")
+ .displayName("Workday report URL")
+ .description("HTTP remote URL of Workday report including a scheme of http or https, as well as a hostname or IP address with optional port and path elements.")
+ .required(true)
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .addValidator(URL_VALIDATOR)
+ .build();
+
+ protected static final PropertyDescriptor WORKDAY_USERNAME = new PropertyDescriptor.Builder()
+ .name("Workday Username")
+ .displayName("Workday Username")
+ .description("The username provided for authentication of Workday requests. Encoded using Base64 for HTTP Basic Authentication as described in RFC 7617.")
+ .required(true)
+ .addValidator(StandardValidators.createRegexMatchingValidator(Pattern.compile("^[\\x20-\\x39\\x3b-\\x7e\\x80-\\xff]+$")))
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .build();
+
+ protected static final PropertyDescriptor WORKDAY_PASSWORD = new PropertyDescriptor.Builder()
+ .name("Workday Password")
+ .displayName("Workday Password")
+ .description("The password provided for authentication of Workday requests. Encoded using Base64 for HTTP Basic Authentication as described in RFC 7617.")
+ .required(true)
+ .sensitive(true)
+ .addValidator(StandardValidators.createRegexMatchingValidator(Pattern.compile("^[\\x20-\\x7e\\x80-\\xff]+$")))
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .build();
+
+ protected static final PropertyDescriptor WEB_CLIENT_SERVICE = new PropertyDescriptor.Builder()
+ .name("Standard Web Client Service")
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@nifi.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [nifi] ferencerdei commented on a diff in pull request #6376: NIFI-10455 Get workday report processor
Posted by GitBox <gi...@apache.org>.
ferencerdei commented on code in PR #6376:
URL: https://github.com/apache/nifi/pull/6376#discussion_r969400859
##########
nifi-nar-bundles/nifi-workday-bundle/nifi-workday-processors/src/main/java/org/apache/nifi/processors/workday/GetWorkdayReport.java:
##########
@@ -0,0 +1,442 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.processors.workday;
+
+import static org.apache.nifi.expression.ExpressionLanguageScope.FLOWFILE_ATTRIBUTES;
+import static org.apache.nifi.expression.ExpressionLanguageScope.NONE;
+import static org.apache.nifi.processor.util.StandardValidators.NON_BLANK_VALIDATOR;
+import static org.apache.nifi.processor.util.StandardValidators.URL_VALIDATOR;
+
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Base64;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import org.apache.nifi.annotation.behavior.EventDriven;
+import org.apache.nifi.annotation.behavior.InputRequirement;
+import org.apache.nifi.annotation.behavior.InputRequirement.Requirement;
+import org.apache.nifi.annotation.behavior.SideEffectFree;
+import org.apache.nifi.annotation.behavior.SupportsBatching;
+import org.apache.nifi.annotation.behavior.WritesAttribute;
+import org.apache.nifi.annotation.behavior.WritesAttributes;
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.annotation.lifecycle.OnScheduled;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.ValidationContext;
+import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.flowfile.attributes.CoreAttributes;
+import org.apache.nifi.logging.ComponentLog;
+import org.apache.nifi.processor.AbstractProcessor;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.processor.util.StandardValidators;
+import org.apache.nifi.record.path.RecordPath;
+import org.apache.nifi.record.path.RecordPathResult;
+import org.apache.nifi.schema.access.SchemaNotFoundException;
+import org.apache.nifi.security.util.crypto.HashAlgorithm;
+import org.apache.nifi.serialization.MalformedRecordException;
+import org.apache.nifi.serialization.RecordReader;
+import org.apache.nifi.serialization.RecordReaderFactory;
+import org.apache.nifi.serialization.RecordSetWriter;
+import org.apache.nifi.serialization.RecordSetWriterFactory;
+import org.apache.nifi.serialization.record.Record;
+import org.apache.nifi.serialization.record.RecordFieldType;
+import org.apache.nifi.serialization.record.RecordSchema;
+import org.apache.nifi.web.client.api.HttpResponseEntity;
+import org.apache.nifi.web.client.api.WebClientService;
+import org.apache.nifi.web.client.provider.api.WebClientServiceProvider;
+
+@Tags({"Workday", "report"})
+@InputRequirement(Requirement.INPUT_ALLOWED)
+@CapabilityDescription("A processor which can interact with a configurable Workday Report. The processor can forward the content without modification, or you can transform it by"
+ + " providing the specific Record Reader and Record Writer services based on your needs. You can also hash, or remove fields using the input parameters and schemes in the Record Writer. "
+ + "Supported Workday report formats are: csv, simplexml, json")
+@EventDriven
+@SideEffectFree
+@SupportsBatching
+@WritesAttributes({
+ @WritesAttribute(attribute = GetWorkdayReport.GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS, description = "The Java exception class raised when the processor fails"),
+ @WritesAttribute(attribute = GetWorkdayReport.GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE, description = "The Java exception message raised when the processor fails"),
+ @WritesAttribute(attribute = "mime.type", description = "Sets the mime.type attribute to the MIME Type specified by the Source / Record Writer"),
+ @WritesAttribute(attribute= GetWorkdayReport.RECORD_COUNT, description = "The number of records in an outgoing FlowFile. This is only populated on the 'success' relationship "
+ + "when Record Reader and Writer is set.")})
+public class GetWorkdayReport extends AbstractProcessor {
+
+ protected static final String STATUS_CODE = "getworkdayreport.status.code";
+ protected static final String REQUEST_URL = "getworkdayreport.request.url";
+ protected static final String REQUEST_DURATION = "getworkdayreport.request.duration";
+ protected static final String TRANSACTION_ID = "getworkdayreport.tx.id";
+ protected static final String GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS = "getworkdayreport.java.exception.class";
+ protected static final String GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE = "getworkdayreport.java.exception.message";
+ protected static final String RECORD_COUNT = "record.count";
+ protected static final String BASIC_PREFIX = "Basic ";
+ protected static final String COLUMNS_TO_HASH_SEPARATOR = ",";
+ protected static final String HEADER_AUTHORIZATION = "Authorization";
+ protected static final String HEADER_CONTENT_TYPE = "Content-Type";
+ protected static final String USERNAME_PASSWORD_SEPARATOR = ":";
+
+ protected static final PropertyDescriptor REPORT_URL = new PropertyDescriptor.Builder()
+ .name("Workday Report URL")
+ .displayName("Workday Report URL")
+ .description("HTTP remote URL of Workday report including a scheme of http or https, as well as a hostname or IP address with optional port and path elements.")
+ .required(true)
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .addValidator(URL_VALIDATOR)
+ .build();
+
+ protected static final PropertyDescriptor WORKDAY_USERNAME = new PropertyDescriptor.Builder()
+ .name("Workday Username")
+ .displayName("Workday Username")
+ .description("The username provided for authentication of Workday requests. Encoded using Base64 for HTTP Basic Authentication as described in RFC 7617.")
+ .required(true)
+ .addValidator(StandardValidators.createRegexMatchingValidator(Pattern.compile("^[\\x20-\\x39\\x3b-\\x7e\\x80-\\xff]+$")))
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .build();
+
+ protected static final PropertyDescriptor WORKDAY_PASSWORD = new PropertyDescriptor.Builder()
+ .name("Workday Password")
+ .displayName("Workday Password")
+ .description("The password provided for authentication of Workday requests. Encoded using Base64 for HTTP Basic Authentication as described in RFC 7617.")
+ .required(true)
+ .sensitive(true)
+ .addValidator(StandardValidators.createRegexMatchingValidator(Pattern.compile("^[\\x20-\\x7e\\x80-\\xff]+$")))
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .build();
+
+ protected static final PropertyDescriptor WEB_CLIENT_SERVICE = new PropertyDescriptor.Builder()
+ .name("Web Client Service Provider")
+ .description("Web client which is used to communicate with the Workday API.")
+ .required(true)
+ .identifiesControllerService(WebClientServiceProvider.class)
+ .build();
+
+ protected static final PropertyDescriptor FIELDS_TO_HASH = new PropertyDescriptor.Builder()
+ .name("Hashed Fields")
+ .displayName("Hashed Fields")
+ .description("Comma separated record paths to be replaced with their corresponding hash.")
+ .required(false)
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .addValidator(NON_BLANK_VALIDATOR)
+ .build();
+
+ protected static final PropertyDescriptor HASHING_ALGORITHM = new PropertyDescriptor.Builder()
+ .name("Hashing Algorithm")
+ .displayName("Hashing Algorithm")
+ .description("Determines what hashing algorithm should be used to perform the hashing function.")
+ .required(true)
+ .allowableValues(HashAlgorithm.SHA256.getName(), HashAlgorithm.SHA512.getName())
+ .defaultValue(HashAlgorithm.SHA256.getName())
+ .addValidator(NON_BLANK_VALIDATOR)
+ .expressionLanguageSupported(NONE)
+ .dependsOn(FIELDS_TO_HASH)
+ .build();
+
+ protected static final PropertyDescriptor RECORD_READER_FACTORY = new PropertyDescriptor.Builder()
+ .name("record-reader")
+ .displayName("Record Reader")
+ .description("Specifies the Controller Service to use for parsing incoming data and determining the data's schema.")
+ .identifiesControllerService(RecordReaderFactory.class)
+ .required(false)
+ .build();
+
+ protected static final PropertyDescriptor RECORD_WRITER_FACTORY = new PropertyDescriptor.Builder()
+ .name("record-writer")
+ .displayName("Record Writer")
+ .description("The Record Writer to use for serializing Records to an output FlowFile.")
+ .identifiesControllerService(RecordSetWriterFactory.class)
+ .dependsOn(RECORD_READER_FACTORY)
+ .required(true)
+ .build();
+
+ protected static final Relationship ORIGINAL = new Relationship.Builder()
+ .name("Original")
+ .description("Request FlowFiles transferred when receiving HTTP responses with a status code between 200 and 299.")
+ .build();
+
+ protected static final Relationship FAILURE = new Relationship.Builder()
+ .name("Failure")
+ .description("Request FlowFiles transferred when receiving socket communication errors.")
+ .build();
+
+ protected static final Relationship RESPONSE = new Relationship.Builder()
+ .name("Response")
+ .description("Response FlowFiles transferred when receiving HTTP responses with a status code between 200 and 299.")
+ .build();
+
+ protected static final Set<Relationship> RELATIONSHIPS = Collections.unmodifiableSet(new HashSet<>(Arrays.asList(ORIGINAL, RESPONSE, FAILURE)));
+ protected static final List<PropertyDescriptor> PROPERTIES = Collections.unmodifiableList(Arrays.asList(REPORT_URL, WORKDAY_USERNAME, WORKDAY_PASSWORD, WEB_CLIENT_SERVICE,
+ FIELDS_TO_HASH, HASHING_ALGORITHM, RECORD_READER_FACTORY, RECORD_WRITER_FACTORY));
Review Comment:
done
##########
nifi-nar-bundles/nifi-workday-bundle/nifi-workday-processors/src/main/java/org/apache/nifi/processors/workday/GetWorkdayReport.java:
##########
@@ -0,0 +1,442 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.processors.workday;
+
+import static org.apache.nifi.expression.ExpressionLanguageScope.FLOWFILE_ATTRIBUTES;
+import static org.apache.nifi.expression.ExpressionLanguageScope.NONE;
+import static org.apache.nifi.processor.util.StandardValidators.NON_BLANK_VALIDATOR;
+import static org.apache.nifi.processor.util.StandardValidators.URL_VALIDATOR;
+
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Base64;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import org.apache.nifi.annotation.behavior.EventDriven;
+import org.apache.nifi.annotation.behavior.InputRequirement;
+import org.apache.nifi.annotation.behavior.InputRequirement.Requirement;
+import org.apache.nifi.annotation.behavior.SideEffectFree;
+import org.apache.nifi.annotation.behavior.SupportsBatching;
+import org.apache.nifi.annotation.behavior.WritesAttribute;
+import org.apache.nifi.annotation.behavior.WritesAttributes;
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.annotation.lifecycle.OnScheduled;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.ValidationContext;
+import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.flowfile.attributes.CoreAttributes;
+import org.apache.nifi.logging.ComponentLog;
+import org.apache.nifi.processor.AbstractProcessor;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.processor.util.StandardValidators;
+import org.apache.nifi.record.path.RecordPath;
+import org.apache.nifi.record.path.RecordPathResult;
+import org.apache.nifi.schema.access.SchemaNotFoundException;
+import org.apache.nifi.security.util.crypto.HashAlgorithm;
+import org.apache.nifi.serialization.MalformedRecordException;
+import org.apache.nifi.serialization.RecordReader;
+import org.apache.nifi.serialization.RecordReaderFactory;
+import org.apache.nifi.serialization.RecordSetWriter;
+import org.apache.nifi.serialization.RecordSetWriterFactory;
+import org.apache.nifi.serialization.record.Record;
+import org.apache.nifi.serialization.record.RecordFieldType;
+import org.apache.nifi.serialization.record.RecordSchema;
+import org.apache.nifi.web.client.api.HttpResponseEntity;
+import org.apache.nifi.web.client.api.WebClientService;
+import org.apache.nifi.web.client.provider.api.WebClientServiceProvider;
+
+@Tags({"Workday", "report"})
+@InputRequirement(Requirement.INPUT_ALLOWED)
+@CapabilityDescription("A processor which can interact with a configurable Workday Report. The processor can forward the content without modification, or you can transform it by"
+ + " providing the specific Record Reader and Record Writer services based on your needs. You can also hash, or remove fields using the input parameters and schemes in the Record Writer. "
+ + "Supported Workday report formats are: csv, simplexml, json")
+@EventDriven
+@SideEffectFree
+@SupportsBatching
+@WritesAttributes({
+ @WritesAttribute(attribute = GetWorkdayReport.GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS, description = "The Java exception class raised when the processor fails"),
+ @WritesAttribute(attribute = GetWorkdayReport.GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE, description = "The Java exception message raised when the processor fails"),
+ @WritesAttribute(attribute = "mime.type", description = "Sets the mime.type attribute to the MIME Type specified by the Source / Record Writer"),
+ @WritesAttribute(attribute= GetWorkdayReport.RECORD_COUNT, description = "The number of records in an outgoing FlowFile. This is only populated on the 'success' relationship "
+ + "when Record Reader and Writer is set.")})
+public class GetWorkdayReport extends AbstractProcessor {
+
+ protected static final String STATUS_CODE = "getworkdayreport.status.code";
+ protected static final String REQUEST_URL = "getworkdayreport.request.url";
+ protected static final String REQUEST_DURATION = "getworkdayreport.request.duration";
+ protected static final String TRANSACTION_ID = "getworkdayreport.tx.id";
+ protected static final String GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS = "getworkdayreport.java.exception.class";
+ protected static final String GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE = "getworkdayreport.java.exception.message";
+ protected static final String RECORD_COUNT = "record.count";
+ protected static final String BASIC_PREFIX = "Basic ";
+ protected static final String COLUMNS_TO_HASH_SEPARATOR = ",";
+ protected static final String HEADER_AUTHORIZATION = "Authorization";
+ protected static final String HEADER_CONTENT_TYPE = "Content-Type";
+ protected static final String USERNAME_PASSWORD_SEPARATOR = ":";
+
+ protected static final PropertyDescriptor REPORT_URL = new PropertyDescriptor.Builder()
+ .name("Workday Report URL")
+ .displayName("Workday Report URL")
+ .description("HTTP remote URL of Workday report including a scheme of http or https, as well as a hostname or IP address with optional port and path elements.")
+ .required(true)
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .addValidator(URL_VALIDATOR)
+ .build();
+
+ protected static final PropertyDescriptor WORKDAY_USERNAME = new PropertyDescriptor.Builder()
+ .name("Workday Username")
+ .displayName("Workday Username")
+ .description("The username provided for authentication of Workday requests. Encoded using Base64 for HTTP Basic Authentication as described in RFC 7617.")
+ .required(true)
+ .addValidator(StandardValidators.createRegexMatchingValidator(Pattern.compile("^[\\x20-\\x39\\x3b-\\x7e\\x80-\\xff]+$")))
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .build();
+
+ protected static final PropertyDescriptor WORKDAY_PASSWORD = new PropertyDescriptor.Builder()
+ .name("Workday Password")
+ .displayName("Workday Password")
+ .description("The password provided for authentication of Workday requests. Encoded using Base64 for HTTP Basic Authentication as described in RFC 7617.")
+ .required(true)
+ .sensitive(true)
+ .addValidator(StandardValidators.createRegexMatchingValidator(Pattern.compile("^[\\x20-\\x7e\\x80-\\xff]+$")))
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .build();
+
+ protected static final PropertyDescriptor WEB_CLIENT_SERVICE = new PropertyDescriptor.Builder()
+ .name("Web Client Service Provider")
+ .description("Web client which is used to communicate with the Workday API.")
+ .required(true)
+ .identifiesControllerService(WebClientServiceProvider.class)
+ .build();
+
+ protected static final PropertyDescriptor FIELDS_TO_HASH = new PropertyDescriptor.Builder()
+ .name("Hashed Fields")
+ .displayName("Hashed Fields")
+ .description("Comma separated record paths to be replaced with their corresponding hash.")
+ .required(false)
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .addValidator(NON_BLANK_VALIDATOR)
+ .build();
+
+ protected static final PropertyDescriptor HASHING_ALGORITHM = new PropertyDescriptor.Builder()
+ .name("Hashing Algorithm")
+ .displayName("Hashing Algorithm")
+ .description("Determines what hashing algorithm should be used to perform the hashing function.")
+ .required(true)
+ .allowableValues(HashAlgorithm.SHA256.getName(), HashAlgorithm.SHA512.getName())
+ .defaultValue(HashAlgorithm.SHA256.getName())
+ .addValidator(NON_BLANK_VALIDATOR)
+ .expressionLanguageSupported(NONE)
+ .dependsOn(FIELDS_TO_HASH)
+ .build();
+
+ protected static final PropertyDescriptor RECORD_READER_FACTORY = new PropertyDescriptor.Builder()
+ .name("record-reader")
+ .displayName("Record Reader")
+ .description("Specifies the Controller Service to use for parsing incoming data and determining the data's schema.")
+ .identifiesControllerService(RecordReaderFactory.class)
+ .required(false)
+ .build();
+
+ protected static final PropertyDescriptor RECORD_WRITER_FACTORY = new PropertyDescriptor.Builder()
+ .name("record-writer")
+ .displayName("Record Writer")
+ .description("The Record Writer to use for serializing Records to an output FlowFile.")
+ .identifiesControllerService(RecordSetWriterFactory.class)
+ .dependsOn(RECORD_READER_FACTORY)
+ .required(true)
+ .build();
+
+ protected static final Relationship ORIGINAL = new Relationship.Builder()
+ .name("Original")
+ .description("Request FlowFiles transferred when receiving HTTP responses with a status code between 200 and 299.")
+ .build();
+
+ protected static final Relationship FAILURE = new Relationship.Builder()
+ .name("Failure")
+ .description("Request FlowFiles transferred when receiving socket communication errors.")
+ .build();
+
+ protected static final Relationship RESPONSE = new Relationship.Builder()
+ .name("Response")
+ .description("Response FlowFiles transferred when receiving HTTP responses with a status code between 200 and 299.")
+ .build();
+
+ protected static final Set<Relationship> RELATIONSHIPS = Collections.unmodifiableSet(new HashSet<>(Arrays.asList(ORIGINAL, RESPONSE, FAILURE)));
+ protected static final List<PropertyDescriptor> PROPERTIES = Collections.unmodifiableList(Arrays.asList(REPORT_URL, WORKDAY_USERNAME, WORKDAY_PASSWORD, WEB_CLIENT_SERVICE,
+ FIELDS_TO_HASH, HASHING_ALGORITHM, RECORD_READER_FACTORY, RECORD_WRITER_FACTORY));
+
+ private final AtomicReference<WebClientService> webClientReference = new AtomicReference<>();
+ private final AtomicReference<RecordReaderFactory> recordReaderFactoryReference = new AtomicReference<>();
+ private final AtomicReference<RecordSetWriterFactory> recordSetWriterFactoryReference = new AtomicReference<>();
+
+ @Override
+ protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
+ return PROPERTIES;
+ }
+
+ @Override
+ public Set<Relationship> getRelationships() {
+ return RELATIONSHIPS;
+ }
+
+ @OnScheduled
+ public void setUpClient(final ProcessContext context) {
+ WebClientServiceProvider standardWebClientServiceProvider = context.getProperty(WEB_CLIENT_SERVICE).asControllerService(WebClientServiceProvider.class);
+ RecordReaderFactory recordReaderFactory = context.getProperty(RECORD_READER_FACTORY).asControllerService(RecordReaderFactory.class);
+ RecordSetWriterFactory recordSetWriterFactory = context.getProperty(RECORD_WRITER_FACTORY).asControllerService(RecordSetWriterFactory.class);
+ WebClientService webClientService = standardWebClientServiceProvider.getWebClientService();
+ webClientReference.set(webClientService);
+ recordReaderFactoryReference.set(recordReaderFactory);
+ recordSetWriterFactoryReference.set(recordSetWriterFactory);
+ }
+
+ @Override
+ public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException {
+ FlowFile flowFile = session.get();
+
+ if (skipExecution(context, flowFile)) {
+ return;
+ }
+
+ ComponentLog logger = getLogger();
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@nifi.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [nifi] ferencerdei commented on a diff in pull request #6376: NIFI-10455 Get workday report processor
Posted by GitBox <gi...@apache.org>.
ferencerdei commented on code in PR #6376:
URL: https://github.com/apache/nifi/pull/6376#discussion_r973021873
##########
nifi-nar-bundles/nifi-workday-bundle/nifi-workday-processors/src/main/java/org/apache/nifi/processors/workday/GetWorkdayReport.java:
##########
@@ -0,0 +1,466 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.processors.workday;
+
+import static org.apache.nifi.expression.ExpressionLanguageScope.FLOWFILE_ATTRIBUTES;
+import static org.apache.nifi.expression.ExpressionLanguageScope.NONE;
+import static org.apache.nifi.processor.util.StandardValidators.NON_BLANK_VALIDATOR;
+import static org.apache.nifi.processor.util.StandardValidators.URL_VALIDATOR;
+
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Base64;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import org.apache.nifi.annotation.behavior.EventDriven;
+import org.apache.nifi.annotation.behavior.InputRequirement;
+import org.apache.nifi.annotation.behavior.InputRequirement.Requirement;
+import org.apache.nifi.annotation.behavior.SideEffectFree;
+import org.apache.nifi.annotation.behavior.SupportsBatching;
+import org.apache.nifi.annotation.behavior.WritesAttribute;
+import org.apache.nifi.annotation.behavior.WritesAttributes;
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.annotation.lifecycle.OnScheduled;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.ValidationContext;
+import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.flowfile.attributes.CoreAttributes;
+import org.apache.nifi.processor.AbstractProcessor;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.processor.util.StandardValidators;
+import org.apache.nifi.record.path.RecordPath;
+import org.apache.nifi.record.path.RecordPathResult;
+import org.apache.nifi.schema.access.SchemaNotFoundException;
+import org.apache.nifi.serialization.MalformedRecordException;
+import org.apache.nifi.serialization.RecordReader;
+import org.apache.nifi.serialization.RecordReaderFactory;
+import org.apache.nifi.serialization.RecordSetWriter;
+import org.apache.nifi.serialization.RecordSetWriterFactory;
+import org.apache.nifi.serialization.record.Record;
+import org.apache.nifi.serialization.record.RecordFieldType;
+import org.apache.nifi.serialization.record.RecordSchema;
+import org.apache.nifi.web.client.api.HttpResponseEntity;
+import org.apache.nifi.web.client.api.WebClientService;
+import org.apache.nifi.web.client.provider.api.WebClientServiceProvider;
+
+@Tags({"Workday", "report"})
+@InputRequirement(Requirement.INPUT_ALLOWED)
+@CapabilityDescription("A processor which can interact with a configurable Workday Report. The processor can forward the content without modification, or you can transform it by"
+ + " providing the specific Record Reader and Record Writer services based on your needs. You can also hash, or remove fields using the input parameters and schemes in the Record Writer. "
+ + "Supported Workday report formats are: csv, simplexml, json")
+@EventDriven
+@SideEffectFree
+@SupportsBatching
+@WritesAttributes({
+ @WritesAttribute(attribute = GetWorkdayReport.GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS, description = "The Java exception class raised when the processor fails"),
+ @WritesAttribute(attribute = GetWorkdayReport.GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE, description = "The Java exception message raised when the processor fails"),
+ @WritesAttribute(attribute = "mime.type", description = "Sets the mime.type attribute to the MIME Type specified by the Source / Record Writer"),
+ @WritesAttribute(attribute= GetWorkdayReport.RECORD_COUNT, description = "The number of records in an outgoing FlowFile. This is only populated on the 'success' relationship "
+ + "when Record Reader and Writer is set.")})
+public class GetWorkdayReport extends AbstractProcessor {
+
+ protected static final String STATUS_CODE = "getworkdayreport.status.code";
+ protected static final String REQUEST_URL = "getworkdayreport.request.url";
+ protected static final String REQUEST_DURATION = "getworkdayreport.request.duration";
+ protected static final String TRANSACTION_ID = "getworkdayreport.tx.id";
+ protected static final String GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS = "getworkdayreport.java.exception.class";
+ protected static final String GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE = "getworkdayreport.java.exception.message";
+ protected static final String RECORD_COUNT = "record.count";
+ protected static final String BASIC_PREFIX = "Basic ";
+ protected static final String COLUMNS_TO_HASH_SEPARATOR = ",";
+ protected static final String HEADER_AUTHORIZATION = "Authorization";
+ protected static final String HEADER_CONTENT_TYPE = "Content-Type";
+ protected static final String USERNAME_PASSWORD_SEPARATOR = ":";
+
+ protected static final PropertyDescriptor REPORT_URL = new PropertyDescriptor.Builder()
+ .name("Workday Report URL")
+ .displayName("Workday Report URL")
+ .description("HTTP remote URL of Workday report including a scheme of http or https, as well as a hostname or IP address with optional port and path elements.")
+ .required(true)
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .addValidator(URL_VALIDATOR)
+ .build();
+
+ protected static final PropertyDescriptor WORKDAY_USERNAME = new PropertyDescriptor.Builder()
+ .name("Workday Username")
+ .displayName("Workday Username")
+ .description("The username provided for authentication of Workday requests. Encoded using Base64 for HTTP Basic Authentication as described in RFC 7617.")
+ .required(true)
+ .addValidator(StandardValidators.createRegexMatchingValidator(Pattern.compile("^[\\x20-\\x39\\x3b-\\x7e\\x80-\\xff]+$")))
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .build();
+
+ protected static final PropertyDescriptor WORKDAY_PASSWORD = new PropertyDescriptor.Builder()
+ .name("Workday Password")
+ .displayName("Workday Password")
+ .description("The password provided for authentication of Workday requests. Encoded using Base64 for HTTP Basic Authentication as described in RFC 7617.")
+ .required(true)
+ .sensitive(true)
+ .addValidator(StandardValidators.createRegexMatchingValidator(Pattern.compile("^[\\x20-\\x7e\\x80-\\xff]+$")))
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .build();
+
+ protected static final PropertyDescriptor WEB_CLIENT_SERVICE = new PropertyDescriptor.Builder()
+ .name("Web Client Service Provider")
+ .description("Web client which is used to communicate with the Workday API.")
+ .required(true)
+ .identifiesControllerService(WebClientServiceProvider.class)
+ .build();
+
+ protected static final PropertyDescriptor FIELDS_TO_HASH = new PropertyDescriptor.Builder()
+ .name("Hashed Fields")
+ .displayName("Hashed Fields")
+ .description("Comma separated record paths to be replaced with their corresponding hash.")
+ .required(false)
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .addValidator(NON_BLANK_VALIDATOR)
+ .build();
+
+ protected static final PropertyDescriptor HASHING_ALGORITHM = new PropertyDescriptor.Builder()
+ .name("Hashing Algorithm")
+ .displayName("Hashing Algorithm")
+ .description("Determines what hashing algorithm should be used to perform the hashing function.")
+ .required(true)
+ .allowableValues(HashAlgorithm.getNames())
+ .defaultValue(HashAlgorithm.SHA256.getName())
+ .addValidator(NON_BLANK_VALIDATOR)
+ .expressionLanguageSupported(NONE)
+ .dependsOn(FIELDS_TO_HASH)
+ .build();
+
+ protected static final PropertyDescriptor RECORD_READER_FACTORY = new PropertyDescriptor.Builder()
+ .name("record-reader")
+ .displayName("Record Reader")
+ .description("Specifies the Controller Service to use for parsing incoming data and determining the data's schema.")
+ .identifiesControllerService(RecordReaderFactory.class)
+ .required(false)
+ .build();
+
+ protected static final PropertyDescriptor RECORD_WRITER_FACTORY = new PropertyDescriptor.Builder()
+ .name("record-writer")
+ .displayName("Record Writer")
+ .description("The Record Writer to use for serializing Records to an output FlowFile.")
+ .identifiesControllerService(RecordSetWriterFactory.class)
+ .dependsOn(RECORD_READER_FACTORY)
+ .required(true)
+ .build();
+
+ protected static final Relationship ORIGINAL = new Relationship.Builder()
+ .name("original")
+ .description("Request FlowFiles transferred when receiving HTTP responses with a status code between 200 and 299.")
+ .build();
+
+ protected static final Relationship FAILURE = new Relationship.Builder()
+ .name("failure")
+ .description("Request FlowFiles transferred when receiving socket communication errors.")
+ .build();
+
+ protected static final Relationship SUCCESS = new Relationship.Builder()
+ .name("success")
+ .description("Response FlowFiles transferred when receiving HTTP responses with a status code between 200 and 299.")
+ .build();
+
+ protected static final Set<Relationship> RELATIONSHIPS = Collections.unmodifiableSet(new HashSet<>(Arrays.asList(ORIGINAL, SUCCESS, FAILURE)));
+ protected static final List<PropertyDescriptor> PROPERTIES = Collections.unmodifiableList(Arrays.asList(
+ REPORT_URL,
+ WORKDAY_USERNAME,
+ WORKDAY_PASSWORD,
+ WEB_CLIENT_SERVICE,
+ FIELDS_TO_HASH,
+ HASHING_ALGORITHM,
+ RECORD_READER_FACTORY,
+ RECORD_WRITER_FACTORY
+ ));
+
+ private final AtomicReference<WebClientService> webClientReference = new AtomicReference<>();
+ private final AtomicReference<RecordReaderFactory> recordReaderFactoryReference = new AtomicReference<>();
+ private final AtomicReference<RecordSetWriterFactory> recordSetWriterFactoryReference = new AtomicReference<>();
+
+ @Override
+ protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
+ return PROPERTIES;
+ }
+
+ @Override
+ public Set<Relationship> getRelationships() {
+ return RELATIONSHIPS;
+ }
+
+ @OnScheduled
+ public void setUpClient(final ProcessContext context) {
+ WebClientServiceProvider standardWebClientServiceProvider = context.getProperty(WEB_CLIENT_SERVICE).asControllerService(WebClientServiceProvider.class);
+ RecordReaderFactory recordReaderFactory = context.getProperty(RECORD_READER_FACTORY).asControllerService(RecordReaderFactory.class);
+ RecordSetWriterFactory recordSetWriterFactory = context.getProperty(RECORD_WRITER_FACTORY).asControllerService(RecordSetWriterFactory.class);
+ WebClientService webClientService = standardWebClientServiceProvider.getWebClientService();
+ webClientReference.set(webClientService);
+ recordReaderFactoryReference.set(recordReaderFactory);
+ recordSetWriterFactoryReference.set(recordSetWriterFactory);
+ }
+
+ @Override
+ public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException {
+ FlowFile flowFile = session.get();
+
+ if (skipExecution(context, flowFile)) {
+ return;
+ }
+
+ FlowFile responseFlowFile = null;
+
+ try {
+ WebClientService webClientService = webClientReference.get();
+ URI uri = new URI(context.getProperty(REPORT_URL).evaluateAttributeExpressions(flowFile).getValue().trim());
+ long startNanos = System.nanoTime();
+ String authorization = createAuthorizationHeader(context, flowFile);
+
+ try(HttpResponseEntity httpResponseEntity = webClientService.get()
+ .uri(uri)
+ .header(HEADER_AUTHORIZATION, authorization)
+ .retrieve()) {
+ responseFlowFile = createResponseFlowFile(flowFile, session, context, httpResponseEntity);
+ long elapsedTime = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos);
+ Map<String, String> commonAttributes = createCommonAttributes(uri, httpResponseEntity, elapsedTime);
+
+ if (flowFile != null) {
+ flowFile = session.putAllAttributes(flowFile, setMimeType(commonAttributes, httpResponseEntity));
+ }
+ if (responseFlowFile != null) {
+ responseFlowFile = session.putAllAttributes(responseFlowFile, commonAttributes);
+ if (flowFile == null) {
+ session.getProvenanceReporter().receive(responseFlowFile, uri.toString(), elapsedTime);
+ } else {
+ session.getProvenanceReporter().fetch(responseFlowFile, uri.toString(), elapsedTime);
+ }
+ }
+
+ route(flowFile, responseFlowFile, session, context, httpResponseEntity.statusCode());
+ }
+ } catch (Exception e) {
+ if (flowFile == null) {
+ getLogger().error("Request Processing failed", e);
+ context.yield();
+ } else {
+ getLogger().error("Request Processing failed: {}", flowFile, e);
+ session.penalize(flowFile);
+ flowFile = session.putAttribute(flowFile, GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS, e.getClass().getSimpleName());
+ flowFile = session.putAttribute(flowFile, GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE, e.getMessage());
+ session.transfer(flowFile, FAILURE);
+ }
+
+ if (responseFlowFile != null) {
+ session.remove(responseFlowFile);
+ }
+ }
+ }
+
+ @Override
+ protected Collection<ValidationResult> customValidate(ValidationContext validationContext) {
+ List<ValidationResult> results = new ArrayList<>(super.customValidate(validationContext));
+ if (validationContext.getProperty(FIELDS_TO_HASH).isSet() && !validationContext.getProperty(RECORD_READER_FACTORY).isSet()) {
+ results.add(new ValidationResult.Builder()
+ .valid(false)
+ .explanation("Record-Reader and Record-Writer must be set if you would like to hash specific fields")
+ .subject("Workday report configuration")
+ .build());
+ }
+ return results;
+ }
+
+ /*
+ * If we have no FlowFile, and all incoming connections are self-loops then we can continue on.
+ * However, if we have no FlowFile and we have connections coming from other Processors, then
+ * we know that we should run only if we have a FlowFile.
+ */
+ private boolean skipExecution(ProcessContext context, FlowFile flowfile) {
+ return context.hasIncomingConnection() && flowfile == null && context.hasNonLoopConnection();
+ }
+
+ private FlowFile createResponseFlowFile(FlowFile flowfile, ProcessSession session, ProcessContext context, HttpResponseEntity httpResponseEntity)
+ throws IOException, SchemaNotFoundException, MalformedRecordException {
+ FlowFile responseFlowFile = null;
+ String hashingAlgorithm = context.getProperty(HASHING_ALGORITHM).getValue();
+ Set<String> columnsToHash = Optional.ofNullable(
+ context.getProperty(FIELDS_TO_HASH)
+ .evaluateAttributeExpressions(flowfile)
+ .getValue())
+ .map(String::trim)
+ .map(columns -> columns.split(COLUMNS_TO_HASH_SEPARATOR))
+ .map(Arrays::stream)
+ .map(columns -> columns.collect(Collectors.toSet()))
+ .orElse(Collections.emptySet());
+ try {
+ if (isSuccess(httpResponseEntity.statusCode())) {
+ responseFlowFile = flowfile == null ? session.create() : session.create(flowfile);
+ InputStream responseBodyStream = httpResponseEntity.body();
+ if (recordReaderFactoryReference.get() != null) {
+ TransformResult transformResult = transformRecords(session, flowfile, responseFlowFile, hashingAlgorithm, columnsToHash, responseBodyStream);
+ Map<String, String> attributes = new HashMap<>();
+ attributes.put(RECORD_COUNT, String.valueOf(transformResult.getNumberOfRecords()));
+ attributes.put(CoreAttributes.MIME_TYPE.key(), transformResult.getMimeType());
+ responseFlowFile = session.putAllAttributes(responseFlowFile, attributes);
+ } else {
+ responseFlowFile = session.importFrom(responseBodyStream, responseFlowFile);
+ Optional<String> mimeType = httpResponseEntity.headers().getFirstHeader(HEADER_CONTENT_TYPE);
+ if (mimeType.isPresent()) {
+ responseFlowFile = session.putAttribute(responseFlowFile, CoreAttributes.MIME_TYPE.key(), mimeType.get());
+ }
+ }
+ }
+ } catch (Exception e) {
+ session.remove(responseFlowFile);
+ throw e;
+ }
+ return responseFlowFile;
+ }
+
+ private String createAuthorizationHeader(ProcessContext context, FlowFile flowfile) {
+ String userName = context.getProperty(WORKDAY_USERNAME).evaluateAttributeExpressions(flowfile).getValue();
+ String password = context.getProperty(WORKDAY_PASSWORD).evaluateAttributeExpressions(flowfile).getValue();
+ String base64Credential = Base64.getEncoder().encodeToString((userName + USERNAME_PASSWORD_SEPARATOR + password).getBytes(StandardCharsets.UTF_8));
+ return BASIC_PREFIX + base64Credential;
+ }
+
+ private TransformResult transformRecords(ProcessSession session, FlowFile flowfile, FlowFile responseFlowFile, String hashingAlgorithm, Set<String> columnsToHash,
+ InputStream responseBodyStream) throws IOException, SchemaNotFoundException, MalformedRecordException {
+ int numberOfRecords = 0;
+ String mimeType = null;
+ try (RecordReader reader = recordReaderFactoryReference.get().createRecordReader(flowfile,
+ new BufferedInputStream(responseBodyStream), getLogger())) {
+ RecordSchema schema = recordSetWriterFactoryReference.get()
+ .getSchema(flowfile == null ? Collections.emptyMap() : flowfile.getAttributes(), reader.getSchema());
+ try (OutputStream responseStream = session.write(responseFlowFile);
+ RecordSetWriter recordSetWriter = recordSetWriterFactoryReference.get().createWriter(getLogger(), schema, responseStream, responseFlowFile)) {
+ mimeType = recordSetWriter.getMimeType();
+ recordSetWriter.beginRecordSet();
+ Record currentRecord;
+ // as the report can be changed independently from the flow, it's safer to ignore field types and unknown fields in the Record Reading process
+ while ((currentRecord = reader.nextRecord(false, true)) != null) {
+ for (String recordPath : columnsToHash) {
+ RecordPathResult evaluate = RecordPath.compile("hash(" + recordPath + ", '" + hashingAlgorithm + "')").evaluate(currentRecord);
+ evaluate.getSelectedFields().forEach(fieldVal -> fieldVal.updateValue(fieldVal.getValue(), RecordFieldType.STRING.getDataType()));
+ }
Review Comment:
I've removed the hashing functionality
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@nifi.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [nifi] bejancsaba commented on a diff in pull request #6376: NIFI-10455 Get workday report processor
Posted by GitBox <gi...@apache.org>.
bejancsaba commented on code in PR #6376:
URL: https://github.com/apache/nifi/pull/6376#discussion_r966777791
##########
nifi-nar-bundles/nifi-workday-bundle/nifi-workday-processors/src/main/java/org/apache/nifi/processors/workday/GetWorkdayReport.java:
##########
@@ -0,0 +1,432 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.processors.workday;
+
+import static org.apache.nifi.expression.ExpressionLanguageScope.FLOWFILE_ATTRIBUTES;
+import static org.apache.nifi.expression.ExpressionLanguageScope.NONE;
+import static org.apache.nifi.processor.util.StandardValidators.NON_BLANK_VALIDATOR;
+import static org.apache.nifi.processor.util.StandardValidators.URL_VALIDATOR;
+
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Base64;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import org.apache.nifi.annotation.behavior.EventDriven;
+import org.apache.nifi.annotation.behavior.InputRequirement;
+import org.apache.nifi.annotation.behavior.InputRequirement.Requirement;
+import org.apache.nifi.annotation.behavior.SideEffectFree;
+import org.apache.nifi.annotation.behavior.SupportsBatching;
+import org.apache.nifi.annotation.behavior.WritesAttribute;
+import org.apache.nifi.annotation.behavior.WritesAttributes;
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.annotation.lifecycle.OnScheduled;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.ValidationContext;
+import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.flowfile.attributes.CoreAttributes;
+import org.apache.nifi.logging.ComponentLog;
+import org.apache.nifi.processor.AbstractProcessor;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.processor.util.StandardValidators;
+import org.apache.nifi.record.path.RecordPath;
+import org.apache.nifi.record.path.RecordPathResult;
+import org.apache.nifi.schema.access.SchemaNotFoundException;
+import org.apache.nifi.security.util.crypto.HashAlgorithm;
+import org.apache.nifi.security.util.crypto.HashService;
+import org.apache.nifi.serialization.MalformedRecordException;
+import org.apache.nifi.serialization.RecordReader;
+import org.apache.nifi.serialization.RecordReaderFactory;
+import org.apache.nifi.serialization.RecordSetWriter;
+import org.apache.nifi.serialization.RecordSetWriterFactory;
+import org.apache.nifi.serialization.record.Record;
+import org.apache.nifi.serialization.record.RecordFieldType;
+import org.apache.nifi.serialization.record.RecordSchema;
+import org.apache.nifi.web.client.api.HttpResponseEntity;
+import org.apache.nifi.web.client.api.WebClientService;
+import org.apache.nifi.web.client.provider.api.WebClientServiceProvider;
+
+@Tags({"Workday", "report", "get"})
Review Comment:
Do we need "get" tag here? It seems very general.
##########
nifi-nar-bundles/nifi-workday-bundle/nifi-workday-processors/src/main/resources/docs/org.apache.nifi.processors.workday.GetWorkdayReport/additionalDetails.html:
##########
@@ -0,0 +1,103 @@
+<!DOCTYPE html>
+<html lang="en">
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<head>
+ <meta charset="utf-8"/>
+ <title>GetWorkdayReport</title>
+ <link rel="stylesheet" href="../../../../../css/component-usage.css" type="text/css"/>
+</head>
+
+<body>
+<h2>Summary</h2>
+<p>
+ This processor acts as a client endpoint to interact with the Workday API.
+ It is capable of reading reports from Workday RaaS and transferring the content directly to the output, or you can define
+ the required Record Reader and RecordSet Writer, so you can transform the report to the required format.
+</p>
+
+<h2>Supported report formats</h2>
+
+<ul>
+ <li>csv</li>
+ <li>simplexml</li>
+ <li>json</li>
+</ul>
+
+<p>
+ In case of json source you need to set the following parameters in the JsonTreeReader:
+ <ul>
+ <li>Starting Field Strategy: Nested Field</li>
+ <li>Starting Field Name: Report_Entry</li>
+ </ul>
+</p>
+
+<p>
+ It is possible to hide specific columns from the response if you define the Writer scheme explicitly in the configuration of the RecordSet Writer.
+</p>
+
+<h2>
+ Example: Remove name2 column from the response
+</h2>
+<p>
+ Let's say we have the following record structure:
+</p>
+<code>
+ <pre>
+ RecordSet (
+ Record (
+ Field "name1" = "value1",
+ Field "name2" = 42
+ ),
+ Record (
+ Field "name1" = "value2",
+ Field "name2" = 84
+ )
+ )
+ </pre>
+</code>
+
+<p>
+ If you would like to remove the "name2" column from the response, then you need to define the following writer schema:
+</p>
+
+<code>
+ <pre>
+ {
+ "name": "test",
+ "namespace": "nifi",
+ "type": "record",
+ "fields": [
+ { "name": "name1", "type": "string" }
+ ]
+ }
+ </pre>
+</code>
+
+<h2>
+ Example: Hash selected columns
+</h2>
+
+<p>
+ If you would like to keep a column in the response, but hash the value, you can do it by defining the record path selector and a hashing algorithm.
+ <ul>
+ <li>Columns to hash: /personalId,/emailAddress</li>
+ <li>Hashing algorithm: SHA-512</li>
+ </ul>
+ The above example hashes the PersonalId and emailAddress columns.
Review Comment:
```suggestion
The above example hashes the personalId and emailAddress columns.
```
##########
nifi-nar-bundles/nifi-workday-bundle/nifi-workday-processors/src/main/java/org/apache/nifi/processors/workday/GetWorkdayReport.java:
##########
@@ -0,0 +1,432 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.processors.workday;
+
+import static org.apache.nifi.expression.ExpressionLanguageScope.FLOWFILE_ATTRIBUTES;
+import static org.apache.nifi.expression.ExpressionLanguageScope.NONE;
+import static org.apache.nifi.processor.util.StandardValidators.NON_BLANK_VALIDATOR;
+import static org.apache.nifi.processor.util.StandardValidators.URL_VALIDATOR;
+
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Base64;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import org.apache.nifi.annotation.behavior.EventDriven;
+import org.apache.nifi.annotation.behavior.InputRequirement;
+import org.apache.nifi.annotation.behavior.InputRequirement.Requirement;
+import org.apache.nifi.annotation.behavior.SideEffectFree;
+import org.apache.nifi.annotation.behavior.SupportsBatching;
+import org.apache.nifi.annotation.behavior.WritesAttribute;
+import org.apache.nifi.annotation.behavior.WritesAttributes;
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.annotation.lifecycle.OnScheduled;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.ValidationContext;
+import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.flowfile.attributes.CoreAttributes;
+import org.apache.nifi.logging.ComponentLog;
+import org.apache.nifi.processor.AbstractProcessor;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.processor.util.StandardValidators;
+import org.apache.nifi.record.path.RecordPath;
+import org.apache.nifi.record.path.RecordPathResult;
+import org.apache.nifi.schema.access.SchemaNotFoundException;
+import org.apache.nifi.security.util.crypto.HashAlgorithm;
+import org.apache.nifi.security.util.crypto.HashService;
+import org.apache.nifi.serialization.MalformedRecordException;
+import org.apache.nifi.serialization.RecordReader;
+import org.apache.nifi.serialization.RecordReaderFactory;
+import org.apache.nifi.serialization.RecordSetWriter;
+import org.apache.nifi.serialization.RecordSetWriterFactory;
+import org.apache.nifi.serialization.record.Record;
+import org.apache.nifi.serialization.record.RecordFieldType;
+import org.apache.nifi.serialization.record.RecordSchema;
+import org.apache.nifi.web.client.api.HttpResponseEntity;
+import org.apache.nifi.web.client.api.WebClientService;
+import org.apache.nifi.web.client.provider.api.WebClientServiceProvider;
+
+@Tags({"Workday", "report", "get"})
+@InputRequirement(Requirement.INPUT_ALLOWED)
+@CapabilityDescription("A processor which can interact with a configurable Workday Report. The processor can forward the content without modification, or you can transform it by"
+ + " providing the specific Record Reader and Record Writer services based on your needs. You can also hash, or remove fields using the input parameters and schemes in the Record writer. "
+ + "Supported Workday report formats are: csv, simplexml, json")
+@EventDriven
+@SideEffectFree
+@SupportsBatching
+@WritesAttributes({
+ @WritesAttribute(attribute = GetWorkdayReport.GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS, description = "The Java exception class raised when the processor fails"),
+ @WritesAttribute(attribute = GetWorkdayReport.GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE, description = "The Java exception message raised when the processor fails"),
+ @WritesAttribute(attribute = "mime.type", description = "Sets the mime.type attribute to the MIME Type specified by the Source / Record Writer"),
+ @WritesAttribute(attribute= GetWorkdayReport.RECORD_COUNT, description = "The number of records in an outgoing FlowFile. This is only populated on the 'success' relationship "
+ + "when Record Reader and Writer is set.")})
+public class GetWorkdayReport extends AbstractProcessor {
+
+ protected static final String STATUS_CODE = "getworkdayreport.status.code";
+ protected static final String REQUEST_URL = "getworkdayreport.request.url";
+ protected static final String REQUEST_DURATION = "getworkdayreport.request.duration";
+ protected static final String TRANSACTION_ID = "getworkdayreport.tx.id";
+ protected static final String GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS = "getworkdayreport.java.exception.class";
+ protected static final String GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE = "getworkdayreport.java.exception.message";
+ protected static final String RECORD_COUNT = "record.count";
+ protected static final String BASIC_PREFIX = "Basic ";
+ protected static final String COLUMNS_TO_HASH_SEPARATOR = ",";
+ protected static final String HEADER_AUTHORIZATION = "Authorization";
+ protected static final String HEADER_CONTENT_TYPE = "Content-Type";
+ protected static final String USERNAME_PASSWORD_SEPARATOR = ":";
+
+ protected static final PropertyDescriptor REPORT_URL = new PropertyDescriptor.Builder()
+ .name("Workday report URL")
+ .displayName("Workday report URL")
+ .description("HTTP remote URL of Workday report including a scheme of http or https, as well as a hostname or IP address with optional port and path elements.")
+ .required(true)
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .addValidator(URL_VALIDATOR)
+ .build();
+
+ protected static final PropertyDescriptor WORKDAY_USERNAME = new PropertyDescriptor.Builder()
+ .name("Workday Username")
+ .displayName("Workday Username")
+ .description("The username provided for authentication of Workday requests. Encoded using Base64 for HTTP Basic Authentication as described in RFC 7617.")
+ .required(true)
+ .addValidator(StandardValidators.createRegexMatchingValidator(Pattern.compile("^[\\x20-\\x39\\x3b-\\x7e\\x80-\\xff]+$")))
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .build();
+
+ protected static final PropertyDescriptor WORKDAY_PASSWORD = new PropertyDescriptor.Builder()
+ .name("Workday Password")
+ .displayName("Workday Password")
+ .description("The password provided for authentication of Workday requests. Encoded using Base64 for HTTP Basic Authentication as described in RFC 7617.")
+ .required(true)
+ .sensitive(true)
+ .addValidator(StandardValidators.createRegexMatchingValidator(Pattern.compile("^[\\x20-\\x7e\\x80-\\xff]+$")))
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .build();
+
+ protected static final PropertyDescriptor WEB_CLIENT_SERVICE = new PropertyDescriptor.Builder()
+ .name("Standard Web Client Service")
+ .description("Web client which is used to communicate with the Workday API.")
+ .required(true)
+ .identifiesControllerService(WebClientServiceProvider.class)
+ .build();
+
+ protected static final PropertyDescriptor FIELDS_TO_HASH = new PropertyDescriptor.Builder()
+ .name("Fields to hash")
+ .displayName("Fields to hash")
+ .description("Comma separated record paths to replace with hash.")
+ .required(false)
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .addValidator(NON_BLANK_VALIDATOR)
+ .build();
+
+ protected static final PropertyDescriptor HASHING_ALGORITHM = new PropertyDescriptor.Builder()
+ .name("Hashing algorithm")
+ .displayName("Hashing algorithm")
+ .description("Determines what hashing algorithm should be used to perform the hashing function.")
+ .required(true)
+ .allowableValues(HashService.buildHashAlgorithmAllowableValues())
+ .defaultValue(HashAlgorithm.SHA256.getName())
+ .addValidator(NON_BLANK_VALIDATOR)
+ .expressionLanguageSupported(NONE)
+ .dependsOn(FIELDS_TO_HASH)
+ .build();
+
+ protected static final PropertyDescriptor RECORD_READER_FACTORY = new PropertyDescriptor.Builder()
+ .name("record-reader")
+ .displayName("Record Reader")
+ .description("Specifies the Controller Service to use for parsing incoming data and determining the data's schema.")
+ .identifiesControllerService(RecordReaderFactory.class)
+ .required(false)
+ .build();
+
+ protected static final PropertyDescriptor RECORD_WRITER_FACTORY = new PropertyDescriptor.Builder()
+ .name("record-writer")
+ .displayName("Record Writer")
+ .description("The Record Writer to use for serializing Records to an output FlowFile.")
+ .identifiesControllerService(RecordSetWriterFactory.class)
+ .dependsOn(RECORD_READER_FACTORY)
+ .required(true)
+ .build();
+
+ protected static final Relationship ORIGINAL = new Relationship.Builder()
+ .name("Original")
+ .description("Request FlowFiles transferred when receiving HTTP responses with a status code between 200 and 299.")
+ .build();
+
+ protected static final Relationship FAILURE = new Relationship.Builder()
+ .name("Failure")
+ .description("Request FlowFiles transferred when receiving socket communication errors.")
+ .build();
+
+ protected static final Relationship RESPONSE = new Relationship.Builder()
+ .name("Response")
+ .description("Response FlowFiles transferred when receiving HTTP responses with a status code between 200 and 299.")
+ .build();
+
+ protected static final Set<Relationship> RELATIONSHIPS = Collections.unmodifiableSet(new HashSet<>(Arrays.asList(ORIGINAL, RESPONSE, FAILURE)));
+ protected static final List<PropertyDescriptor> PROPERTIES = Collections.unmodifiableList(Arrays.asList(REPORT_URL, WORKDAY_USERNAME, WORKDAY_PASSWORD, WEB_CLIENT_SERVICE,
+ FIELDS_TO_HASH, HASHING_ALGORITHM, RECORD_READER_FACTORY, RECORD_WRITER_FACTORY));
+
+ private final AtomicReference<WebClientService> webClientAtomicReference = new AtomicReference<>();
+ private final AtomicReference<RecordReaderFactory> recordReaderFactoryAtomicReference = new AtomicReference<>();
+ private final AtomicReference<RecordSetWriterFactory> recordSetWriterFactoryAtomicReference = new AtomicReference<>();
+
+ @Override
+ protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
+ return PROPERTIES;
+ }
+
+ @Override
+ public Set<Relationship> getRelationships() {
+ return RELATIONSHIPS;
+ }
+
+ @OnScheduled
+ public void setUpClient(final ProcessContext context) {
+ WebClientServiceProvider standardWebClientServiceProvider = context.getProperty(WEB_CLIENT_SERVICE).asControllerService(WebClientServiceProvider.class);
+ RecordReaderFactory recordReaderFactory = context.getProperty(RECORD_READER_FACTORY).asControllerService(RecordReaderFactory.class);
+ RecordSetWriterFactory recordSetWriterFactory = context.getProperty(RECORD_WRITER_FACTORY).asControllerService(RecordSetWriterFactory.class);
+ WebClientService webClientService = standardWebClientServiceProvider.getWebClientService();
+ webClientAtomicReference.set(webClientService);
+ recordReaderFactoryAtomicReference.set(recordReaderFactory);
+ recordSetWriterFactoryAtomicReference.set(recordSetWriterFactory);
+ }
+
+ @Override
+ public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException {
+ FlowFile flowfile = session.get();
+
+ if (skipExecution(context, flowfile)) {
+ return;
+ }
+
+ ComponentLog logger = getLogger();
+ FlowFile responseFlowFile = null;
+
+ try {
+ WebClientService webClientService = webClientAtomicReference.get();
+ URI uri = new URI(context.getProperty(REPORT_URL).evaluateAttributeExpressions(flowfile).getValue().trim());
+ long startNanos = System.nanoTime();
+ String authorization = createAuthorizationHeader(context, flowfile);
+
+ try(HttpResponseEntity httpResponseEntity = webClientService.get().uri(uri).header(HEADER_AUTHORIZATION, authorization).retrieve()) {
+ responseFlowFile = createResponseFlowFile(flowfile, session, context, httpResponseEntity);
+ long elapsedTime = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos);
+ Map<String, String> commonAttributes = createCommonAttributes(uri, httpResponseEntity, elapsedTime);
+
+ if (flowfile != null) {
+ flowfile = session.putAllAttributes(flowfile, decorateWithMimeAttribute(commonAttributes, httpResponseEntity));
+ }
+ if (responseFlowFile != null) {
+ responseFlowFile = session.putAllAttributes(responseFlowFile, commonAttributes);
+ if (flowfile != null) {
+ session.getProvenanceReporter().fetch(responseFlowFile, uri.toString(), elapsedTime);
+ } else {
+ session.getProvenanceReporter().receive(responseFlowFile, uri.toString(), elapsedTime);
+ }
+ }
+
+ route(flowfile, responseFlowFile, session, context, httpResponseEntity.statusCode());
+ }
+ } catch (Exception e) {
+ if (flowfile == null) {
+ logger.error("Request Processing failed", e);
+ context.yield();
+ } else {
+ logger.error("Request Processing failed: {}", flowfile, e);
+ session.penalize(flowfile);
+ flowfile = session.putAttribute(flowfile, GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS, e.getClass().getName());
+ flowfile = session.putAttribute(flowfile, GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE, e.getMessage());
+ session.transfer(flowfile, FAILURE);
+ }
+
+ if (responseFlowFile != null) {
+ session.remove(responseFlowFile);
+ }
+ }
+ }
+
+ @Override
+ protected Collection<ValidationResult> customValidate(ValidationContext validationContext) {
+ List<ValidationResult> results = new ArrayList<>(super.customValidate(validationContext));
+ if (validationContext.getProperty(FIELDS_TO_HASH).isSet() && !validationContext.getProperty(RECORD_READER_FACTORY).isSet()) {
+ results.add(new ValidationResult.Builder()
+ .valid(false)
+ .explanation("Record-Reader and Record-Writer must be set if you would like to hash specific fields")
+ .subject("Workday report configuration")
+ .build());
+ }
+ return results;
+ }
+
+ /*
+ * If we have no FlowFile, and all incoming connections are self-loops then we can continue on.
+ * However, if we have no FlowFile and we have connections coming from other Processors, then
+ * we know that we should run only if we have a FlowFile.
+ */
+ private boolean skipExecution(ProcessContext context, FlowFile flowfile) {
+ return context.hasIncomingConnection() && flowfile == null && context.hasNonLoopConnection();
+ }
+
+ private FlowFile createResponseFlowFile(FlowFile flowfile, ProcessSession session, ProcessContext context, HttpResponseEntity httpResponseEntity)
+ throws IOException, SchemaNotFoundException, MalformedRecordException {
+ FlowFile responseFlowFile = null;
+ String hashingAlgorithm = context.getProperty(HASHING_ALGORITHM).getValue();
+ Set<String> columnsToHash = Optional.ofNullable(context.getProperty(FIELDS_TO_HASH).evaluateAttributeExpressions(flowfile).getValue()).map(String::trim)
+ .map(columns -> columns.split(COLUMNS_TO_HASH_SEPARATOR)).map(Arrays::stream).map(columns -> columns.collect(Collectors.toSet())).orElse(Collections.emptySet());
+ try {
+ if (isSuccess(httpResponseEntity.statusCode())) {
+ responseFlowFile = flowfile != null ? session.create(flowfile) : session.create();
+ InputStream responseBodyStream = httpResponseEntity.body();
+ if (recordReaderFactoryAtomicReference.get() != null) {
+ TransformResult transformResult = transformRecords(session, flowfile, responseFlowFile, hashingAlgorithm, columnsToHash, responseBodyStream);
+ Map<String, String> attributes = new HashMap<>();
+ attributes.put(RECORD_COUNT, String.valueOf(transformResult.getNumberOfRecords()));
+ attributes.put(CoreAttributes.MIME_TYPE.key(), transformResult.getMimeType());
+ responseFlowFile = session.putAllAttributes(responseFlowFile, attributes);
+ } else {
+ responseFlowFile = session.importFrom(responseBodyStream, responseFlowFile);
+ Optional<String> mimeType = httpResponseEntity.headers().getFirstHeader(HEADER_CONTENT_TYPE);
+ if (mimeType.isPresent()) {
+ responseFlowFile = session.putAttribute(responseFlowFile, CoreAttributes.MIME_TYPE.key(), mimeType.get());
+ }
+ }
+ }
+ } catch (Exception e) {
+ session.remove(responseFlowFile);
+ throw e;
+ }
+ return responseFlowFile;
+ }
+
+ private String createAuthorizationHeader(ProcessContext context, FlowFile flowfile) {
+ String userName = context.getProperty(WORKDAY_USERNAME).evaluateAttributeExpressions(flowfile).getValue();
+ String password = context.getProperty(WORKDAY_PASSWORD).evaluateAttributeExpressions(flowfile).getValue();
+ String base64Credential = Base64.getEncoder().encodeToString((userName + USERNAME_PASSWORD_SEPARATOR + password).getBytes(StandardCharsets.ISO_8859_1));
+ return BASIC_PREFIX + base64Credential;
+ }
+
+ private TransformResult transformRecords(ProcessSession session, FlowFile flowfile, FlowFile responseFlowFile, String hashingAlgorithm, Set<String> columnsToHash,
+ InputStream responseBodyStream) throws IOException, SchemaNotFoundException, MalformedRecordException {
+ int numberOfRecords = 0;
+ String mimeType = null;
+ try (RecordReader reader = recordReaderFactoryAtomicReference.get().createRecordReader(flowfile,
+ new BufferedInputStream(responseBodyStream), getLogger())) {
+ RecordSchema schema = recordSetWriterFactoryAtomicReference.get()
+ .getSchema(flowfile == null ? Collections.emptyMap() : flowfile.getAttributes(), reader.getSchema());
+ try (OutputStream responseStream = session.write(responseFlowFile);
+ RecordSetWriter recordSetWriter = recordSetWriterFactoryAtomicReference.get().createWriter(getLogger(), schema, responseStream, responseFlowFile)) {
+ mimeType = recordSetWriter.getMimeType();
+ recordSetWriter.beginRecordSet();
+ Record currentRecord;
+ while ((currentRecord = reader.nextRecord(false, true)) != null) {
Review Comment:
Is it possible to either add explanation what the "false,true" is or handle this via processor properties?
##########
nifi-nar-bundles/nifi-workday-bundle/nifi-workday-processors/src/main/java/org/apache/nifi/processors/workday/GetWorkdayReport.java:
##########
@@ -0,0 +1,432 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.processors.workday;
+
+import static org.apache.nifi.expression.ExpressionLanguageScope.FLOWFILE_ATTRIBUTES;
+import static org.apache.nifi.expression.ExpressionLanguageScope.NONE;
+import static org.apache.nifi.processor.util.StandardValidators.NON_BLANK_VALIDATOR;
+import static org.apache.nifi.processor.util.StandardValidators.URL_VALIDATOR;
+
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Base64;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import org.apache.nifi.annotation.behavior.EventDriven;
+import org.apache.nifi.annotation.behavior.InputRequirement;
+import org.apache.nifi.annotation.behavior.InputRequirement.Requirement;
+import org.apache.nifi.annotation.behavior.SideEffectFree;
+import org.apache.nifi.annotation.behavior.SupportsBatching;
+import org.apache.nifi.annotation.behavior.WritesAttribute;
+import org.apache.nifi.annotation.behavior.WritesAttributes;
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.annotation.lifecycle.OnScheduled;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.ValidationContext;
+import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.flowfile.attributes.CoreAttributes;
+import org.apache.nifi.logging.ComponentLog;
+import org.apache.nifi.processor.AbstractProcessor;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.processor.util.StandardValidators;
+import org.apache.nifi.record.path.RecordPath;
+import org.apache.nifi.record.path.RecordPathResult;
+import org.apache.nifi.schema.access.SchemaNotFoundException;
+import org.apache.nifi.security.util.crypto.HashAlgorithm;
+import org.apache.nifi.security.util.crypto.HashService;
+import org.apache.nifi.serialization.MalformedRecordException;
+import org.apache.nifi.serialization.RecordReader;
+import org.apache.nifi.serialization.RecordReaderFactory;
+import org.apache.nifi.serialization.RecordSetWriter;
+import org.apache.nifi.serialization.RecordSetWriterFactory;
+import org.apache.nifi.serialization.record.Record;
+import org.apache.nifi.serialization.record.RecordFieldType;
+import org.apache.nifi.serialization.record.RecordSchema;
+import org.apache.nifi.web.client.api.HttpResponseEntity;
+import org.apache.nifi.web.client.api.WebClientService;
+import org.apache.nifi.web.client.provider.api.WebClientServiceProvider;
+
+@Tags({"Workday", "report", "get"})
+@InputRequirement(Requirement.INPUT_ALLOWED)
+@CapabilityDescription("A processor which can interact with a configurable Workday Report. The processor can forward the content without modification, or you can transform it by"
+ + " providing the specific Record Reader and Record Writer services based on your needs. You can also hash, or remove fields using the input parameters and schemes in the Record writer. "
+ + "Supported Workday report formats are: csv, simplexml, json")
+@EventDriven
+@SideEffectFree
+@SupportsBatching
+@WritesAttributes({
+ @WritesAttribute(attribute = GetWorkdayReport.GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS, description = "The Java exception class raised when the processor fails"),
+ @WritesAttribute(attribute = GetWorkdayReport.GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE, description = "The Java exception message raised when the processor fails"),
+ @WritesAttribute(attribute = "mime.type", description = "Sets the mime.type attribute to the MIME Type specified by the Source / Record Writer"),
+ @WritesAttribute(attribute= GetWorkdayReport.RECORD_COUNT, description = "The number of records in an outgoing FlowFile. This is only populated on the 'success' relationship "
+ + "when Record Reader and Writer is set.")})
+public class GetWorkdayReport extends AbstractProcessor {
+
+ protected static final String STATUS_CODE = "getworkdayreport.status.code";
+ protected static final String REQUEST_URL = "getworkdayreport.request.url";
+ protected static final String REQUEST_DURATION = "getworkdayreport.request.duration";
+ protected static final String TRANSACTION_ID = "getworkdayreport.tx.id";
+ protected static final String GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS = "getworkdayreport.java.exception.class";
+ protected static final String GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE = "getworkdayreport.java.exception.message";
+ protected static final String RECORD_COUNT = "record.count";
+ protected static final String BASIC_PREFIX = "Basic ";
+ protected static final String COLUMNS_TO_HASH_SEPARATOR = ",";
+ protected static final String HEADER_AUTHORIZATION = "Authorization";
+ protected static final String HEADER_CONTENT_TYPE = "Content-Type";
+ protected static final String USERNAME_PASSWORD_SEPARATOR = ":";
+
+ protected static final PropertyDescriptor REPORT_URL = new PropertyDescriptor.Builder()
+ .name("Workday report URL")
+ .displayName("Workday report URL")
+ .description("HTTP remote URL of Workday report including a scheme of http or https, as well as a hostname or IP address with optional port and path elements.")
+ .required(true)
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .addValidator(URL_VALIDATOR)
+ .build();
+
+ protected static final PropertyDescriptor WORKDAY_USERNAME = new PropertyDescriptor.Builder()
+ .name("Workday Username")
+ .displayName("Workday Username")
+ .description("The username provided for authentication of Workday requests. Encoded using Base64 for HTTP Basic Authentication as described in RFC 7617.")
+ .required(true)
+ .addValidator(StandardValidators.createRegexMatchingValidator(Pattern.compile("^[\\x20-\\x39\\x3b-\\x7e\\x80-\\xff]+$")))
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .build();
+
+ protected static final PropertyDescriptor WORKDAY_PASSWORD = new PropertyDescriptor.Builder()
+ .name("Workday Password")
+ .displayName("Workday Password")
+ .description("The password provided for authentication of Workday requests. Encoded using Base64 for HTTP Basic Authentication as described in RFC 7617.")
+ .required(true)
+ .sensitive(true)
+ .addValidator(StandardValidators.createRegexMatchingValidator(Pattern.compile("^[\\x20-\\x7e\\x80-\\xff]+$")))
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .build();
+
+ protected static final PropertyDescriptor WEB_CLIENT_SERVICE = new PropertyDescriptor.Builder()
+ .name("Standard Web Client Service")
+ .description("Web client which is used to communicate with the Workday API.")
+ .required(true)
+ .identifiesControllerService(WebClientServiceProvider.class)
+ .build();
+
+ protected static final PropertyDescriptor FIELDS_TO_HASH = new PropertyDescriptor.Builder()
+ .name("Fields to hash")
+ .displayName("Fields to hash")
+ .description("Comma separated record paths to replace with hash.")
+ .required(false)
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .addValidator(NON_BLANK_VALIDATOR)
+ .build();
+
+ protected static final PropertyDescriptor HASHING_ALGORITHM = new PropertyDescriptor.Builder()
+ .name("Hashing algorithm")
+ .displayName("Hashing algorithm")
+ .description("Determines what hashing algorithm should be used to perform the hashing function.")
+ .required(true)
+ .allowableValues(HashService.buildHashAlgorithmAllowableValues())
+ .defaultValue(HashAlgorithm.SHA256.getName())
+ .addValidator(NON_BLANK_VALIDATOR)
+ .expressionLanguageSupported(NONE)
+ .dependsOn(FIELDS_TO_HASH)
+ .build();
+
+ protected static final PropertyDescriptor RECORD_READER_FACTORY = new PropertyDescriptor.Builder()
+ .name("record-reader")
+ .displayName("Record Reader")
+ .description("Specifies the Controller Service to use for parsing incoming data and determining the data's schema.")
+ .identifiesControllerService(RecordReaderFactory.class)
+ .required(false)
+ .build();
+
+ protected static final PropertyDescriptor RECORD_WRITER_FACTORY = new PropertyDescriptor.Builder()
+ .name("record-writer")
+ .displayName("Record Writer")
+ .description("The Record Writer to use for serializing Records to an output FlowFile.")
+ .identifiesControllerService(RecordSetWriterFactory.class)
+ .dependsOn(RECORD_READER_FACTORY)
+ .required(true)
+ .build();
+
+ protected static final Relationship ORIGINAL = new Relationship.Builder()
+ .name("Original")
+ .description("Request FlowFiles transferred when receiving HTTP responses with a status code between 200 and 299.")
+ .build();
+
+ protected static final Relationship FAILURE = new Relationship.Builder()
+ .name("Failure")
+ .description("Request FlowFiles transferred when receiving socket communication errors.")
+ .build();
+
+ protected static final Relationship RESPONSE = new Relationship.Builder()
+ .name("Response")
+ .description("Response FlowFiles transferred when receiving HTTP responses with a status code between 200 and 299.")
+ .build();
+
+ protected static final Set<Relationship> RELATIONSHIPS = Collections.unmodifiableSet(new HashSet<>(Arrays.asList(ORIGINAL, RESPONSE, FAILURE)));
+ protected static final List<PropertyDescriptor> PROPERTIES = Collections.unmodifiableList(Arrays.asList(REPORT_URL, WORKDAY_USERNAME, WORKDAY_PASSWORD, WEB_CLIENT_SERVICE,
+ FIELDS_TO_HASH, HASHING_ALGORITHM, RECORD_READER_FACTORY, RECORD_WRITER_FACTORY));
+
+ private final AtomicReference<WebClientService> webClientAtomicReference = new AtomicReference<>();
+ private final AtomicReference<RecordReaderFactory> recordReaderFactoryAtomicReference = new AtomicReference<>();
+ private final AtomicReference<RecordSetWriterFactory> recordSetWriterFactoryAtomicReference = new AtomicReference<>();
+
+ @Override
+ protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
+ return PROPERTIES;
+ }
+
+ @Override
+ public Set<Relationship> getRelationships() {
+ return RELATIONSHIPS;
+ }
+
+ @OnScheduled
+ public void setUpClient(final ProcessContext context) {
+ WebClientServiceProvider standardWebClientServiceProvider = context.getProperty(WEB_CLIENT_SERVICE).asControllerService(WebClientServiceProvider.class);
+ RecordReaderFactory recordReaderFactory = context.getProperty(RECORD_READER_FACTORY).asControllerService(RecordReaderFactory.class);
+ RecordSetWriterFactory recordSetWriterFactory = context.getProperty(RECORD_WRITER_FACTORY).asControllerService(RecordSetWriterFactory.class);
+ WebClientService webClientService = standardWebClientServiceProvider.getWebClientService();
+ webClientAtomicReference.set(webClientService);
+ recordReaderFactoryAtomicReference.set(recordReaderFactory);
+ recordSetWriterFactoryAtomicReference.set(recordSetWriterFactory);
+ }
+
+ @Override
+ public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException {
+ FlowFile flowfile = session.get();
+
+ if (skipExecution(context, flowfile)) {
+ return;
+ }
+
+ ComponentLog logger = getLogger();
+ FlowFile responseFlowFile = null;
+
+ try {
+ WebClientService webClientService = webClientAtomicReference.get();
+ URI uri = new URI(context.getProperty(REPORT_URL).evaluateAttributeExpressions(flowfile).getValue().trim());
+ long startNanos = System.nanoTime();
+ String authorization = createAuthorizationHeader(context, flowfile);
+
+ try(HttpResponseEntity httpResponseEntity = webClientService.get().uri(uri).header(HEADER_AUTHORIZATION, authorization).retrieve()) {
+ responseFlowFile = createResponseFlowFile(flowfile, session, context, httpResponseEntity);
+ long elapsedTime = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos);
+ Map<String, String> commonAttributes = createCommonAttributes(uri, httpResponseEntity, elapsedTime);
+
+ if (flowfile != null) {
+ flowfile = session.putAllAttributes(flowfile, decorateWithMimeAttribute(commonAttributes, httpResponseEntity));
+ }
+ if (responseFlowFile != null) {
+ responseFlowFile = session.putAllAttributes(responseFlowFile, commonAttributes);
+ if (flowfile != null) {
+ session.getProvenanceReporter().fetch(responseFlowFile, uri.toString(), elapsedTime);
+ } else {
+ session.getProvenanceReporter().receive(responseFlowFile, uri.toString(), elapsedTime);
+ }
+ }
+
+ route(flowfile, responseFlowFile, session, context, httpResponseEntity.statusCode());
+ }
+ } catch (Exception e) {
+ if (flowfile == null) {
+ logger.error("Request Processing failed", e);
+ context.yield();
+ } else {
+ logger.error("Request Processing failed: {}", flowfile, e);
+ session.penalize(flowfile);
+ flowfile = session.putAttribute(flowfile, GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS, e.getClass().getName());
+ flowfile = session.putAttribute(flowfile, GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE, e.getMessage());
+ session.transfer(flowfile, FAILURE);
+ }
+
+ if (responseFlowFile != null) {
+ session.remove(responseFlowFile);
+ }
+ }
+ }
+
+ @Override
+ protected Collection<ValidationResult> customValidate(ValidationContext validationContext) {
+ List<ValidationResult> results = new ArrayList<>(super.customValidate(validationContext));
+ if (validationContext.getProperty(FIELDS_TO_HASH).isSet() && !validationContext.getProperty(RECORD_READER_FACTORY).isSet()) {
+ results.add(new ValidationResult.Builder()
+ .valid(false)
+ .explanation("Record-Reader and Record-Writer must be set if you would like to hash specific fields")
+ .subject("Workday report configuration")
+ .build());
+ }
+ return results;
+ }
+
+ /*
+ * If we have no FlowFile, and all incoming connections are self-loops then we can continue on.
+ * However, if we have no FlowFile and we have connections coming from other Processors, then
+ * we know that we should run only if we have a FlowFile.
+ */
+ private boolean skipExecution(ProcessContext context, FlowFile flowfile) {
+ return context.hasIncomingConnection() && flowfile == null && context.hasNonLoopConnection();
+ }
+
+ private FlowFile createResponseFlowFile(FlowFile flowfile, ProcessSession session, ProcessContext context, HttpResponseEntity httpResponseEntity)
+ throws IOException, SchemaNotFoundException, MalformedRecordException {
+ FlowFile responseFlowFile = null;
+ String hashingAlgorithm = context.getProperty(HASHING_ALGORITHM).getValue();
+ Set<String> columnsToHash = Optional.ofNullable(context.getProperty(FIELDS_TO_HASH).evaluateAttributeExpressions(flowfile).getValue()).map(String::trim)
+ .map(columns -> columns.split(COLUMNS_TO_HASH_SEPARATOR)).map(Arrays::stream).map(columns -> columns.collect(Collectors.toSet())).orElse(Collections.emptySet());
+ try {
+ if (isSuccess(httpResponseEntity.statusCode())) {
+ responseFlowFile = flowfile != null ? session.create(flowfile) : session.create();
+ InputStream responseBodyStream = httpResponseEntity.body();
+ if (recordReaderFactoryAtomicReference.get() != null) {
+ TransformResult transformResult = transformRecords(session, flowfile, responseFlowFile, hashingAlgorithm, columnsToHash, responseBodyStream);
+ Map<String, String> attributes = new HashMap<>();
+ attributes.put(RECORD_COUNT, String.valueOf(transformResult.getNumberOfRecords()));
+ attributes.put(CoreAttributes.MIME_TYPE.key(), transformResult.getMimeType());
+ responseFlowFile = session.putAllAttributes(responseFlowFile, attributes);
+ } else {
+ responseFlowFile = session.importFrom(responseBodyStream, responseFlowFile);
+ Optional<String> mimeType = httpResponseEntity.headers().getFirstHeader(HEADER_CONTENT_TYPE);
+ if (mimeType.isPresent()) {
+ responseFlowFile = session.putAttribute(responseFlowFile, CoreAttributes.MIME_TYPE.key(), mimeType.get());
+ }
+ }
+ }
+ } catch (Exception e) {
+ session.remove(responseFlowFile);
+ throw e;
+ }
+ return responseFlowFile;
+ }
+
+ private String createAuthorizationHeader(ProcessContext context, FlowFile flowfile) {
+ String userName = context.getProperty(WORKDAY_USERNAME).evaluateAttributeExpressions(flowfile).getValue();
+ String password = context.getProperty(WORKDAY_PASSWORD).evaluateAttributeExpressions(flowfile).getValue();
+ String base64Credential = Base64.getEncoder().encodeToString((userName + USERNAME_PASSWORD_SEPARATOR + password).getBytes(StandardCharsets.ISO_8859_1));
+ return BASIC_PREFIX + base64Credential;
+ }
+
+ private TransformResult transformRecords(ProcessSession session, FlowFile flowfile, FlowFile responseFlowFile, String hashingAlgorithm, Set<String> columnsToHash,
+ InputStream responseBodyStream) throws IOException, SchemaNotFoundException, MalformedRecordException {
+ int numberOfRecords = 0;
+ String mimeType = null;
+ try (RecordReader reader = recordReaderFactoryAtomicReference.get().createRecordReader(flowfile,
+ new BufferedInputStream(responseBodyStream), getLogger())) {
+ RecordSchema schema = recordSetWriterFactoryAtomicReference.get()
+ .getSchema(flowfile == null ? Collections.emptyMap() : flowfile.getAttributes(), reader.getSchema());
+ try (OutputStream responseStream = session.write(responseFlowFile);
+ RecordSetWriter recordSetWriter = recordSetWriterFactoryAtomicReference.get().createWriter(getLogger(), schema, responseStream, responseFlowFile)) {
+ mimeType = recordSetWriter.getMimeType();
+ recordSetWriter.beginRecordSet();
+ Record currentRecord;
+ while ((currentRecord = reader.nextRecord(false, true)) != null) {
+ for (String recordPath : columnsToHash) {
+ RecordPathResult evaluate = RecordPath.compile("hash(" + recordPath + ", '" + hashingAlgorithm + "')").evaluate(currentRecord);
+ evaluate.getSelectedFields().forEach(fieldVal -> fieldVal.updateValue(fieldVal.getValue(), RecordFieldType.STRING.getDataType()));
+ }
+ currentRecord.incorporateInactiveFields();
+ recordSetWriter.write(currentRecord);
+ numberOfRecords++;
+ }
+ }
+ }
+ return new TransformResult(numberOfRecords, mimeType);
+ }
+
+ private void route(FlowFile request, FlowFile response, ProcessSession session, ProcessContext context, int statusCode) {
+ if (!isSuccess(statusCode) && request == null) {
+ context.yield();
+ }
+
+ if (isSuccess(statusCode)) {
+ if (request != null) {
+ session.transfer(request, ORIGINAL);
+ }
+ if (response != null) {
+ session.transfer(response, RESPONSE);
+ }
+ } else {
+ if (request != null) {
+ session.transfer(request, FAILURE);
+ }
+ }
+ }
+
+ private boolean isSuccess(int statusCode) {
+ return statusCode / 100 == 2;
Review Comment:
What do you think about making this a little more descriptive like
```
return statuscode >= 200 && statuscode < 300;
```
##########
nifi-nar-bundles/nifi-workday-bundle/nifi-workday-processors/src/main/java/org/apache/nifi/processors/workday/GetWorkdayReport.java:
##########
@@ -0,0 +1,432 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.processors.workday;
+
+import static org.apache.nifi.expression.ExpressionLanguageScope.FLOWFILE_ATTRIBUTES;
+import static org.apache.nifi.expression.ExpressionLanguageScope.NONE;
+import static org.apache.nifi.processor.util.StandardValidators.NON_BLANK_VALIDATOR;
+import static org.apache.nifi.processor.util.StandardValidators.URL_VALIDATOR;
+
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Base64;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import org.apache.nifi.annotation.behavior.EventDriven;
+import org.apache.nifi.annotation.behavior.InputRequirement;
+import org.apache.nifi.annotation.behavior.InputRequirement.Requirement;
+import org.apache.nifi.annotation.behavior.SideEffectFree;
+import org.apache.nifi.annotation.behavior.SupportsBatching;
+import org.apache.nifi.annotation.behavior.WritesAttribute;
+import org.apache.nifi.annotation.behavior.WritesAttributes;
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.annotation.lifecycle.OnScheduled;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.ValidationContext;
+import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.flowfile.attributes.CoreAttributes;
+import org.apache.nifi.logging.ComponentLog;
+import org.apache.nifi.processor.AbstractProcessor;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.processor.util.StandardValidators;
+import org.apache.nifi.record.path.RecordPath;
+import org.apache.nifi.record.path.RecordPathResult;
+import org.apache.nifi.schema.access.SchemaNotFoundException;
+import org.apache.nifi.security.util.crypto.HashAlgorithm;
+import org.apache.nifi.security.util.crypto.HashService;
+import org.apache.nifi.serialization.MalformedRecordException;
+import org.apache.nifi.serialization.RecordReader;
+import org.apache.nifi.serialization.RecordReaderFactory;
+import org.apache.nifi.serialization.RecordSetWriter;
+import org.apache.nifi.serialization.RecordSetWriterFactory;
+import org.apache.nifi.serialization.record.Record;
+import org.apache.nifi.serialization.record.RecordFieldType;
+import org.apache.nifi.serialization.record.RecordSchema;
+import org.apache.nifi.web.client.api.HttpResponseEntity;
+import org.apache.nifi.web.client.api.WebClientService;
+import org.apache.nifi.web.client.provider.api.WebClientServiceProvider;
+
+@Tags({"Workday", "report", "get"})
+@InputRequirement(Requirement.INPUT_ALLOWED)
+@CapabilityDescription("A processor which can interact with a configurable Workday Report. The processor can forward the content without modification, or you can transform it by"
+ + " providing the specific Record Reader and Record Writer services based on your needs. You can also hash, or remove fields using the input parameters and schemes in the Record writer. "
Review Comment:
```suggestion
+ " providing the specific Record Reader and Record Writer services based on your needs. You can also hash, or remove fields using the input parameters and schemes in the Record Writer. "
```
##########
nifi-nar-bundles/nifi-workday-bundle/nifi-workday-processors/src/test/java/org/apache/nifi/processors/workday/GetWorkdayReportTest.java:
##########
@@ -0,0 +1,438 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.processors.workday;
+
+import static org.apache.nifi.processors.workday.GetWorkdayReport.FAILURE;
+import static org.apache.nifi.processors.workday.GetWorkdayReport.FIELDS_TO_HASH;
+import static org.apache.nifi.processors.workday.GetWorkdayReport.GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS;
+import static org.apache.nifi.processors.workday.GetWorkdayReport.GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE;
+import static org.apache.nifi.processors.workday.GetWorkdayReport.HASHING_ALGORITHM;
+import static org.apache.nifi.processors.workday.GetWorkdayReport.HEADER_AUTHORIZATION;
+import static org.apache.nifi.processors.workday.GetWorkdayReport.ORIGINAL;
+import static org.apache.nifi.processors.workday.GetWorkdayReport.RECORD_COUNT;
+import static org.apache.nifi.processors.workday.GetWorkdayReport.RECORD_READER_FACTORY;
+import static org.apache.nifi.processors.workday.GetWorkdayReport.RECORD_WRITER_FACTORY;
+import static org.apache.nifi.processors.workday.GetWorkdayReport.RESPONSE;
+import static org.apache.nifi.processors.workday.GetWorkdayReport.STATUS_CODE;
+import static org.apache.nifi.processors.workday.GetWorkdayReport.WEB_CLIENT_SERVICE;
+import static org.junit.jupiter.api.Assertions.assertFalse;
+import static org.junit.jupiter.api.Assertions.assertNotNull;
+import static org.junit.jupiter.api.Assertions.assertTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.when;
+
+import java.io.IOException;
+import java.net.URISyntaxException;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.TimeUnit;
+import java.util.regex.Pattern;
+import okhttp3.mockwebserver.MockResponse;
+import okhttp3.mockwebserver.MockWebServer;
+import okhttp3.mockwebserver.RecordedRequest;
+import org.apache.nifi.csv.CSVRecordSetWriter;
+import org.apache.nifi.flowfile.attributes.CoreAttributes;
+import org.apache.nifi.json.JsonTreeReader;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.reporting.InitializationException;
+import org.apache.nifi.security.util.crypto.HashAlgorithm;
+import org.apache.nifi.security.util.crypto.HashService;
+import org.apache.nifi.serialization.RecordReaderFactory;
+import org.apache.nifi.serialization.RecordSetWriterFactory;
+import org.apache.nifi.util.MockFlowFile;
+import org.apache.nifi.util.MockProcessContext;
+import org.apache.nifi.util.TestRunner;
+import org.apache.nifi.util.TestRunners;
+import org.apache.nifi.web.client.provider.api.WebClientServiceProvider;
+import org.apache.nifi.web.client.provider.service.StandardWebClientServiceProvider;
+import org.junit.jupiter.api.AfterEach;
+import org.junit.jupiter.api.BeforeEach;
+import org.junit.jupiter.api.Test;
+
+class GetWorkdayReportTest {
+
+ private static final String LOCALHOST = "localhost";
+ private static final String REPORT_URL = "http://" + LOCALHOST;
+ private static final String INVALID_URL = "invalid";
+ private static final String FIELD_TO_HASH = "/name";
+ private static final String INVALID_URL_PARAM = ":invalid_url";
+ private static final String APPLICATION_JSON = "application/json";
+ private static final String OK_STATUS_CODE = "200";
+ private static final String CONTENT_TYPE = "Content-Type";
+ private static final String TEXT_CSV = "text/csv";
+ private static final String USER_NAME = "userName";
+ private static final String PASSWORD = "password";
+
+ private TestRunner runner;
+ private MockWebServer mockWebServer;
+
+ @BeforeEach
+ public void setRunner() {
+ runner = TestRunners.newTestRunner(new GetWorkdayReport());
+ mockWebServer = new MockWebServer();
+ }
+
+ @AfterEach
+ public void shutdownServer() throws IOException {
+ mockWebServer.shutdown();
+ }
+
+ @Test
+ public void testNotValidWithoutReportUrlProperty() throws InitializationException {
+ withWebClientService();
+ runner.setProperty(GetWorkdayReport.WORKDAY_USERNAME, USER_NAME);
+ runner.setProperty(GetWorkdayReport.WORKDAY_PASSWORD, PASSWORD);
+
+ runner.assertNotValid();
+ }
+
+ @Test
+ public void testNotValidWithInvalidReportUrlProperty() throws InitializationException {
+ withWebClientService();
+ runner.setProperty(GetWorkdayReport.WORKDAY_USERNAME, USER_NAME);
+ runner.setProperty(GetWorkdayReport.WORKDAY_PASSWORD, PASSWORD);
+ runner.setProperty(GetWorkdayReport.REPORT_URL, INVALID_URL);
+ runner.assertNotValid();
+ }
+
+ @Test
+ public void testNotValidWithoutUserName() throws InitializationException {
+ withWebClientService();
+ runner.setProperty(GetWorkdayReport.WORKDAY_PASSWORD, PASSWORD);
+ runner.setProperty(GetWorkdayReport.REPORT_URL, REPORT_URL);
+
+ runner.assertNotValid();
+ }
+
+ @Test
+ public void testNotValidWithoutPassword() throws InitializationException {
+ withWebClientService();
+ runner.setProperty(GetWorkdayReport.WORKDAY_USERNAME, USER_NAME);
+ runner.setProperty(GetWorkdayReport.REPORT_URL, REPORT_URL);
+
+ runner.assertNotValid();
+ }
+
+ @Test
+ public void testNotValidIfFieldsToHashIsGivenWithoutRecordReader() throws InitializationException {
+ withWebClientService();
+ withMockRecordSetWriterFactory();
+ runner.setProperty(GetWorkdayReport.WORKDAY_USERNAME, USER_NAME);
+ runner.setProperty(GetWorkdayReport.WORKDAY_PASSWORD, PASSWORD);
+ runner.setProperty(GetWorkdayReport.REPORT_URL, REPORT_URL);
+ runner.setProperty(FIELDS_TO_HASH, FIELD_TO_HASH);
+ runner.assertNotValid();
+ }
+
+ @Test
+ public void testNotValidIfFieldsToHashIsGivenWithoutRecordWriter() throws InitializationException {
+ withWebClientService();
+ withMockRecordReaderFactory();
+ runner.setProperty(GetWorkdayReport.WORKDAY_USERNAME, USER_NAME);
+ runner.setProperty(GetWorkdayReport.WORKDAY_PASSWORD, PASSWORD);
+ runner.setProperty(GetWorkdayReport.REPORT_URL, REPORT_URL);
+ runner.setProperty(FIELDS_TO_HASH, FIELD_TO_HASH);
+ runner.assertNotValid();
+ }
+
+ @Test
+ public void testRunIncomingConnectionsWithNonLoopConnections() throws InitializationException {
+ withWebClientService();
+ runner.setProperty(GetWorkdayReport.WORKDAY_USERNAME, USER_NAME);
+ runner.setProperty(GetWorkdayReport.WORKDAY_PASSWORD, PASSWORD);
+ runner.setProperty(GetWorkdayReport.REPORT_URL, REPORT_URL);
+ runner.setIncomingConnection(true);
+ runner.setNonLoopConnection(true);
+
+ runner.run();
+ runner.assertQueueEmpty();
+ }
+
+ @Test
+ public void testRunThrowsURISyntaxExceptionFailure() throws InitializationException {
+ withWebClientService();
+ runner.setProperty(GetWorkdayReport.WORKDAY_USERNAME, USER_NAME);
+ runner.setProperty(GetWorkdayReport.WORKDAY_PASSWORD, PASSWORD);
+
+ String urlAttributeKey = "request.url";
+ runner.setProperty(GetWorkdayReport.REPORT_URL, String.format("${%s}", urlAttributeKey));
+
+ Map<String, String> attributes = new HashMap<>();
+ attributes.put(urlAttributeKey, INVALID_URL_PARAM);
+
+ runner.enqueue("", attributes);
+ runner.run();
+
+ runner.assertAllFlowFilesTransferred(FAILURE);
+ runner.assertPenalizeCount(1);
+
+ MockFlowFile flowFile = getFlowFile(FAILURE);
+ flowFile.assertAttributeEquals(GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS, URISyntaxException.class.getName());
+ flowFile.assertAttributeExists(GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE);
+ }
+
+ @Test
+ void testContextYieldIfHttpStatusIsNot2xxAndThereIsNoIncomingConnection() throws InitializationException {
+ runner.setIncomingConnection(false);
+ runner.setProperty(GetWorkdayReport.WORKDAY_USERNAME, USER_NAME);
+ runner.setProperty(GetWorkdayReport.WORKDAY_PASSWORD, PASSWORD);
+ withWebClientService();
+ runner.setProperty(GetWorkdayReport.REPORT_URL, getMockWebServerUrl());
+
+ mockWebServer.enqueue(new MockResponse().setResponseCode(500));
+
+ runner.run();
+
+ assertTrue(((MockProcessContext) runner.getProcessContext()).isYieldCalled());
+ runner.assertTransferCount(ORIGINAL, 0);
+ runner.assertTransferCount(RESPONSE, 0);
+ runner.assertTransferCount(FAILURE, 0);
+ }
+
+ @Test
+ void testContextYieldAndForwardFlowFileToFailureIfHttpStatusIsNot2xxAndThereIsIncomingConnection() throws InitializationException {
+ runner.setIncomingConnection(true);
+ runner.setProperty(GetWorkdayReport.WORKDAY_USERNAME, USER_NAME);
+ runner.setProperty(GetWorkdayReport.WORKDAY_PASSWORD, PASSWORD);
+ withWebClientService();
+ runner.setProperty(GetWorkdayReport.REPORT_URL, getMockWebServerUrl());
+
+ mockWebServer.enqueue(new MockResponse().setResponseCode(500));
+
+ runner.enqueue("test");
+ runner.run();
+
+ assertFalse(((MockProcessContext) runner.getProcessContext()).isYieldCalled());
+ runner.assertTransferCount(ORIGINAL, 0);
+ runner.assertTransferCount(RESPONSE, 0);
+ runner.assertTransferCount(FAILURE, 1);
+
+ final MockFlowFile flowFile = runner.getFlowFilesForRelationship(FAILURE).iterator().next();
+ flowFile.assertAttributeEquals("getworkdayreport.status.code", "500");
+ }
+
+ @Test
+ void testYieldShouldBeCalledWhenExceptionHappensAndThereIsNoRequestFlowFile() throws InitializationException {
+ runner.setIncomingConnection(false);
+ runner.setProperty(GetWorkdayReport.WORKDAY_USERNAME, USER_NAME);
+ runner.setProperty(GetWorkdayReport.WORKDAY_PASSWORD, PASSWORD);
+ withWebClientService();
+ String urlAttributeKey = "request.url";
+ runner.setProperty(GetWorkdayReport.REPORT_URL, String.format("${%s}", urlAttributeKey));
+
+ runner.run();
+
+ assertTrue(((MockProcessContext) runner.getProcessContext()).isYieldCalled());
+ runner.assertTransferCount(ORIGINAL, 0);
+ runner.assertTransferCount(RESPONSE, 0);
+ runner.assertTransferCount(FAILURE, 0);
+ }
+
+ @Test
+ void testPassThroughContentWithoutModificationIfNoRecordReaderAndWriterDefined() throws InitializationException {
+ withWebClientService();
+ runner.setProperty(GetWorkdayReport.WORKDAY_USERNAME, USER_NAME);
+ runner.setProperty(GetWorkdayReport.WORKDAY_PASSWORD, PASSWORD);
+ runner.setIncomingConnection(false);
+ runner.setProperty(GetWorkdayReport.REPORT_URL, getMockWebServerUrl());
+
+ String content = "id,name\n1,2";
+ mockWebServer.enqueue(new MockResponse().setResponseCode(200).setBody(content).setHeader(CONTENT_TYPE, TEXT_CSV));
+
+ runner.run();
+
+ assertFalse(((MockProcessContext) runner.getProcessContext()).isYieldCalled());
+ runner.assertTransferCount(ORIGINAL, 0);
+ runner.assertTransferCount(RESPONSE, 1);
+ runner.assertTransferCount(FAILURE, 0);
+
+ MockFlowFile flowFile = runner.getFlowFilesForRelationship(RESPONSE).iterator().next();
+ flowFile.assertAttributeEquals(STATUS_CODE, OK_STATUS_CODE);
+ flowFile.assertAttributeEquals(CoreAttributes.MIME_TYPE.key(), TEXT_CSV);
+ flowFile.assertAttributeNotExists(RECORD_COUNT);
+ flowFile.assertContentEquals(content);
+ }
+
+ @Test
+ void testRequestFlowFileIsTransferredToOriginalRelationship() throws InitializationException {
+ withWebClientService();
+ runner.setProperty(GetWorkdayReport.WORKDAY_USERNAME, USER_NAME);
+ runner.setProperty(GetWorkdayReport.WORKDAY_PASSWORD, PASSWORD);
+ runner.setIncomingConnection(true);
+ runner.setProperty(GetWorkdayReport.REPORT_URL, getMockWebServerUrl());
+
+ String content = "id,name\n1,2";
+ mockWebServer.enqueue(new MockResponse().setResponseCode(200).setBody(content).setHeader(CONTENT_TYPE, TEXT_CSV));
+ runner.enqueue("");
+
+ runner.run();
+
+ assertFalse(((MockProcessContext) runner.getProcessContext()).isYieldCalled());
+ runner.assertTransferCount(ORIGINAL, 1);
+ runner.assertTransferCount(RESPONSE, 1);
+ runner.assertTransferCount(FAILURE, 0);
+
+ MockFlowFile originalFlowFile = runner.getFlowFilesForRelationship(ORIGINAL).iterator().next();
+ MockFlowFile responseFlowFile = runner.getFlowFilesForRelationship(RESPONSE).iterator().next();
+ originalFlowFile.assertAttributeEquals(STATUS_CODE, OK_STATUS_CODE);
+ originalFlowFile.assertAttributeEquals(CoreAttributes.MIME_TYPE.key(), TEXT_CSV);
+ responseFlowFile.assertAttributeEquals(STATUS_CODE, OK_STATUS_CODE);
+ responseFlowFile.assertAttributeEquals(CoreAttributes.MIME_TYPE.key(), TEXT_CSV);
+ responseFlowFile.assertAttributeNotExists(RECORD_COUNT);
+ responseFlowFile.assertContentEquals(content);
+ }
+
+ @Test
+ void testContentIsTransformedIfRecordReaderAndWriterIsDefined() throws InitializationException {
+ withWebClientService();
+ withJsonRecordReader();
+ withCsvRecordSetWriter();
+ runner.setProperty(GetWorkdayReport.WORKDAY_USERNAME, USER_NAME);
+ runner.setProperty(GetWorkdayReport.WORKDAY_PASSWORD, PASSWORD);
+ runner.setIncomingConnection(false);
+ runner.setProperty(GetWorkdayReport.REPORT_URL, getMockWebServerUrl());
+
+ String jsonContent = "{\"id\": 1, \"name\": \"test\"}";
+ String csvContent = "id,name\n1,test\n";
+ mockWebServer.enqueue(new MockResponse().setResponseCode(200).setBody(jsonContent).setHeader(CONTENT_TYPE, APPLICATION_JSON));
+
+ runner.run();
+
+ assertFalse(((MockProcessContext) runner.getProcessContext()).isYieldCalled());
+ runner.assertTransferCount(ORIGINAL, 0);
+ runner.assertTransferCount(RESPONSE, 1);
+ runner.assertTransferCount(FAILURE, 0);
+
+ MockFlowFile flowFile = runner.getFlowFilesForRelationship(RESPONSE).iterator().next();
+ flowFile.assertAttributeEquals(STATUS_CODE, OK_STATUS_CODE);
+ flowFile.assertAttributeEquals(CoreAttributes.MIME_TYPE.key(), TEXT_CSV);
+
+ flowFile.assertAttributeEquals(RECORD_COUNT, "1");
+ flowFile.assertContentEquals(csvContent);
+ }
+
+ @Test
+ void testAttributeIsHashed() throws InitializationException {
Review Comment:
Nice test!
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@nifi.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [nifi] ferencerdei commented on a diff in pull request #6376: NIFI-10455 Get workday report processor
Posted by GitBox <gi...@apache.org>.
ferencerdei commented on code in PR #6376:
URL: https://github.com/apache/nifi/pull/6376#discussion_r968137393
##########
nifi-nar-bundles/nifi-workday-bundle/nifi-workday-processors/src/main/java/org/apache/nifi/processors/workday/GetWorkdayReport.java:
##########
@@ -0,0 +1,432 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.processors.workday;
+
+import static org.apache.nifi.expression.ExpressionLanguageScope.FLOWFILE_ATTRIBUTES;
+import static org.apache.nifi.expression.ExpressionLanguageScope.NONE;
+import static org.apache.nifi.processor.util.StandardValidators.NON_BLANK_VALIDATOR;
+import static org.apache.nifi.processor.util.StandardValidators.URL_VALIDATOR;
+
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Base64;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import org.apache.nifi.annotation.behavior.EventDriven;
+import org.apache.nifi.annotation.behavior.InputRequirement;
+import org.apache.nifi.annotation.behavior.InputRequirement.Requirement;
+import org.apache.nifi.annotation.behavior.SideEffectFree;
+import org.apache.nifi.annotation.behavior.SupportsBatching;
+import org.apache.nifi.annotation.behavior.WritesAttribute;
+import org.apache.nifi.annotation.behavior.WritesAttributes;
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.annotation.lifecycle.OnScheduled;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.ValidationContext;
+import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.flowfile.attributes.CoreAttributes;
+import org.apache.nifi.logging.ComponentLog;
+import org.apache.nifi.processor.AbstractProcessor;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.processor.util.StandardValidators;
+import org.apache.nifi.record.path.RecordPath;
+import org.apache.nifi.record.path.RecordPathResult;
+import org.apache.nifi.schema.access.SchemaNotFoundException;
+import org.apache.nifi.security.util.crypto.HashAlgorithm;
+import org.apache.nifi.security.util.crypto.HashService;
+import org.apache.nifi.serialization.MalformedRecordException;
+import org.apache.nifi.serialization.RecordReader;
+import org.apache.nifi.serialization.RecordReaderFactory;
+import org.apache.nifi.serialization.RecordSetWriter;
+import org.apache.nifi.serialization.RecordSetWriterFactory;
+import org.apache.nifi.serialization.record.Record;
+import org.apache.nifi.serialization.record.RecordFieldType;
+import org.apache.nifi.serialization.record.RecordSchema;
+import org.apache.nifi.web.client.api.HttpResponseEntity;
+import org.apache.nifi.web.client.api.WebClientService;
+import org.apache.nifi.web.client.provider.api.WebClientServiceProvider;
+
+@Tags({"Workday", "report", "get"})
+@InputRequirement(Requirement.INPUT_ALLOWED)
+@CapabilityDescription("A processor which can interact with a configurable Workday Report. The processor can forward the content without modification, or you can transform it by"
+ + " providing the specific Record Reader and Record Writer services based on your needs. You can also hash, or remove fields using the input parameters and schemes in the Record writer. "
+ + "Supported Workday report formats are: csv, simplexml, json")
+@EventDriven
+@SideEffectFree
+@SupportsBatching
+@WritesAttributes({
+ @WritesAttribute(attribute = GetWorkdayReport.GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS, description = "The Java exception class raised when the processor fails"),
+ @WritesAttribute(attribute = GetWorkdayReport.GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE, description = "The Java exception message raised when the processor fails"),
+ @WritesAttribute(attribute = "mime.type", description = "Sets the mime.type attribute to the MIME Type specified by the Source / Record Writer"),
+ @WritesAttribute(attribute= GetWorkdayReport.RECORD_COUNT, description = "The number of records in an outgoing FlowFile. This is only populated on the 'success' relationship "
+ + "when Record Reader and Writer is set.")})
+public class GetWorkdayReport extends AbstractProcessor {
+
+ protected static final String STATUS_CODE = "getworkdayreport.status.code";
+ protected static final String REQUEST_URL = "getworkdayreport.request.url";
+ protected static final String REQUEST_DURATION = "getworkdayreport.request.duration";
+ protected static final String TRANSACTION_ID = "getworkdayreport.tx.id";
+ protected static final String GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS = "getworkdayreport.java.exception.class";
+ protected static final String GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE = "getworkdayreport.java.exception.message";
+ protected static final String RECORD_COUNT = "record.count";
+ protected static final String BASIC_PREFIX = "Basic ";
+ protected static final String COLUMNS_TO_HASH_SEPARATOR = ",";
+ protected static final String HEADER_AUTHORIZATION = "Authorization";
+ protected static final String HEADER_CONTENT_TYPE = "Content-Type";
+ protected static final String USERNAME_PASSWORD_SEPARATOR = ":";
+
+ protected static final PropertyDescriptor REPORT_URL = new PropertyDescriptor.Builder()
+ .name("Workday report URL")
+ .displayName("Workday report URL")
+ .description("HTTP remote URL of Workday report including a scheme of http or https, as well as a hostname or IP address with optional port and path elements.")
+ .required(true)
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .addValidator(URL_VALIDATOR)
+ .build();
+
+ protected static final PropertyDescriptor WORKDAY_USERNAME = new PropertyDescriptor.Builder()
+ .name("Workday Username")
+ .displayName("Workday Username")
+ .description("The username provided for authentication of Workday requests. Encoded using Base64 for HTTP Basic Authentication as described in RFC 7617.")
+ .required(true)
+ .addValidator(StandardValidators.createRegexMatchingValidator(Pattern.compile("^[\\x20-\\x39\\x3b-\\x7e\\x80-\\xff]+$")))
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .build();
+
+ protected static final PropertyDescriptor WORKDAY_PASSWORD = new PropertyDescriptor.Builder()
+ .name("Workday Password")
+ .displayName("Workday Password")
+ .description("The password provided for authentication of Workday requests. Encoded using Base64 for HTTP Basic Authentication as described in RFC 7617.")
+ .required(true)
+ .sensitive(true)
+ .addValidator(StandardValidators.createRegexMatchingValidator(Pattern.compile("^[\\x20-\\x7e\\x80-\\xff]+$")))
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .build();
+
+ protected static final PropertyDescriptor WEB_CLIENT_SERVICE = new PropertyDescriptor.Builder()
+ .name("Standard Web Client Service")
+ .description("Web client which is used to communicate with the Workday API.")
+ .required(true)
+ .identifiesControllerService(WebClientServiceProvider.class)
+ .build();
+
+ protected static final PropertyDescriptor FIELDS_TO_HASH = new PropertyDescriptor.Builder()
+ .name("Fields to hash")
+ .displayName("Fields to hash")
+ .description("Comma separated record paths to replace with hash.")
+ .required(false)
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .addValidator(NON_BLANK_VALIDATOR)
+ .build();
+
+ protected static final PropertyDescriptor HASHING_ALGORITHM = new PropertyDescriptor.Builder()
+ .name("Hashing algorithm")
+ .displayName("Hashing algorithm")
+ .description("Determines what hashing algorithm should be used to perform the hashing function.")
+ .required(true)
+ .allowableValues(HashService.buildHashAlgorithmAllowableValues())
+ .defaultValue(HashAlgorithm.SHA256.getName())
+ .addValidator(NON_BLANK_VALIDATOR)
+ .expressionLanguageSupported(NONE)
+ .dependsOn(FIELDS_TO_HASH)
+ .build();
+
+ protected static final PropertyDescriptor RECORD_READER_FACTORY = new PropertyDescriptor.Builder()
+ .name("record-reader")
+ .displayName("Record Reader")
+ .description("Specifies the Controller Service to use for parsing incoming data and determining the data's schema.")
+ .identifiesControllerService(RecordReaderFactory.class)
+ .required(false)
+ .build();
+
+ protected static final PropertyDescriptor RECORD_WRITER_FACTORY = new PropertyDescriptor.Builder()
+ .name("record-writer")
+ .displayName("Record Writer")
+ .description("The Record Writer to use for serializing Records to an output FlowFile.")
+ .identifiesControllerService(RecordSetWriterFactory.class)
+ .dependsOn(RECORD_READER_FACTORY)
+ .required(true)
+ .build();
+
+ protected static final Relationship ORIGINAL = new Relationship.Builder()
+ .name("Original")
+ .description("Request FlowFiles transferred when receiving HTTP responses with a status code between 200 and 299.")
+ .build();
+
+ protected static final Relationship FAILURE = new Relationship.Builder()
+ .name("Failure")
+ .description("Request FlowFiles transferred when receiving socket communication errors.")
+ .build();
+
+ protected static final Relationship RESPONSE = new Relationship.Builder()
+ .name("Response")
+ .description("Response FlowFiles transferred when receiving HTTP responses with a status code between 200 and 299.")
+ .build();
+
+ protected static final Set<Relationship> RELATIONSHIPS = Collections.unmodifiableSet(new HashSet<>(Arrays.asList(ORIGINAL, RESPONSE, FAILURE)));
+ protected static final List<PropertyDescriptor> PROPERTIES = Collections.unmodifiableList(Arrays.asList(REPORT_URL, WORKDAY_USERNAME, WORKDAY_PASSWORD, WEB_CLIENT_SERVICE,
+ FIELDS_TO_HASH, HASHING_ALGORITHM, RECORD_READER_FACTORY, RECORD_WRITER_FACTORY));
+
+ private final AtomicReference<WebClientService> webClientAtomicReference = new AtomicReference<>();
+ private final AtomicReference<RecordReaderFactory> recordReaderFactoryAtomicReference = new AtomicReference<>();
+ private final AtomicReference<RecordSetWriterFactory> recordSetWriterFactoryAtomicReference = new AtomicReference<>();
+
+ @Override
+ protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
+ return PROPERTIES;
+ }
+
+ @Override
+ public Set<Relationship> getRelationships() {
+ return RELATIONSHIPS;
+ }
+
+ @OnScheduled
+ public void setUpClient(final ProcessContext context) {
+ WebClientServiceProvider standardWebClientServiceProvider = context.getProperty(WEB_CLIENT_SERVICE).asControllerService(WebClientServiceProvider.class);
+ RecordReaderFactory recordReaderFactory = context.getProperty(RECORD_READER_FACTORY).asControllerService(RecordReaderFactory.class);
+ RecordSetWriterFactory recordSetWriterFactory = context.getProperty(RECORD_WRITER_FACTORY).asControllerService(RecordSetWriterFactory.class);
+ WebClientService webClientService = standardWebClientServiceProvider.getWebClientService();
+ webClientAtomicReference.set(webClientService);
+ recordReaderFactoryAtomicReference.set(recordReaderFactory);
+ recordSetWriterFactoryAtomicReference.set(recordSetWriterFactory);
+ }
+
+ @Override
+ public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException {
+ FlowFile flowfile = session.get();
+
+ if (skipExecution(context, flowfile)) {
+ return;
+ }
+
+ ComponentLog logger = getLogger();
+ FlowFile responseFlowFile = null;
+
+ try {
+ WebClientService webClientService = webClientAtomicReference.get();
+ URI uri = new URI(context.getProperty(REPORT_URL).evaluateAttributeExpressions(flowfile).getValue().trim());
+ long startNanos = System.nanoTime();
+ String authorization = createAuthorizationHeader(context, flowfile);
+
+ try(HttpResponseEntity httpResponseEntity = webClientService.get().uri(uri).header(HEADER_AUTHORIZATION, authorization).retrieve()) {
+ responseFlowFile = createResponseFlowFile(flowfile, session, context, httpResponseEntity);
+ long elapsedTime = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos);
+ Map<String, String> commonAttributes = createCommonAttributes(uri, httpResponseEntity, elapsedTime);
+
+ if (flowfile != null) {
+ flowfile = session.putAllAttributes(flowfile, decorateWithMimeAttribute(commonAttributes, httpResponseEntity));
+ }
+ if (responseFlowFile != null) {
+ responseFlowFile = session.putAllAttributes(responseFlowFile, commonAttributes);
+ if (flowfile != null) {
+ session.getProvenanceReporter().fetch(responseFlowFile, uri.toString(), elapsedTime);
+ } else {
+ session.getProvenanceReporter().receive(responseFlowFile, uri.toString(), elapsedTime);
+ }
+ }
+
+ route(flowfile, responseFlowFile, session, context, httpResponseEntity.statusCode());
+ }
+ } catch (Exception e) {
+ if (flowfile == null) {
+ logger.error("Request Processing failed", e);
+ context.yield();
+ } else {
+ logger.error("Request Processing failed: {}", flowfile, e);
+ session.penalize(flowfile);
+ flowfile = session.putAttribute(flowfile, GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS, e.getClass().getName());
+ flowfile = session.putAttribute(flowfile, GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE, e.getMessage());
+ session.transfer(flowfile, FAILURE);
+ }
+
+ if (responseFlowFile != null) {
+ session.remove(responseFlowFile);
+ }
+ }
+ }
+
+ @Override
+ protected Collection<ValidationResult> customValidate(ValidationContext validationContext) {
+ List<ValidationResult> results = new ArrayList<>(super.customValidate(validationContext));
+ if (validationContext.getProperty(FIELDS_TO_HASH).isSet() && !validationContext.getProperty(RECORD_READER_FACTORY).isSet()) {
+ results.add(new ValidationResult.Builder()
+ .valid(false)
+ .explanation("Record-Reader and Record-Writer must be set if you would like to hash specific fields")
+ .subject("Workday report configuration")
+ .build());
+ }
+ return results;
+ }
+
+ /*
+ * If we have no FlowFile, and all incoming connections are self-loops then we can continue on.
+ * However, if we have no FlowFile and we have connections coming from other Processors, then
+ * we know that we should run only if we have a FlowFile.
+ */
+ private boolean skipExecution(ProcessContext context, FlowFile flowfile) {
+ return context.hasIncomingConnection() && flowfile == null && context.hasNonLoopConnection();
+ }
+
+ private FlowFile createResponseFlowFile(FlowFile flowfile, ProcessSession session, ProcessContext context, HttpResponseEntity httpResponseEntity)
+ throws IOException, SchemaNotFoundException, MalformedRecordException {
+ FlowFile responseFlowFile = null;
+ String hashingAlgorithm = context.getProperty(HASHING_ALGORITHM).getValue();
+ Set<String> columnsToHash = Optional.ofNullable(context.getProperty(FIELDS_TO_HASH).evaluateAttributeExpressions(flowfile).getValue()).map(String::trim)
+ .map(columns -> columns.split(COLUMNS_TO_HASH_SEPARATOR)).map(Arrays::stream).map(columns -> columns.collect(Collectors.toSet())).orElse(Collections.emptySet());
+ try {
+ if (isSuccess(httpResponseEntity.statusCode())) {
+ responseFlowFile = flowfile != null ? session.create(flowfile) : session.create();
+ InputStream responseBodyStream = httpResponseEntity.body();
+ if (recordReaderFactoryAtomicReference.get() != null) {
+ TransformResult transformResult = transformRecords(session, flowfile, responseFlowFile, hashingAlgorithm, columnsToHash, responseBodyStream);
+ Map<String, String> attributes = new HashMap<>();
+ attributes.put(RECORD_COUNT, String.valueOf(transformResult.getNumberOfRecords()));
+ attributes.put(CoreAttributes.MIME_TYPE.key(), transformResult.getMimeType());
+ responseFlowFile = session.putAllAttributes(responseFlowFile, attributes);
+ } else {
+ responseFlowFile = session.importFrom(responseBodyStream, responseFlowFile);
+ Optional<String> mimeType = httpResponseEntity.headers().getFirstHeader(HEADER_CONTENT_TYPE);
+ if (mimeType.isPresent()) {
+ responseFlowFile = session.putAttribute(responseFlowFile, CoreAttributes.MIME_TYPE.key(), mimeType.get());
+ }
+ }
+ }
+ } catch (Exception e) {
+ session.remove(responseFlowFile);
+ throw e;
+ }
+ return responseFlowFile;
+ }
+
+ private String createAuthorizationHeader(ProcessContext context, FlowFile flowfile) {
+ String userName = context.getProperty(WORKDAY_USERNAME).evaluateAttributeExpressions(flowfile).getValue();
+ String password = context.getProperty(WORKDAY_PASSWORD).evaluateAttributeExpressions(flowfile).getValue();
+ String base64Credential = Base64.getEncoder().encodeToString((userName + USERNAME_PASSWORD_SEPARATOR + password).getBytes(StandardCharsets.ISO_8859_1));
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@nifi.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [nifi] ferencerdei commented on a diff in pull request #6376: NIFI-10455 Get workday report processor
Posted by GitBox <gi...@apache.org>.
ferencerdei commented on code in PR #6376:
URL: https://github.com/apache/nifi/pull/6376#discussion_r968132987
##########
nifi-nar-bundles/nifi-workday-bundle/nifi-workday-processors/src/main/java/org/apache/nifi/processors/workday/GetWorkdayReport.java:
##########
@@ -0,0 +1,432 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.processors.workday;
+
+import static org.apache.nifi.expression.ExpressionLanguageScope.FLOWFILE_ATTRIBUTES;
+import static org.apache.nifi.expression.ExpressionLanguageScope.NONE;
+import static org.apache.nifi.processor.util.StandardValidators.NON_BLANK_VALIDATOR;
+import static org.apache.nifi.processor.util.StandardValidators.URL_VALIDATOR;
+
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Base64;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import org.apache.nifi.annotation.behavior.EventDriven;
+import org.apache.nifi.annotation.behavior.InputRequirement;
+import org.apache.nifi.annotation.behavior.InputRequirement.Requirement;
+import org.apache.nifi.annotation.behavior.SideEffectFree;
+import org.apache.nifi.annotation.behavior.SupportsBatching;
+import org.apache.nifi.annotation.behavior.WritesAttribute;
+import org.apache.nifi.annotation.behavior.WritesAttributes;
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.annotation.lifecycle.OnScheduled;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.ValidationContext;
+import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.flowfile.attributes.CoreAttributes;
+import org.apache.nifi.logging.ComponentLog;
+import org.apache.nifi.processor.AbstractProcessor;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.processor.util.StandardValidators;
+import org.apache.nifi.record.path.RecordPath;
+import org.apache.nifi.record.path.RecordPathResult;
+import org.apache.nifi.schema.access.SchemaNotFoundException;
+import org.apache.nifi.security.util.crypto.HashAlgorithm;
+import org.apache.nifi.security.util.crypto.HashService;
+import org.apache.nifi.serialization.MalformedRecordException;
+import org.apache.nifi.serialization.RecordReader;
+import org.apache.nifi.serialization.RecordReaderFactory;
+import org.apache.nifi.serialization.RecordSetWriter;
+import org.apache.nifi.serialization.RecordSetWriterFactory;
+import org.apache.nifi.serialization.record.Record;
+import org.apache.nifi.serialization.record.RecordFieldType;
+import org.apache.nifi.serialization.record.RecordSchema;
+import org.apache.nifi.web.client.api.HttpResponseEntity;
+import org.apache.nifi.web.client.api.WebClientService;
+import org.apache.nifi.web.client.provider.api.WebClientServiceProvider;
+
+@Tags({"Workday", "report", "get"})
+@InputRequirement(Requirement.INPUT_ALLOWED)
+@CapabilityDescription("A processor which can interact with a configurable Workday Report. The processor can forward the content without modification, or you can transform it by"
+ + " providing the specific Record Reader and Record Writer services based on your needs. You can also hash, or remove fields using the input parameters and schemes in the Record writer. "
+ + "Supported Workday report formats are: csv, simplexml, json")
+@EventDriven
+@SideEffectFree
+@SupportsBatching
+@WritesAttributes({
+ @WritesAttribute(attribute = GetWorkdayReport.GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS, description = "The Java exception class raised when the processor fails"),
+ @WritesAttribute(attribute = GetWorkdayReport.GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE, description = "The Java exception message raised when the processor fails"),
+ @WritesAttribute(attribute = "mime.type", description = "Sets the mime.type attribute to the MIME Type specified by the Source / Record Writer"),
+ @WritesAttribute(attribute= GetWorkdayReport.RECORD_COUNT, description = "The number of records in an outgoing FlowFile. This is only populated on the 'success' relationship "
+ + "when Record Reader and Writer is set.")})
+public class GetWorkdayReport extends AbstractProcessor {
+
+ protected static final String STATUS_CODE = "getworkdayreport.status.code";
+ protected static final String REQUEST_URL = "getworkdayreport.request.url";
+ protected static final String REQUEST_DURATION = "getworkdayreport.request.duration";
+ protected static final String TRANSACTION_ID = "getworkdayreport.tx.id";
+ protected static final String GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS = "getworkdayreport.java.exception.class";
+ protected static final String GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE = "getworkdayreport.java.exception.message";
+ protected static final String RECORD_COUNT = "record.count";
+ protected static final String BASIC_PREFIX = "Basic ";
+ protected static final String COLUMNS_TO_HASH_SEPARATOR = ",";
+ protected static final String HEADER_AUTHORIZATION = "Authorization";
+ protected static final String HEADER_CONTENT_TYPE = "Content-Type";
+ protected static final String USERNAME_PASSWORD_SEPARATOR = ":";
+
+ protected static final PropertyDescriptor REPORT_URL = new PropertyDescriptor.Builder()
+ .name("Workday report URL")
+ .displayName("Workday report URL")
+ .description("HTTP remote URL of Workday report including a scheme of http or https, as well as a hostname or IP address with optional port and path elements.")
+ .required(true)
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .addValidator(URL_VALIDATOR)
+ .build();
+
+ protected static final PropertyDescriptor WORKDAY_USERNAME = new PropertyDescriptor.Builder()
+ .name("Workday Username")
+ .displayName("Workday Username")
+ .description("The username provided for authentication of Workday requests. Encoded using Base64 for HTTP Basic Authentication as described in RFC 7617.")
+ .required(true)
+ .addValidator(StandardValidators.createRegexMatchingValidator(Pattern.compile("^[\\x20-\\x39\\x3b-\\x7e\\x80-\\xff]+$")))
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .build();
+
+ protected static final PropertyDescriptor WORKDAY_PASSWORD = new PropertyDescriptor.Builder()
+ .name("Workday Password")
+ .displayName("Workday Password")
+ .description("The password provided for authentication of Workday requests. Encoded using Base64 for HTTP Basic Authentication as described in RFC 7617.")
+ .required(true)
+ .sensitive(true)
+ .addValidator(StandardValidators.createRegexMatchingValidator(Pattern.compile("^[\\x20-\\x7e\\x80-\\xff]+$")))
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .build();
+
+ protected static final PropertyDescriptor WEB_CLIENT_SERVICE = new PropertyDescriptor.Builder()
+ .name("Standard Web Client Service")
+ .description("Web client which is used to communicate with the Workday API.")
+ .required(true)
+ .identifiesControllerService(WebClientServiceProvider.class)
+ .build();
+
+ protected static final PropertyDescriptor FIELDS_TO_HASH = new PropertyDescriptor.Builder()
+ .name("Fields to hash")
+ .displayName("Fields to hash")
+ .description("Comma separated record paths to replace with hash.")
+ .required(false)
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .addValidator(NON_BLANK_VALIDATOR)
+ .build();
+
+ protected static final PropertyDescriptor HASHING_ALGORITHM = new PropertyDescriptor.Builder()
+ .name("Hashing algorithm")
+ .displayName("Hashing algorithm")
+ .description("Determines what hashing algorithm should be used to perform the hashing function.")
+ .required(true)
+ .allowableValues(HashService.buildHashAlgorithmAllowableValues())
+ .defaultValue(HashAlgorithm.SHA256.getName())
+ .addValidator(NON_BLANK_VALIDATOR)
+ .expressionLanguageSupported(NONE)
+ .dependsOn(FIELDS_TO_HASH)
+ .build();
+
+ protected static final PropertyDescriptor RECORD_READER_FACTORY = new PropertyDescriptor.Builder()
+ .name("record-reader")
+ .displayName("Record Reader")
+ .description("Specifies the Controller Service to use for parsing incoming data and determining the data's schema.")
+ .identifiesControllerService(RecordReaderFactory.class)
+ .required(false)
+ .build();
+
+ protected static final PropertyDescriptor RECORD_WRITER_FACTORY = new PropertyDescriptor.Builder()
+ .name("record-writer")
+ .displayName("Record Writer")
+ .description("The Record Writer to use for serializing Records to an output FlowFile.")
+ .identifiesControllerService(RecordSetWriterFactory.class)
+ .dependsOn(RECORD_READER_FACTORY)
+ .required(true)
+ .build();
+
+ protected static final Relationship ORIGINAL = new Relationship.Builder()
+ .name("Original")
+ .description("Request FlowFiles transferred when receiving HTTP responses with a status code between 200 and 299.")
+ .build();
+
+ protected static final Relationship FAILURE = new Relationship.Builder()
+ .name("Failure")
+ .description("Request FlowFiles transferred when receiving socket communication errors.")
+ .build();
+
+ protected static final Relationship RESPONSE = new Relationship.Builder()
+ .name("Response")
+ .description("Response FlowFiles transferred when receiving HTTP responses with a status code between 200 and 299.")
+ .build();
+
+ protected static final Set<Relationship> RELATIONSHIPS = Collections.unmodifiableSet(new HashSet<>(Arrays.asList(ORIGINAL, RESPONSE, FAILURE)));
+ protected static final List<PropertyDescriptor> PROPERTIES = Collections.unmodifiableList(Arrays.asList(REPORT_URL, WORKDAY_USERNAME, WORKDAY_PASSWORD, WEB_CLIENT_SERVICE,
+ FIELDS_TO_HASH, HASHING_ALGORITHM, RECORD_READER_FACTORY, RECORD_WRITER_FACTORY));
+
+ private final AtomicReference<WebClientService> webClientAtomicReference = new AtomicReference<>();
+ private final AtomicReference<RecordReaderFactory> recordReaderFactoryAtomicReference = new AtomicReference<>();
+ private final AtomicReference<RecordSetWriterFactory> recordSetWriterFactoryAtomicReference = new AtomicReference<>();
+
+ @Override
+ protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
+ return PROPERTIES;
+ }
+
+ @Override
+ public Set<Relationship> getRelationships() {
+ return RELATIONSHIPS;
+ }
+
+ @OnScheduled
+ public void setUpClient(final ProcessContext context) {
+ WebClientServiceProvider standardWebClientServiceProvider = context.getProperty(WEB_CLIENT_SERVICE).asControllerService(WebClientServiceProvider.class);
+ RecordReaderFactory recordReaderFactory = context.getProperty(RECORD_READER_FACTORY).asControllerService(RecordReaderFactory.class);
+ RecordSetWriterFactory recordSetWriterFactory = context.getProperty(RECORD_WRITER_FACTORY).asControllerService(RecordSetWriterFactory.class);
+ WebClientService webClientService = standardWebClientServiceProvider.getWebClientService();
+ webClientAtomicReference.set(webClientService);
+ recordReaderFactoryAtomicReference.set(recordReaderFactory);
+ recordSetWriterFactoryAtomicReference.set(recordSetWriterFactory);
+ }
+
+ @Override
+ public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException {
+ FlowFile flowfile = session.get();
+
+ if (skipExecution(context, flowfile)) {
+ return;
+ }
+
+ ComponentLog logger = getLogger();
+ FlowFile responseFlowFile = null;
+
+ try {
+ WebClientService webClientService = webClientAtomicReference.get();
+ URI uri = new URI(context.getProperty(REPORT_URL).evaluateAttributeExpressions(flowfile).getValue().trim());
+ long startNanos = System.nanoTime();
+ String authorization = createAuthorizationHeader(context, flowfile);
+
+ try(HttpResponseEntity httpResponseEntity = webClientService.get().uri(uri).header(HEADER_AUTHORIZATION, authorization).retrieve()) {
+ responseFlowFile = createResponseFlowFile(flowfile, session, context, httpResponseEntity);
+ long elapsedTime = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos);
+ Map<String, String> commonAttributes = createCommonAttributes(uri, httpResponseEntity, elapsedTime);
+
+ if (flowfile != null) {
+ flowfile = session.putAllAttributes(flowfile, decorateWithMimeAttribute(commonAttributes, httpResponseEntity));
+ }
+ if (responseFlowFile != null) {
+ responseFlowFile = session.putAllAttributes(responseFlowFile, commonAttributes);
+ if (flowfile != null) {
+ session.getProvenanceReporter().fetch(responseFlowFile, uri.toString(), elapsedTime);
+ } else {
+ session.getProvenanceReporter().receive(responseFlowFile, uri.toString(), elapsedTime);
+ }
+ }
+
+ route(flowfile, responseFlowFile, session, context, httpResponseEntity.statusCode());
+ }
+ } catch (Exception e) {
+ if (flowfile == null) {
+ logger.error("Request Processing failed", e);
+ context.yield();
+ } else {
+ logger.error("Request Processing failed: {}", flowfile, e);
+ session.penalize(flowfile);
+ flowfile = session.putAttribute(flowfile, GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS, e.getClass().getName());
+ flowfile = session.putAttribute(flowfile, GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE, e.getMessage());
+ session.transfer(flowfile, FAILURE);
+ }
+
+ if (responseFlowFile != null) {
+ session.remove(responseFlowFile);
+ }
+ }
+ }
+
+ @Override
+ protected Collection<ValidationResult> customValidate(ValidationContext validationContext) {
+ List<ValidationResult> results = new ArrayList<>(super.customValidate(validationContext));
+ if (validationContext.getProperty(FIELDS_TO_HASH).isSet() && !validationContext.getProperty(RECORD_READER_FACTORY).isSet()) {
+ results.add(new ValidationResult.Builder()
+ .valid(false)
+ .explanation("Record-Reader and Record-Writer must be set if you would like to hash specific fields")
+ .subject("Workday report configuration")
+ .build());
+ }
+ return results;
+ }
+
+ /*
+ * If we have no FlowFile, and all incoming connections are self-loops then we can continue on.
+ * However, if we have no FlowFile and we have connections coming from other Processors, then
+ * we know that we should run only if we have a FlowFile.
+ */
+ private boolean skipExecution(ProcessContext context, FlowFile flowfile) {
+ return context.hasIncomingConnection() && flowfile == null && context.hasNonLoopConnection();
+ }
+
+ private FlowFile createResponseFlowFile(FlowFile flowfile, ProcessSession session, ProcessContext context, HttpResponseEntity httpResponseEntity)
+ throws IOException, SchemaNotFoundException, MalformedRecordException {
+ FlowFile responseFlowFile = null;
+ String hashingAlgorithm = context.getProperty(HASHING_ALGORITHM).getValue();
+ Set<String> columnsToHash = Optional.ofNullable(context.getProperty(FIELDS_TO_HASH).evaluateAttributeExpressions(flowfile).getValue()).map(String::trim)
+ .map(columns -> columns.split(COLUMNS_TO_HASH_SEPARATOR)).map(Arrays::stream).map(columns -> columns.collect(Collectors.toSet())).orElse(Collections.emptySet());
Review Comment:
done
##########
nifi-nar-bundles/nifi-workday-bundle/nifi-workday-processors/src/main/java/org/apache/nifi/processors/workday/GetWorkdayReport.java:
##########
@@ -0,0 +1,432 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.processors.workday;
+
+import static org.apache.nifi.expression.ExpressionLanguageScope.FLOWFILE_ATTRIBUTES;
+import static org.apache.nifi.expression.ExpressionLanguageScope.NONE;
+import static org.apache.nifi.processor.util.StandardValidators.NON_BLANK_VALIDATOR;
+import static org.apache.nifi.processor.util.StandardValidators.URL_VALIDATOR;
+
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Base64;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import org.apache.nifi.annotation.behavior.EventDriven;
+import org.apache.nifi.annotation.behavior.InputRequirement;
+import org.apache.nifi.annotation.behavior.InputRequirement.Requirement;
+import org.apache.nifi.annotation.behavior.SideEffectFree;
+import org.apache.nifi.annotation.behavior.SupportsBatching;
+import org.apache.nifi.annotation.behavior.WritesAttribute;
+import org.apache.nifi.annotation.behavior.WritesAttributes;
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.annotation.lifecycle.OnScheduled;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.ValidationContext;
+import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.flowfile.attributes.CoreAttributes;
+import org.apache.nifi.logging.ComponentLog;
+import org.apache.nifi.processor.AbstractProcessor;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.processor.util.StandardValidators;
+import org.apache.nifi.record.path.RecordPath;
+import org.apache.nifi.record.path.RecordPathResult;
+import org.apache.nifi.schema.access.SchemaNotFoundException;
+import org.apache.nifi.security.util.crypto.HashAlgorithm;
+import org.apache.nifi.security.util.crypto.HashService;
+import org.apache.nifi.serialization.MalformedRecordException;
+import org.apache.nifi.serialization.RecordReader;
+import org.apache.nifi.serialization.RecordReaderFactory;
+import org.apache.nifi.serialization.RecordSetWriter;
+import org.apache.nifi.serialization.RecordSetWriterFactory;
+import org.apache.nifi.serialization.record.Record;
+import org.apache.nifi.serialization.record.RecordFieldType;
+import org.apache.nifi.serialization.record.RecordSchema;
+import org.apache.nifi.web.client.api.HttpResponseEntity;
+import org.apache.nifi.web.client.api.WebClientService;
+import org.apache.nifi.web.client.provider.api.WebClientServiceProvider;
+
+@Tags({"Workday", "report", "get"})
+@InputRequirement(Requirement.INPUT_ALLOWED)
+@CapabilityDescription("A processor which can interact with a configurable Workday Report. The processor can forward the content without modification, or you can transform it by"
+ + " providing the specific Record Reader and Record Writer services based on your needs. You can also hash, or remove fields using the input parameters and schemes in the Record writer. "
+ + "Supported Workday report formats are: csv, simplexml, json")
+@EventDriven
+@SideEffectFree
+@SupportsBatching
+@WritesAttributes({
+ @WritesAttribute(attribute = GetWorkdayReport.GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS, description = "The Java exception class raised when the processor fails"),
+ @WritesAttribute(attribute = GetWorkdayReport.GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE, description = "The Java exception message raised when the processor fails"),
+ @WritesAttribute(attribute = "mime.type", description = "Sets the mime.type attribute to the MIME Type specified by the Source / Record Writer"),
+ @WritesAttribute(attribute= GetWorkdayReport.RECORD_COUNT, description = "The number of records in an outgoing FlowFile. This is only populated on the 'success' relationship "
+ + "when Record Reader and Writer is set.")})
+public class GetWorkdayReport extends AbstractProcessor {
+
+ protected static final String STATUS_CODE = "getworkdayreport.status.code";
+ protected static final String REQUEST_URL = "getworkdayreport.request.url";
+ protected static final String REQUEST_DURATION = "getworkdayreport.request.duration";
+ protected static final String TRANSACTION_ID = "getworkdayreport.tx.id";
+ protected static final String GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS = "getworkdayreport.java.exception.class";
+ protected static final String GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE = "getworkdayreport.java.exception.message";
+ protected static final String RECORD_COUNT = "record.count";
+ protected static final String BASIC_PREFIX = "Basic ";
+ protected static final String COLUMNS_TO_HASH_SEPARATOR = ",";
+ protected static final String HEADER_AUTHORIZATION = "Authorization";
+ protected static final String HEADER_CONTENT_TYPE = "Content-Type";
+ protected static final String USERNAME_PASSWORD_SEPARATOR = ":";
+
+ protected static final PropertyDescriptor REPORT_URL = new PropertyDescriptor.Builder()
+ .name("Workday report URL")
+ .displayName("Workday report URL")
+ .description("HTTP remote URL of Workday report including a scheme of http or https, as well as a hostname or IP address with optional port and path elements.")
+ .required(true)
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .addValidator(URL_VALIDATOR)
+ .build();
+
+ protected static final PropertyDescriptor WORKDAY_USERNAME = new PropertyDescriptor.Builder()
+ .name("Workday Username")
+ .displayName("Workday Username")
+ .description("The username provided for authentication of Workday requests. Encoded using Base64 for HTTP Basic Authentication as described in RFC 7617.")
+ .required(true)
+ .addValidator(StandardValidators.createRegexMatchingValidator(Pattern.compile("^[\\x20-\\x39\\x3b-\\x7e\\x80-\\xff]+$")))
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .build();
+
+ protected static final PropertyDescriptor WORKDAY_PASSWORD = new PropertyDescriptor.Builder()
+ .name("Workday Password")
+ .displayName("Workday Password")
+ .description("The password provided for authentication of Workday requests. Encoded using Base64 for HTTP Basic Authentication as described in RFC 7617.")
+ .required(true)
+ .sensitive(true)
+ .addValidator(StandardValidators.createRegexMatchingValidator(Pattern.compile("^[\\x20-\\x7e\\x80-\\xff]+$")))
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .build();
+
+ protected static final PropertyDescriptor WEB_CLIENT_SERVICE = new PropertyDescriptor.Builder()
+ .name("Standard Web Client Service")
+ .description("Web client which is used to communicate with the Workday API.")
+ .required(true)
+ .identifiesControllerService(WebClientServiceProvider.class)
+ .build();
+
+ protected static final PropertyDescriptor FIELDS_TO_HASH = new PropertyDescriptor.Builder()
+ .name("Fields to hash")
+ .displayName("Fields to hash")
+ .description("Comma separated record paths to replace with hash.")
+ .required(false)
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .addValidator(NON_BLANK_VALIDATOR)
+ .build();
+
+ protected static final PropertyDescriptor HASHING_ALGORITHM = new PropertyDescriptor.Builder()
+ .name("Hashing algorithm")
+ .displayName("Hashing algorithm")
+ .description("Determines what hashing algorithm should be used to perform the hashing function.")
+ .required(true)
+ .allowableValues(HashService.buildHashAlgorithmAllowableValues())
+ .defaultValue(HashAlgorithm.SHA256.getName())
+ .addValidator(NON_BLANK_VALIDATOR)
+ .expressionLanguageSupported(NONE)
+ .dependsOn(FIELDS_TO_HASH)
+ .build();
+
+ protected static final PropertyDescriptor RECORD_READER_FACTORY = new PropertyDescriptor.Builder()
+ .name("record-reader")
+ .displayName("Record Reader")
+ .description("Specifies the Controller Service to use for parsing incoming data and determining the data's schema.")
+ .identifiesControllerService(RecordReaderFactory.class)
+ .required(false)
+ .build();
+
+ protected static final PropertyDescriptor RECORD_WRITER_FACTORY = new PropertyDescriptor.Builder()
+ .name("record-writer")
+ .displayName("Record Writer")
+ .description("The Record Writer to use for serializing Records to an output FlowFile.")
+ .identifiesControllerService(RecordSetWriterFactory.class)
+ .dependsOn(RECORD_READER_FACTORY)
+ .required(true)
+ .build();
+
+ protected static final Relationship ORIGINAL = new Relationship.Builder()
+ .name("Original")
+ .description("Request FlowFiles transferred when receiving HTTP responses with a status code between 200 and 299.")
+ .build();
+
+ protected static final Relationship FAILURE = new Relationship.Builder()
+ .name("Failure")
+ .description("Request FlowFiles transferred when receiving socket communication errors.")
+ .build();
+
+ protected static final Relationship RESPONSE = new Relationship.Builder()
+ .name("Response")
+ .description("Response FlowFiles transferred when receiving HTTP responses with a status code between 200 and 299.")
+ .build();
+
+ protected static final Set<Relationship> RELATIONSHIPS = Collections.unmodifiableSet(new HashSet<>(Arrays.asList(ORIGINAL, RESPONSE, FAILURE)));
+ protected static final List<PropertyDescriptor> PROPERTIES = Collections.unmodifiableList(Arrays.asList(REPORT_URL, WORKDAY_USERNAME, WORKDAY_PASSWORD, WEB_CLIENT_SERVICE,
+ FIELDS_TO_HASH, HASHING_ALGORITHM, RECORD_READER_FACTORY, RECORD_WRITER_FACTORY));
+
+ private final AtomicReference<WebClientService> webClientAtomicReference = new AtomicReference<>();
+ private final AtomicReference<RecordReaderFactory> recordReaderFactoryAtomicReference = new AtomicReference<>();
+ private final AtomicReference<RecordSetWriterFactory> recordSetWriterFactoryAtomicReference = new AtomicReference<>();
+
+ @Override
+ protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
+ return PROPERTIES;
+ }
+
+ @Override
+ public Set<Relationship> getRelationships() {
+ return RELATIONSHIPS;
+ }
+
+ @OnScheduled
+ public void setUpClient(final ProcessContext context) {
+ WebClientServiceProvider standardWebClientServiceProvider = context.getProperty(WEB_CLIENT_SERVICE).asControllerService(WebClientServiceProvider.class);
+ RecordReaderFactory recordReaderFactory = context.getProperty(RECORD_READER_FACTORY).asControllerService(RecordReaderFactory.class);
+ RecordSetWriterFactory recordSetWriterFactory = context.getProperty(RECORD_WRITER_FACTORY).asControllerService(RecordSetWriterFactory.class);
+ WebClientService webClientService = standardWebClientServiceProvider.getWebClientService();
+ webClientAtomicReference.set(webClientService);
+ recordReaderFactoryAtomicReference.set(recordReaderFactory);
+ recordSetWriterFactoryAtomicReference.set(recordSetWriterFactory);
+ }
+
+ @Override
+ public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException {
+ FlowFile flowfile = session.get();
+
+ if (skipExecution(context, flowfile)) {
+ return;
+ }
+
+ ComponentLog logger = getLogger();
+ FlowFile responseFlowFile = null;
+
+ try {
+ WebClientService webClientService = webClientAtomicReference.get();
+ URI uri = new URI(context.getProperty(REPORT_URL).evaluateAttributeExpressions(flowfile).getValue().trim());
+ long startNanos = System.nanoTime();
+ String authorization = createAuthorizationHeader(context, flowfile);
+
+ try(HttpResponseEntity httpResponseEntity = webClientService.get().uri(uri).header(HEADER_AUTHORIZATION, authorization).retrieve()) {
+ responseFlowFile = createResponseFlowFile(flowfile, session, context, httpResponseEntity);
+ long elapsedTime = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos);
+ Map<String, String> commonAttributes = createCommonAttributes(uri, httpResponseEntity, elapsedTime);
+
+ if (flowfile != null) {
+ flowfile = session.putAllAttributes(flowfile, decorateWithMimeAttribute(commonAttributes, httpResponseEntity));
+ }
+ if (responseFlowFile != null) {
+ responseFlowFile = session.putAllAttributes(responseFlowFile, commonAttributes);
+ if (flowfile != null) {
+ session.getProvenanceReporter().fetch(responseFlowFile, uri.toString(), elapsedTime);
+ } else {
+ session.getProvenanceReporter().receive(responseFlowFile, uri.toString(), elapsedTime);
+ }
+ }
+
+ route(flowfile, responseFlowFile, session, context, httpResponseEntity.statusCode());
+ }
+ } catch (Exception e) {
+ if (flowfile == null) {
+ logger.error("Request Processing failed", e);
+ context.yield();
+ } else {
+ logger.error("Request Processing failed: {}", flowfile, e);
+ session.penalize(flowfile);
+ flowfile = session.putAttribute(flowfile, GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS, e.getClass().getName());
+ flowfile = session.putAttribute(flowfile, GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE, e.getMessage());
+ session.transfer(flowfile, FAILURE);
+ }
+
+ if (responseFlowFile != null) {
+ session.remove(responseFlowFile);
+ }
+ }
+ }
+
+ @Override
+ protected Collection<ValidationResult> customValidate(ValidationContext validationContext) {
+ List<ValidationResult> results = new ArrayList<>(super.customValidate(validationContext));
+ if (validationContext.getProperty(FIELDS_TO_HASH).isSet() && !validationContext.getProperty(RECORD_READER_FACTORY).isSet()) {
+ results.add(new ValidationResult.Builder()
+ .valid(false)
+ .explanation("Record-Reader and Record-Writer must be set if you would like to hash specific fields")
+ .subject("Workday report configuration")
+ .build());
+ }
+ return results;
+ }
+
+ /*
+ * If we have no FlowFile, and all incoming connections are self-loops then we can continue on.
+ * However, if we have no FlowFile and we have connections coming from other Processors, then
+ * we know that we should run only if we have a FlowFile.
+ */
+ private boolean skipExecution(ProcessContext context, FlowFile flowfile) {
+ return context.hasIncomingConnection() && flowfile == null && context.hasNonLoopConnection();
+ }
+
+ private FlowFile createResponseFlowFile(FlowFile flowfile, ProcessSession session, ProcessContext context, HttpResponseEntity httpResponseEntity)
+ throws IOException, SchemaNotFoundException, MalformedRecordException {
+ FlowFile responseFlowFile = null;
+ String hashingAlgorithm = context.getProperty(HASHING_ALGORITHM).getValue();
+ Set<String> columnsToHash = Optional.ofNullable(context.getProperty(FIELDS_TO_HASH).evaluateAttributeExpressions(flowfile).getValue()).map(String::trim)
+ .map(columns -> columns.split(COLUMNS_TO_HASH_SEPARATOR)).map(Arrays::stream).map(columns -> columns.collect(Collectors.toSet())).orElse(Collections.emptySet());
+ try {
+ if (isSuccess(httpResponseEntity.statusCode())) {
+ responseFlowFile = flowfile != null ? session.create(flowfile) : session.create();
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@nifi.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [nifi] ferencerdei commented on a diff in pull request #6376: NIFI-10455 Get workday report processor
Posted by GitBox <gi...@apache.org>.
ferencerdei commented on code in PR #6376:
URL: https://github.com/apache/nifi/pull/6376#discussion_r968132236
##########
nifi-nar-bundles/nifi-workday-bundle/nifi-workday-processors/src/main/java/org/apache/nifi/processors/workday/GetWorkdayReport.java:
##########
@@ -0,0 +1,432 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.processors.workday;
+
+import static org.apache.nifi.expression.ExpressionLanguageScope.FLOWFILE_ATTRIBUTES;
+import static org.apache.nifi.expression.ExpressionLanguageScope.NONE;
+import static org.apache.nifi.processor.util.StandardValidators.NON_BLANK_VALIDATOR;
+import static org.apache.nifi.processor.util.StandardValidators.URL_VALIDATOR;
+
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Base64;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import org.apache.nifi.annotation.behavior.EventDriven;
+import org.apache.nifi.annotation.behavior.InputRequirement;
+import org.apache.nifi.annotation.behavior.InputRequirement.Requirement;
+import org.apache.nifi.annotation.behavior.SideEffectFree;
+import org.apache.nifi.annotation.behavior.SupportsBatching;
+import org.apache.nifi.annotation.behavior.WritesAttribute;
+import org.apache.nifi.annotation.behavior.WritesAttributes;
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.annotation.lifecycle.OnScheduled;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.ValidationContext;
+import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.flowfile.attributes.CoreAttributes;
+import org.apache.nifi.logging.ComponentLog;
+import org.apache.nifi.processor.AbstractProcessor;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.processor.util.StandardValidators;
+import org.apache.nifi.record.path.RecordPath;
+import org.apache.nifi.record.path.RecordPathResult;
+import org.apache.nifi.schema.access.SchemaNotFoundException;
+import org.apache.nifi.security.util.crypto.HashAlgorithm;
+import org.apache.nifi.security.util.crypto.HashService;
+import org.apache.nifi.serialization.MalformedRecordException;
+import org.apache.nifi.serialization.RecordReader;
+import org.apache.nifi.serialization.RecordReaderFactory;
+import org.apache.nifi.serialization.RecordSetWriter;
+import org.apache.nifi.serialization.RecordSetWriterFactory;
+import org.apache.nifi.serialization.record.Record;
+import org.apache.nifi.serialization.record.RecordFieldType;
+import org.apache.nifi.serialization.record.RecordSchema;
+import org.apache.nifi.web.client.api.HttpResponseEntity;
+import org.apache.nifi.web.client.api.WebClientService;
+import org.apache.nifi.web.client.provider.api.WebClientServiceProvider;
+
+@Tags({"Workday", "report", "get"})
+@InputRequirement(Requirement.INPUT_ALLOWED)
+@CapabilityDescription("A processor which can interact with a configurable Workday Report. The processor can forward the content without modification, or you can transform it by"
+ + " providing the specific Record Reader and Record Writer services based on your needs. You can also hash, or remove fields using the input parameters and schemes in the Record writer. "
+ + "Supported Workday report formats are: csv, simplexml, json")
+@EventDriven
+@SideEffectFree
+@SupportsBatching
+@WritesAttributes({
+ @WritesAttribute(attribute = GetWorkdayReport.GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS, description = "The Java exception class raised when the processor fails"),
+ @WritesAttribute(attribute = GetWorkdayReport.GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE, description = "The Java exception message raised when the processor fails"),
+ @WritesAttribute(attribute = "mime.type", description = "Sets the mime.type attribute to the MIME Type specified by the Source / Record Writer"),
+ @WritesAttribute(attribute= GetWorkdayReport.RECORD_COUNT, description = "The number of records in an outgoing FlowFile. This is only populated on the 'success' relationship "
+ + "when Record Reader and Writer is set.")})
+public class GetWorkdayReport extends AbstractProcessor {
+
+ protected static final String STATUS_CODE = "getworkdayreport.status.code";
+ protected static final String REQUEST_URL = "getworkdayreport.request.url";
+ protected static final String REQUEST_DURATION = "getworkdayreport.request.duration";
+ protected static final String TRANSACTION_ID = "getworkdayreport.tx.id";
+ protected static final String GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS = "getworkdayreport.java.exception.class";
+ protected static final String GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE = "getworkdayreport.java.exception.message";
+ protected static final String RECORD_COUNT = "record.count";
+ protected static final String BASIC_PREFIX = "Basic ";
+ protected static final String COLUMNS_TO_HASH_SEPARATOR = ",";
+ protected static final String HEADER_AUTHORIZATION = "Authorization";
+ protected static final String HEADER_CONTENT_TYPE = "Content-Type";
+ protected static final String USERNAME_PASSWORD_SEPARATOR = ":";
+
+ protected static final PropertyDescriptor REPORT_URL = new PropertyDescriptor.Builder()
+ .name("Workday report URL")
+ .displayName("Workday report URL")
+ .description("HTTP remote URL of Workday report including a scheme of http or https, as well as a hostname or IP address with optional port and path elements.")
+ .required(true)
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .addValidator(URL_VALIDATOR)
+ .build();
+
+ protected static final PropertyDescriptor WORKDAY_USERNAME = new PropertyDescriptor.Builder()
+ .name("Workday Username")
+ .displayName("Workday Username")
+ .description("The username provided for authentication of Workday requests. Encoded using Base64 for HTTP Basic Authentication as described in RFC 7617.")
+ .required(true)
+ .addValidator(StandardValidators.createRegexMatchingValidator(Pattern.compile("^[\\x20-\\x39\\x3b-\\x7e\\x80-\\xff]+$")))
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .build();
+
+ protected static final PropertyDescriptor WORKDAY_PASSWORD = new PropertyDescriptor.Builder()
+ .name("Workday Password")
+ .displayName("Workday Password")
+ .description("The password provided for authentication of Workday requests. Encoded using Base64 for HTTP Basic Authentication as described in RFC 7617.")
+ .required(true)
+ .sensitive(true)
+ .addValidator(StandardValidators.createRegexMatchingValidator(Pattern.compile("^[\\x20-\\x7e\\x80-\\xff]+$")))
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .build();
+
+ protected static final PropertyDescriptor WEB_CLIENT_SERVICE = new PropertyDescriptor.Builder()
+ .name("Standard Web Client Service")
+ .description("Web client which is used to communicate with the Workday API.")
+ .required(true)
+ .identifiesControllerService(WebClientServiceProvider.class)
+ .build();
+
+ protected static final PropertyDescriptor FIELDS_TO_HASH = new PropertyDescriptor.Builder()
+ .name("Fields to hash")
+ .displayName("Fields to hash")
+ .description("Comma separated record paths to replace with hash.")
Review Comment:
updated the description
##########
nifi-nar-bundles/nifi-workday-bundle/nifi-workday-processors/src/main/java/org/apache/nifi/processors/workday/GetWorkdayReport.java:
##########
@@ -0,0 +1,432 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.processors.workday;
+
+import static org.apache.nifi.expression.ExpressionLanguageScope.FLOWFILE_ATTRIBUTES;
+import static org.apache.nifi.expression.ExpressionLanguageScope.NONE;
+import static org.apache.nifi.processor.util.StandardValidators.NON_BLANK_VALIDATOR;
+import static org.apache.nifi.processor.util.StandardValidators.URL_VALIDATOR;
+
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Base64;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import org.apache.nifi.annotation.behavior.EventDriven;
+import org.apache.nifi.annotation.behavior.InputRequirement;
+import org.apache.nifi.annotation.behavior.InputRequirement.Requirement;
+import org.apache.nifi.annotation.behavior.SideEffectFree;
+import org.apache.nifi.annotation.behavior.SupportsBatching;
+import org.apache.nifi.annotation.behavior.WritesAttribute;
+import org.apache.nifi.annotation.behavior.WritesAttributes;
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.annotation.lifecycle.OnScheduled;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.ValidationContext;
+import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.flowfile.attributes.CoreAttributes;
+import org.apache.nifi.logging.ComponentLog;
+import org.apache.nifi.processor.AbstractProcessor;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.processor.util.StandardValidators;
+import org.apache.nifi.record.path.RecordPath;
+import org.apache.nifi.record.path.RecordPathResult;
+import org.apache.nifi.schema.access.SchemaNotFoundException;
+import org.apache.nifi.security.util.crypto.HashAlgorithm;
+import org.apache.nifi.security.util.crypto.HashService;
+import org.apache.nifi.serialization.MalformedRecordException;
+import org.apache.nifi.serialization.RecordReader;
+import org.apache.nifi.serialization.RecordReaderFactory;
+import org.apache.nifi.serialization.RecordSetWriter;
+import org.apache.nifi.serialization.RecordSetWriterFactory;
+import org.apache.nifi.serialization.record.Record;
+import org.apache.nifi.serialization.record.RecordFieldType;
+import org.apache.nifi.serialization.record.RecordSchema;
+import org.apache.nifi.web.client.api.HttpResponseEntity;
+import org.apache.nifi.web.client.api.WebClientService;
+import org.apache.nifi.web.client.provider.api.WebClientServiceProvider;
+
+@Tags({"Workday", "report", "get"})
+@InputRequirement(Requirement.INPUT_ALLOWED)
+@CapabilityDescription("A processor which can interact with a configurable Workday Report. The processor can forward the content without modification, or you can transform it by"
+ + " providing the specific Record Reader and Record Writer services based on your needs. You can also hash, or remove fields using the input parameters and schemes in the Record writer. "
+ + "Supported Workday report formats are: csv, simplexml, json")
+@EventDriven
+@SideEffectFree
+@SupportsBatching
+@WritesAttributes({
+ @WritesAttribute(attribute = GetWorkdayReport.GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS, description = "The Java exception class raised when the processor fails"),
+ @WritesAttribute(attribute = GetWorkdayReport.GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE, description = "The Java exception message raised when the processor fails"),
+ @WritesAttribute(attribute = "mime.type", description = "Sets the mime.type attribute to the MIME Type specified by the Source / Record Writer"),
+ @WritesAttribute(attribute= GetWorkdayReport.RECORD_COUNT, description = "The number of records in an outgoing FlowFile. This is only populated on the 'success' relationship "
+ + "when Record Reader and Writer is set.")})
+public class GetWorkdayReport extends AbstractProcessor {
+
+ protected static final String STATUS_CODE = "getworkdayreport.status.code";
+ protected static final String REQUEST_URL = "getworkdayreport.request.url";
+ protected static final String REQUEST_DURATION = "getworkdayreport.request.duration";
+ protected static final String TRANSACTION_ID = "getworkdayreport.tx.id";
+ protected static final String GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS = "getworkdayreport.java.exception.class";
+ protected static final String GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE = "getworkdayreport.java.exception.message";
+ protected static final String RECORD_COUNT = "record.count";
+ protected static final String BASIC_PREFIX = "Basic ";
+ protected static final String COLUMNS_TO_HASH_SEPARATOR = ",";
+ protected static final String HEADER_AUTHORIZATION = "Authorization";
+ protected static final String HEADER_CONTENT_TYPE = "Content-Type";
+ protected static final String USERNAME_PASSWORD_SEPARATOR = ":";
+
+ protected static final PropertyDescriptor REPORT_URL = new PropertyDescriptor.Builder()
+ .name("Workday report URL")
+ .displayName("Workday report URL")
+ .description("HTTP remote URL of Workday report including a scheme of http or https, as well as a hostname or IP address with optional port and path elements.")
+ .required(true)
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .addValidator(URL_VALIDATOR)
+ .build();
+
+ protected static final PropertyDescriptor WORKDAY_USERNAME = new PropertyDescriptor.Builder()
+ .name("Workday Username")
+ .displayName("Workday Username")
+ .description("The username provided for authentication of Workday requests. Encoded using Base64 for HTTP Basic Authentication as described in RFC 7617.")
+ .required(true)
+ .addValidator(StandardValidators.createRegexMatchingValidator(Pattern.compile("^[\\x20-\\x39\\x3b-\\x7e\\x80-\\xff]+$")))
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .build();
+
+ protected static final PropertyDescriptor WORKDAY_PASSWORD = new PropertyDescriptor.Builder()
+ .name("Workday Password")
+ .displayName("Workday Password")
+ .description("The password provided for authentication of Workday requests. Encoded using Base64 for HTTP Basic Authentication as described in RFC 7617.")
+ .required(true)
+ .sensitive(true)
+ .addValidator(StandardValidators.createRegexMatchingValidator(Pattern.compile("^[\\x20-\\x7e\\x80-\\xff]+$")))
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .build();
+
+ protected static final PropertyDescriptor WEB_CLIENT_SERVICE = new PropertyDescriptor.Builder()
+ .name("Standard Web Client Service")
+ .description("Web client which is used to communicate with the Workday API.")
+ .required(true)
+ .identifiesControllerService(WebClientServiceProvider.class)
+ .build();
+
+ protected static final PropertyDescriptor FIELDS_TO_HASH = new PropertyDescriptor.Builder()
+ .name("Fields to hash")
+ .displayName("Fields to hash")
+ .description("Comma separated record paths to replace with hash.")
+ .required(false)
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .addValidator(NON_BLANK_VALIDATOR)
+ .build();
+
+ protected static final PropertyDescriptor HASHING_ALGORITHM = new PropertyDescriptor.Builder()
+ .name("Hashing algorithm")
+ .displayName("Hashing algorithm")
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@nifi.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [nifi] exceptionfactory commented on a diff in pull request #6376: NIFI-10455 Get workday report processor
Posted by GitBox <gi...@apache.org>.
exceptionfactory commented on code in PR #6376:
URL: https://github.com/apache/nifi/pull/6376#discussion_r968423628
##########
nifi-nar-bundles/nifi-workday-bundle/nifi-workday-processors/src/main/java/org/apache/nifi/processors/workday/GetWorkdayReport.java:
##########
@@ -0,0 +1,442 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.processors.workday;
+
+import static org.apache.nifi.expression.ExpressionLanguageScope.FLOWFILE_ATTRIBUTES;
+import static org.apache.nifi.expression.ExpressionLanguageScope.NONE;
+import static org.apache.nifi.processor.util.StandardValidators.NON_BLANK_VALIDATOR;
+import static org.apache.nifi.processor.util.StandardValidators.URL_VALIDATOR;
+
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Base64;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import org.apache.nifi.annotation.behavior.EventDriven;
+import org.apache.nifi.annotation.behavior.InputRequirement;
+import org.apache.nifi.annotation.behavior.InputRequirement.Requirement;
+import org.apache.nifi.annotation.behavior.SideEffectFree;
+import org.apache.nifi.annotation.behavior.SupportsBatching;
+import org.apache.nifi.annotation.behavior.WritesAttribute;
+import org.apache.nifi.annotation.behavior.WritesAttributes;
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.annotation.lifecycle.OnScheduled;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.ValidationContext;
+import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.flowfile.attributes.CoreAttributes;
+import org.apache.nifi.logging.ComponentLog;
+import org.apache.nifi.processor.AbstractProcessor;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.processor.util.StandardValidators;
+import org.apache.nifi.record.path.RecordPath;
+import org.apache.nifi.record.path.RecordPathResult;
+import org.apache.nifi.schema.access.SchemaNotFoundException;
+import org.apache.nifi.security.util.crypto.HashAlgorithm;
+import org.apache.nifi.serialization.MalformedRecordException;
+import org.apache.nifi.serialization.RecordReader;
+import org.apache.nifi.serialization.RecordReaderFactory;
+import org.apache.nifi.serialization.RecordSetWriter;
+import org.apache.nifi.serialization.RecordSetWriterFactory;
+import org.apache.nifi.serialization.record.Record;
+import org.apache.nifi.serialization.record.RecordFieldType;
+import org.apache.nifi.serialization.record.RecordSchema;
+import org.apache.nifi.web.client.api.HttpResponseEntity;
+import org.apache.nifi.web.client.api.WebClientService;
+import org.apache.nifi.web.client.provider.api.WebClientServiceProvider;
+
+@Tags({"Workday", "report"})
+@InputRequirement(Requirement.INPUT_ALLOWED)
+@CapabilityDescription("A processor which can interact with a configurable Workday Report. The processor can forward the content without modification, or you can transform it by"
+ + " providing the specific Record Reader and Record Writer services based on your needs. You can also hash, or remove fields using the input parameters and schemes in the Record Writer. "
+ + "Supported Workday report formats are: csv, simplexml, json")
+@EventDriven
+@SideEffectFree
+@SupportsBatching
+@WritesAttributes({
+ @WritesAttribute(attribute = GetWorkdayReport.GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS, description = "The Java exception class raised when the processor fails"),
+ @WritesAttribute(attribute = GetWorkdayReport.GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE, description = "The Java exception message raised when the processor fails"),
+ @WritesAttribute(attribute = "mime.type", description = "Sets the mime.type attribute to the MIME Type specified by the Source / Record Writer"),
+ @WritesAttribute(attribute= GetWorkdayReport.RECORD_COUNT, description = "The number of records in an outgoing FlowFile. This is only populated on the 'success' relationship "
+ + "when Record Reader and Writer is set.")})
+public class GetWorkdayReport extends AbstractProcessor {
+
+ protected static final String STATUS_CODE = "getworkdayreport.status.code";
+ protected static final String REQUEST_URL = "getworkdayreport.request.url";
+ protected static final String REQUEST_DURATION = "getworkdayreport.request.duration";
+ protected static final String TRANSACTION_ID = "getworkdayreport.tx.id";
+ protected static final String GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS = "getworkdayreport.java.exception.class";
+ protected static final String GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE = "getworkdayreport.java.exception.message";
+ protected static final String RECORD_COUNT = "record.count";
+ protected static final String BASIC_PREFIX = "Basic ";
+ protected static final String COLUMNS_TO_HASH_SEPARATOR = ",";
+ protected static final String HEADER_AUTHORIZATION = "Authorization";
+ protected static final String HEADER_CONTENT_TYPE = "Content-Type";
+ protected static final String USERNAME_PASSWORD_SEPARATOR = ":";
+
+ protected static final PropertyDescriptor REPORT_URL = new PropertyDescriptor.Builder()
+ .name("Workday Report URL")
+ .displayName("Workday Report URL")
+ .description("HTTP remote URL of Workday report including a scheme of http or https, as well as a hostname or IP address with optional port and path elements.")
+ .required(true)
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .addValidator(URL_VALIDATOR)
+ .build();
+
+ protected static final PropertyDescriptor WORKDAY_USERNAME = new PropertyDescriptor.Builder()
+ .name("Workday Username")
+ .displayName("Workday Username")
+ .description("The username provided for authentication of Workday requests. Encoded using Base64 for HTTP Basic Authentication as described in RFC 7617.")
+ .required(true)
+ .addValidator(StandardValidators.createRegexMatchingValidator(Pattern.compile("^[\\x20-\\x39\\x3b-\\x7e\\x80-\\xff]+$")))
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .build();
+
+ protected static final PropertyDescriptor WORKDAY_PASSWORD = new PropertyDescriptor.Builder()
+ .name("Workday Password")
+ .displayName("Workday Password")
+ .description("The password provided for authentication of Workday requests. Encoded using Base64 for HTTP Basic Authentication as described in RFC 7617.")
+ .required(true)
+ .sensitive(true)
+ .addValidator(StandardValidators.createRegexMatchingValidator(Pattern.compile("^[\\x20-\\x7e\\x80-\\xff]+$")))
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .build();
+
+ protected static final PropertyDescriptor WEB_CLIENT_SERVICE = new PropertyDescriptor.Builder()
+ .name("Web Client Service Provider")
+ .description("Web client which is used to communicate with the Workday API.")
+ .required(true)
+ .identifiesControllerService(WebClientServiceProvider.class)
+ .build();
+
+ protected static final PropertyDescriptor FIELDS_TO_HASH = new PropertyDescriptor.Builder()
+ .name("Hashed Fields")
+ .displayName("Hashed Fields")
+ .description("Comma separated record paths to be replaced with their corresponding hash.")
+ .required(false)
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .addValidator(NON_BLANK_VALIDATOR)
+ .build();
+
+ protected static final PropertyDescriptor HASHING_ALGORITHM = new PropertyDescriptor.Builder()
+ .name("Hashing Algorithm")
+ .displayName("Hashing Algorithm")
+ .description("Determines what hashing algorithm should be used to perform the hashing function.")
+ .required(true)
+ .allowableValues(HashAlgorithm.SHA256.getName(), HashAlgorithm.SHA512.getName())
+ .defaultValue(HashAlgorithm.SHA256.getName())
+ .addValidator(NON_BLANK_VALIDATOR)
+ .expressionLanguageSupported(NONE)
+ .dependsOn(FIELDS_TO_HASH)
+ .build();
+
+ protected static final PropertyDescriptor RECORD_READER_FACTORY = new PropertyDescriptor.Builder()
+ .name("record-reader")
+ .displayName("Record Reader")
+ .description("Specifies the Controller Service to use for parsing incoming data and determining the data's schema.")
+ .identifiesControllerService(RecordReaderFactory.class)
+ .required(false)
+ .build();
+
+ protected static final PropertyDescriptor RECORD_WRITER_FACTORY = new PropertyDescriptor.Builder()
+ .name("record-writer")
+ .displayName("Record Writer")
+ .description("The Record Writer to use for serializing Records to an output FlowFile.")
+ .identifiesControllerService(RecordSetWriterFactory.class)
+ .dependsOn(RECORD_READER_FACTORY)
+ .required(true)
+ .build();
+
+ protected static final Relationship ORIGINAL = new Relationship.Builder()
+ .name("Original")
+ .description("Request FlowFiles transferred when receiving HTTP responses with a status code between 200 and 299.")
+ .build();
+
+ protected static final Relationship FAILURE = new Relationship.Builder()
+ .name("Failure")
+ .description("Request FlowFiles transferred when receiving socket communication errors.")
+ .build();
+
+ protected static final Relationship RESPONSE = new Relationship.Builder()
+ .name("Response")
+ .description("Response FlowFiles transferred when receiving HTTP responses with a status code between 200 and 299.")
+ .build();
+
+ protected static final Set<Relationship> RELATIONSHIPS = Collections.unmodifiableSet(new HashSet<>(Arrays.asList(ORIGINAL, RESPONSE, FAILURE)));
+ protected static final List<PropertyDescriptor> PROPERTIES = Collections.unmodifiableList(Arrays.asList(REPORT_URL, WORKDAY_USERNAME, WORKDAY_PASSWORD, WEB_CLIENT_SERVICE,
+ FIELDS_TO_HASH, HASHING_ALGORITHM, RECORD_READER_FACTORY, RECORD_WRITER_FACTORY));
Review Comment:
Recommend reformatting this declaration to list each property on a separate line.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@nifi.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [nifi] ferencerdei commented on a diff in pull request #6376: NIFI-10455 Get workday report processor
Posted by GitBox <gi...@apache.org>.
ferencerdei commented on code in PR #6376:
URL: https://github.com/apache/nifi/pull/6376#discussion_r969399412
##########
nifi-nar-bundles/nifi-workday-bundle/nifi-workday-processors/src/main/java/org/apache/nifi/processors/workday/GetWorkdayReport.java:
##########
@@ -0,0 +1,442 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.processors.workday;
+
+import static org.apache.nifi.expression.ExpressionLanguageScope.FLOWFILE_ATTRIBUTES;
+import static org.apache.nifi.expression.ExpressionLanguageScope.NONE;
+import static org.apache.nifi.processor.util.StandardValidators.NON_BLANK_VALIDATOR;
+import static org.apache.nifi.processor.util.StandardValidators.URL_VALIDATOR;
+
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Base64;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import org.apache.nifi.annotation.behavior.EventDriven;
+import org.apache.nifi.annotation.behavior.InputRequirement;
+import org.apache.nifi.annotation.behavior.InputRequirement.Requirement;
+import org.apache.nifi.annotation.behavior.SideEffectFree;
+import org.apache.nifi.annotation.behavior.SupportsBatching;
+import org.apache.nifi.annotation.behavior.WritesAttribute;
+import org.apache.nifi.annotation.behavior.WritesAttributes;
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.annotation.lifecycle.OnScheduled;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.ValidationContext;
+import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.flowfile.attributes.CoreAttributes;
+import org.apache.nifi.logging.ComponentLog;
+import org.apache.nifi.processor.AbstractProcessor;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.processor.util.StandardValidators;
+import org.apache.nifi.record.path.RecordPath;
+import org.apache.nifi.record.path.RecordPathResult;
+import org.apache.nifi.schema.access.SchemaNotFoundException;
+import org.apache.nifi.security.util.crypto.HashAlgorithm;
+import org.apache.nifi.serialization.MalformedRecordException;
+import org.apache.nifi.serialization.RecordReader;
+import org.apache.nifi.serialization.RecordReaderFactory;
+import org.apache.nifi.serialization.RecordSetWriter;
+import org.apache.nifi.serialization.RecordSetWriterFactory;
+import org.apache.nifi.serialization.record.Record;
+import org.apache.nifi.serialization.record.RecordFieldType;
+import org.apache.nifi.serialization.record.RecordSchema;
+import org.apache.nifi.web.client.api.HttpResponseEntity;
+import org.apache.nifi.web.client.api.WebClientService;
+import org.apache.nifi.web.client.provider.api.WebClientServiceProvider;
+
+@Tags({"Workday", "report"})
+@InputRequirement(Requirement.INPUT_ALLOWED)
+@CapabilityDescription("A processor which can interact with a configurable Workday Report. The processor can forward the content without modification, or you can transform it by"
+ + " providing the specific Record Reader and Record Writer services based on your needs. You can also hash, or remove fields using the input parameters and schemes in the Record Writer. "
+ + "Supported Workday report formats are: csv, simplexml, json")
+@EventDriven
+@SideEffectFree
+@SupportsBatching
+@WritesAttributes({
+ @WritesAttribute(attribute = GetWorkdayReport.GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS, description = "The Java exception class raised when the processor fails"),
+ @WritesAttribute(attribute = GetWorkdayReport.GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE, description = "The Java exception message raised when the processor fails"),
+ @WritesAttribute(attribute = "mime.type", description = "Sets the mime.type attribute to the MIME Type specified by the Source / Record Writer"),
+ @WritesAttribute(attribute= GetWorkdayReport.RECORD_COUNT, description = "The number of records in an outgoing FlowFile. This is only populated on the 'success' relationship "
+ + "when Record Reader and Writer is set.")})
+public class GetWorkdayReport extends AbstractProcessor {
+
+ protected static final String STATUS_CODE = "getworkdayreport.status.code";
+ protected static final String REQUEST_URL = "getworkdayreport.request.url";
+ protected static final String REQUEST_DURATION = "getworkdayreport.request.duration";
+ protected static final String TRANSACTION_ID = "getworkdayreport.tx.id";
+ protected static final String GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS = "getworkdayreport.java.exception.class";
+ protected static final String GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE = "getworkdayreport.java.exception.message";
+ protected static final String RECORD_COUNT = "record.count";
+ protected static final String BASIC_PREFIX = "Basic ";
+ protected static final String COLUMNS_TO_HASH_SEPARATOR = ",";
+ protected static final String HEADER_AUTHORIZATION = "Authorization";
+ protected static final String HEADER_CONTENT_TYPE = "Content-Type";
+ protected static final String USERNAME_PASSWORD_SEPARATOR = ":";
+
+ protected static final PropertyDescriptor REPORT_URL = new PropertyDescriptor.Builder()
+ .name("Workday Report URL")
+ .displayName("Workday Report URL")
+ .description("HTTP remote URL of Workday report including a scheme of http or https, as well as a hostname or IP address with optional port and path elements.")
+ .required(true)
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .addValidator(URL_VALIDATOR)
+ .build();
+
+ protected static final PropertyDescriptor WORKDAY_USERNAME = new PropertyDescriptor.Builder()
+ .name("Workday Username")
+ .displayName("Workday Username")
+ .description("The username provided for authentication of Workday requests. Encoded using Base64 for HTTP Basic Authentication as described in RFC 7617.")
+ .required(true)
+ .addValidator(StandardValidators.createRegexMatchingValidator(Pattern.compile("^[\\x20-\\x39\\x3b-\\x7e\\x80-\\xff]+$")))
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .build();
+
+ protected static final PropertyDescriptor WORKDAY_PASSWORD = new PropertyDescriptor.Builder()
+ .name("Workday Password")
+ .displayName("Workday Password")
+ .description("The password provided for authentication of Workday requests. Encoded using Base64 for HTTP Basic Authentication as described in RFC 7617.")
+ .required(true)
+ .sensitive(true)
+ .addValidator(StandardValidators.createRegexMatchingValidator(Pattern.compile("^[\\x20-\\x7e\\x80-\\xff]+$")))
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .build();
+
+ protected static final PropertyDescriptor WEB_CLIENT_SERVICE = new PropertyDescriptor.Builder()
+ .name("Web Client Service Provider")
+ .description("Web client which is used to communicate with the Workday API.")
+ .required(true)
+ .identifiesControllerService(WebClientServiceProvider.class)
+ .build();
+
+ protected static final PropertyDescriptor FIELDS_TO_HASH = new PropertyDescriptor.Builder()
+ .name("Hashed Fields")
+ .displayName("Hashed Fields")
+ .description("Comma separated record paths to be replaced with their corresponding hash.")
+ .required(false)
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .addValidator(NON_BLANK_VALIDATOR)
+ .build();
+
+ protected static final PropertyDescriptor HASHING_ALGORITHM = new PropertyDescriptor.Builder()
+ .name("Hashing Algorithm")
+ .displayName("Hashing Algorithm")
+ .description("Determines what hashing algorithm should be used to perform the hashing function.")
+ .required(true)
+ .allowableValues(HashAlgorithm.SHA256.getName(), HashAlgorithm.SHA512.getName())
+ .defaultValue(HashAlgorithm.SHA256.getName())
+ .addValidator(NON_BLANK_VALIDATOR)
+ .expressionLanguageSupported(NONE)
+ .dependsOn(FIELDS_TO_HASH)
+ .build();
+
+ protected static final PropertyDescriptor RECORD_READER_FACTORY = new PropertyDescriptor.Builder()
+ .name("record-reader")
+ .displayName("Record Reader")
+ .description("Specifies the Controller Service to use for parsing incoming data and determining the data's schema.")
+ .identifiesControllerService(RecordReaderFactory.class)
+ .required(false)
+ .build();
+
+ protected static final PropertyDescriptor RECORD_WRITER_FACTORY = new PropertyDescriptor.Builder()
+ .name("record-writer")
+ .displayName("Record Writer")
+ .description("The Record Writer to use for serializing Records to an output FlowFile.")
+ .identifiesControllerService(RecordSetWriterFactory.class)
+ .dependsOn(RECORD_READER_FACTORY)
+ .required(true)
+ .build();
+
+ protected static final Relationship ORIGINAL = new Relationship.Builder()
+ .name("Original")
+ .description("Request FlowFiles transferred when receiving HTTP responses with a status code between 200 and 299.")
+ .build();
+
+ protected static final Relationship FAILURE = new Relationship.Builder()
+ .name("Failure")
+ .description("Request FlowFiles transferred when receiving socket communication errors.")
+ .build();
+
+ protected static final Relationship RESPONSE = new Relationship.Builder()
+ .name("Response")
+ .description("Response FlowFiles transferred when receiving HTTP responses with a status code between 200 and 299.")
+ .build();
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@nifi.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [nifi] exceptionfactory commented on a diff in pull request #6376: NIFI-10455 Get workday report processor
Posted by GitBox <gi...@apache.org>.
exceptionfactory commented on code in PR #6376:
URL: https://github.com/apache/nifi/pull/6376#discussion_r970953378
##########
nifi-nar-bundles/nifi-workday-bundle/nifi-workday-processors/src/main/java/org/apache/nifi/processors/workday/GetWorkdayReport.java:
##########
@@ -0,0 +1,466 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.processors.workday;
+
+import static org.apache.nifi.expression.ExpressionLanguageScope.FLOWFILE_ATTRIBUTES;
+import static org.apache.nifi.expression.ExpressionLanguageScope.NONE;
+import static org.apache.nifi.processor.util.StandardValidators.NON_BLANK_VALIDATOR;
+import static org.apache.nifi.processor.util.StandardValidators.URL_VALIDATOR;
+
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Base64;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import org.apache.nifi.annotation.behavior.EventDriven;
+import org.apache.nifi.annotation.behavior.InputRequirement;
+import org.apache.nifi.annotation.behavior.InputRequirement.Requirement;
+import org.apache.nifi.annotation.behavior.SideEffectFree;
+import org.apache.nifi.annotation.behavior.SupportsBatching;
+import org.apache.nifi.annotation.behavior.WritesAttribute;
+import org.apache.nifi.annotation.behavior.WritesAttributes;
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.annotation.lifecycle.OnScheduled;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.ValidationContext;
+import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.flowfile.attributes.CoreAttributes;
+import org.apache.nifi.processor.AbstractProcessor;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.processor.util.StandardValidators;
+import org.apache.nifi.record.path.RecordPath;
+import org.apache.nifi.record.path.RecordPathResult;
+import org.apache.nifi.schema.access.SchemaNotFoundException;
+import org.apache.nifi.serialization.MalformedRecordException;
+import org.apache.nifi.serialization.RecordReader;
+import org.apache.nifi.serialization.RecordReaderFactory;
+import org.apache.nifi.serialization.RecordSetWriter;
+import org.apache.nifi.serialization.RecordSetWriterFactory;
+import org.apache.nifi.serialization.record.Record;
+import org.apache.nifi.serialization.record.RecordFieldType;
+import org.apache.nifi.serialization.record.RecordSchema;
+import org.apache.nifi.web.client.api.HttpResponseEntity;
+import org.apache.nifi.web.client.api.WebClientService;
+import org.apache.nifi.web.client.provider.api.WebClientServiceProvider;
+
+@Tags({"Workday", "report"})
+@InputRequirement(Requirement.INPUT_ALLOWED)
+@CapabilityDescription("A processor which can interact with a configurable Workday Report. The processor can forward the content without modification, or you can transform it by"
+ + " providing the specific Record Reader and Record Writer services based on your needs. You can also hash, or remove fields using the input parameters and schemes in the Record Writer. "
+ + "Supported Workday report formats are: csv, simplexml, json")
+@EventDriven
+@SideEffectFree
+@SupportsBatching
+@WritesAttributes({
+ @WritesAttribute(attribute = GetWorkdayReport.GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS, description = "The Java exception class raised when the processor fails"),
+ @WritesAttribute(attribute = GetWorkdayReport.GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE, description = "The Java exception message raised when the processor fails"),
+ @WritesAttribute(attribute = "mime.type", description = "Sets the mime.type attribute to the MIME Type specified by the Source / Record Writer"),
+ @WritesAttribute(attribute= GetWorkdayReport.RECORD_COUNT, description = "The number of records in an outgoing FlowFile. This is only populated on the 'success' relationship "
+ + "when Record Reader and Writer is set.")})
+public class GetWorkdayReport extends AbstractProcessor {
+
+ protected static final String STATUS_CODE = "getworkdayreport.status.code";
+ protected static final String REQUEST_URL = "getworkdayreport.request.url";
+ protected static final String REQUEST_DURATION = "getworkdayreport.request.duration";
+ protected static final String TRANSACTION_ID = "getworkdayreport.tx.id";
+ protected static final String GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS = "getworkdayreport.java.exception.class";
+ protected static final String GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE = "getworkdayreport.java.exception.message";
+ protected static final String RECORD_COUNT = "record.count";
+ protected static final String BASIC_PREFIX = "Basic ";
+ protected static final String COLUMNS_TO_HASH_SEPARATOR = ",";
+ protected static final String HEADER_AUTHORIZATION = "Authorization";
+ protected static final String HEADER_CONTENT_TYPE = "Content-Type";
+ protected static final String USERNAME_PASSWORD_SEPARATOR = ":";
+
+ protected static final PropertyDescriptor REPORT_URL = new PropertyDescriptor.Builder()
+ .name("Workday Report URL")
+ .displayName("Workday Report URL")
+ .description("HTTP remote URL of Workday report including a scheme of http or https, as well as a hostname or IP address with optional port and path elements.")
+ .required(true)
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .addValidator(URL_VALIDATOR)
+ .build();
+
+ protected static final PropertyDescriptor WORKDAY_USERNAME = new PropertyDescriptor.Builder()
+ .name("Workday Username")
+ .displayName("Workday Username")
+ .description("The username provided for authentication of Workday requests. Encoded using Base64 for HTTP Basic Authentication as described in RFC 7617.")
+ .required(true)
+ .addValidator(StandardValidators.createRegexMatchingValidator(Pattern.compile("^[\\x20-\\x39\\x3b-\\x7e\\x80-\\xff]+$")))
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .build();
+
+ protected static final PropertyDescriptor WORKDAY_PASSWORD = new PropertyDescriptor.Builder()
+ .name("Workday Password")
+ .displayName("Workday Password")
+ .description("The password provided for authentication of Workday requests. Encoded using Base64 for HTTP Basic Authentication as described in RFC 7617.")
+ .required(true)
+ .sensitive(true)
+ .addValidator(StandardValidators.createRegexMatchingValidator(Pattern.compile("^[\\x20-\\x7e\\x80-\\xff]+$")))
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .build();
+
+ protected static final PropertyDescriptor WEB_CLIENT_SERVICE = new PropertyDescriptor.Builder()
+ .name("Web Client Service Provider")
+ .description("Web client which is used to communicate with the Workday API.")
+ .required(true)
+ .identifiesControllerService(WebClientServiceProvider.class)
+ .build();
+
+ protected static final PropertyDescriptor FIELDS_TO_HASH = new PropertyDescriptor.Builder()
+ .name("Hashed Fields")
+ .displayName("Hashed Fields")
+ .description("Comma separated record paths to be replaced with their corresponding hash.")
+ .required(false)
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .addValidator(NON_BLANK_VALIDATOR)
+ .build();
+
+ protected static final PropertyDescriptor HASHING_ALGORITHM = new PropertyDescriptor.Builder()
+ .name("Hashing Algorithm")
+ .displayName("Hashing Algorithm")
+ .description("Determines what hashing algorithm should be used to perform the hashing function.")
+ .required(true)
+ .allowableValues(HashAlgorithm.getNames())
+ .defaultValue(HashAlgorithm.SHA256.getName())
+ .addValidator(NON_BLANK_VALIDATOR)
+ .expressionLanguageSupported(NONE)
+ .dependsOn(FIELDS_TO_HASH)
+ .build();
+
+ protected static final PropertyDescriptor RECORD_READER_FACTORY = new PropertyDescriptor.Builder()
+ .name("record-reader")
+ .displayName("Record Reader")
+ .description("Specifies the Controller Service to use for parsing incoming data and determining the data's schema.")
+ .identifiesControllerService(RecordReaderFactory.class)
+ .required(false)
+ .build();
+
+ protected static final PropertyDescriptor RECORD_WRITER_FACTORY = new PropertyDescriptor.Builder()
+ .name("record-writer")
+ .displayName("Record Writer")
+ .description("The Record Writer to use for serializing Records to an output FlowFile.")
+ .identifiesControllerService(RecordSetWriterFactory.class)
+ .dependsOn(RECORD_READER_FACTORY)
+ .required(true)
+ .build();
+
+ protected static final Relationship ORIGINAL = new Relationship.Builder()
+ .name("original")
+ .description("Request FlowFiles transferred when receiving HTTP responses with a status code between 200 and 299.")
+ .build();
+
+ protected static final Relationship FAILURE = new Relationship.Builder()
+ .name("failure")
+ .description("Request FlowFiles transferred when receiving socket communication errors.")
+ .build();
+
+ protected static final Relationship SUCCESS = new Relationship.Builder()
+ .name("success")
+ .description("Response FlowFiles transferred when receiving HTTP responses with a status code between 200 and 299.")
+ .build();
+
+ protected static final Set<Relationship> RELATIONSHIPS = Collections.unmodifiableSet(new HashSet<>(Arrays.asList(ORIGINAL, SUCCESS, FAILURE)));
+ protected static final List<PropertyDescriptor> PROPERTIES = Collections.unmodifiableList(Arrays.asList(
+ REPORT_URL,
+ WORKDAY_USERNAME,
+ WORKDAY_PASSWORD,
+ WEB_CLIENT_SERVICE,
+ FIELDS_TO_HASH,
+ HASHING_ALGORITHM,
+ RECORD_READER_FACTORY,
+ RECORD_WRITER_FACTORY
+ ));
+
+ private final AtomicReference<WebClientService> webClientReference = new AtomicReference<>();
+ private final AtomicReference<RecordReaderFactory> recordReaderFactoryReference = new AtomicReference<>();
+ private final AtomicReference<RecordSetWriterFactory> recordSetWriterFactoryReference = new AtomicReference<>();
+
+ @Override
+ protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
+ return PROPERTIES;
+ }
+
+ @Override
+ public Set<Relationship> getRelationships() {
+ return RELATIONSHIPS;
+ }
+
+ @OnScheduled
+ public void setUpClient(final ProcessContext context) {
+ WebClientServiceProvider standardWebClientServiceProvider = context.getProperty(WEB_CLIENT_SERVICE).asControllerService(WebClientServiceProvider.class);
+ RecordReaderFactory recordReaderFactory = context.getProperty(RECORD_READER_FACTORY).asControllerService(RecordReaderFactory.class);
+ RecordSetWriterFactory recordSetWriterFactory = context.getProperty(RECORD_WRITER_FACTORY).asControllerService(RecordSetWriterFactory.class);
+ WebClientService webClientService = standardWebClientServiceProvider.getWebClientService();
+ webClientReference.set(webClientService);
+ recordReaderFactoryReference.set(recordReaderFactory);
+ recordSetWriterFactoryReference.set(recordSetWriterFactory);
+ }
+
+ @Override
+ public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException {
+ FlowFile flowFile = session.get();
+
+ if (skipExecution(context, flowFile)) {
+ return;
+ }
+
+ FlowFile responseFlowFile = null;
+
+ try {
+ WebClientService webClientService = webClientReference.get();
+ URI uri = new URI(context.getProperty(REPORT_URL).evaluateAttributeExpressions(flowFile).getValue().trim());
+ long startNanos = System.nanoTime();
+ String authorization = createAuthorizationHeader(context, flowFile);
+
+ try(HttpResponseEntity httpResponseEntity = webClientService.get()
+ .uri(uri)
+ .header(HEADER_AUTHORIZATION, authorization)
+ .retrieve()) {
+ responseFlowFile = createResponseFlowFile(flowFile, session, context, httpResponseEntity);
+ long elapsedTime = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos);
+ Map<String, String> commonAttributes = createCommonAttributes(uri, httpResponseEntity, elapsedTime);
+
+ if (flowFile != null) {
+ flowFile = session.putAllAttributes(flowFile, setMimeType(commonAttributes, httpResponseEntity));
+ }
+ if (responseFlowFile != null) {
+ responseFlowFile = session.putAllAttributes(responseFlowFile, commonAttributes);
+ if (flowFile == null) {
+ session.getProvenanceReporter().receive(responseFlowFile, uri.toString(), elapsedTime);
+ } else {
+ session.getProvenanceReporter().fetch(responseFlowFile, uri.toString(), elapsedTime);
+ }
+ }
+
+ route(flowFile, responseFlowFile, session, context, httpResponseEntity.statusCode());
+ }
+ } catch (Exception e) {
+ if (flowFile == null) {
+ getLogger().error("Request Processing failed", e);
+ context.yield();
+ } else {
+ getLogger().error("Request Processing failed: {}", flowFile, e);
+ session.penalize(flowFile);
+ flowFile = session.putAttribute(flowFile, GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS, e.getClass().getSimpleName());
+ flowFile = session.putAttribute(flowFile, GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE, e.getMessage());
+ session.transfer(flowFile, FAILURE);
+ }
+
+ if (responseFlowFile != null) {
+ session.remove(responseFlowFile);
+ }
+ }
+ }
+
+ @Override
+ protected Collection<ValidationResult> customValidate(ValidationContext validationContext) {
+ List<ValidationResult> results = new ArrayList<>(super.customValidate(validationContext));
+ if (validationContext.getProperty(FIELDS_TO_HASH).isSet() && !validationContext.getProperty(RECORD_READER_FACTORY).isSet()) {
+ results.add(new ValidationResult.Builder()
+ .valid(false)
+ .explanation("Record-Reader and Record-Writer must be set if you would like to hash specific fields")
+ .subject("Workday report configuration")
+ .build());
+ }
+ return results;
+ }
+
+ /*
+ * If we have no FlowFile, and all incoming connections are self-loops then we can continue on.
+ * However, if we have no FlowFile and we have connections coming from other Processors, then
+ * we know that we should run only if we have a FlowFile.
+ */
+ private boolean skipExecution(ProcessContext context, FlowFile flowfile) {
+ return context.hasIncomingConnection() && flowfile == null && context.hasNonLoopConnection();
+ }
+
+ private FlowFile createResponseFlowFile(FlowFile flowfile, ProcessSession session, ProcessContext context, HttpResponseEntity httpResponseEntity)
+ throws IOException, SchemaNotFoundException, MalformedRecordException {
+ FlowFile responseFlowFile = null;
+ String hashingAlgorithm = context.getProperty(HASHING_ALGORITHM).getValue();
+ Set<String> columnsToHash = Optional.ofNullable(
+ context.getProperty(FIELDS_TO_HASH)
+ .evaluateAttributeExpressions(flowfile)
+ .getValue())
+ .map(String::trim)
+ .map(columns -> columns.split(COLUMNS_TO_HASH_SEPARATOR))
+ .map(Arrays::stream)
+ .map(columns -> columns.collect(Collectors.toSet()))
+ .orElse(Collections.emptySet());
+ try {
+ if (isSuccess(httpResponseEntity.statusCode())) {
+ responseFlowFile = flowfile == null ? session.create() : session.create(flowfile);
+ InputStream responseBodyStream = httpResponseEntity.body();
+ if (recordReaderFactoryReference.get() != null) {
+ TransformResult transformResult = transformRecords(session, flowfile, responseFlowFile, hashingAlgorithm, columnsToHash, responseBodyStream);
+ Map<String, String> attributes = new HashMap<>();
+ attributes.put(RECORD_COUNT, String.valueOf(transformResult.getNumberOfRecords()));
+ attributes.put(CoreAttributes.MIME_TYPE.key(), transformResult.getMimeType());
+ responseFlowFile = session.putAllAttributes(responseFlowFile, attributes);
+ } else {
+ responseFlowFile = session.importFrom(responseBodyStream, responseFlowFile);
+ Optional<String> mimeType = httpResponseEntity.headers().getFirstHeader(HEADER_CONTENT_TYPE);
+ if (mimeType.isPresent()) {
+ responseFlowFile = session.putAttribute(responseFlowFile, CoreAttributes.MIME_TYPE.key(), mimeType.get());
+ }
+ }
+ }
+ } catch (Exception e) {
+ session.remove(responseFlowFile);
+ throw e;
+ }
+ return responseFlowFile;
+ }
+
+ private String createAuthorizationHeader(ProcessContext context, FlowFile flowfile) {
+ String userName = context.getProperty(WORKDAY_USERNAME).evaluateAttributeExpressions(flowfile).getValue();
+ String password = context.getProperty(WORKDAY_PASSWORD).evaluateAttributeExpressions(flowfile).getValue();
+ String base64Credential = Base64.getEncoder().encodeToString((userName + USERNAME_PASSWORD_SEPARATOR + password).getBytes(StandardCharsets.UTF_8));
+ return BASIC_PREFIX + base64Credential;
+ }
+
+ private TransformResult transformRecords(ProcessSession session, FlowFile flowfile, FlowFile responseFlowFile, String hashingAlgorithm, Set<String> columnsToHash,
+ InputStream responseBodyStream) throws IOException, SchemaNotFoundException, MalformedRecordException {
+ int numberOfRecords = 0;
+ String mimeType = null;
+ try (RecordReader reader = recordReaderFactoryReference.get().createRecordReader(flowfile,
+ new BufferedInputStream(responseBodyStream), getLogger())) {
+ RecordSchema schema = recordSetWriterFactoryReference.get()
+ .getSchema(flowfile == null ? Collections.emptyMap() : flowfile.getAttributes(), reader.getSchema());
+ try (OutputStream responseStream = session.write(responseFlowFile);
+ RecordSetWriter recordSetWriter = recordSetWriterFactoryReference.get().createWriter(getLogger(), schema, responseStream, responseFlowFile)) {
+ mimeType = recordSetWriter.getMimeType();
+ recordSetWriter.beginRecordSet();
+ Record currentRecord;
+ // as the report can be changed independently from the flow, it's safer to ignore field types and unknown fields in the Record Reading process
+ while ((currentRecord = reader.nextRecord(false, true)) != null) {
+ for (String recordPath : columnsToHash) {
+ RecordPathResult evaluate = RecordPath.compile("hash(" + recordPath + ", '" + hashingAlgorithm + "')").evaluate(currentRecord);
+ evaluate.getSelectedFields().forEach(fieldVal -> fieldVal.updateValue(fieldVal.getValue(), RecordFieldType.STRING.getDataType()));
+ }
+ currentRecord.incorporateInactiveFields();
+ recordSetWriter.write(currentRecord);
+ numberOfRecords++;
+ }
+ }
+ }
+ return new TransformResult(numberOfRecords, mimeType);
+ }
+
+ private void route(FlowFile request, FlowFile response, ProcessSession session, ProcessContext context, int statusCode) {
+ if (!isSuccess(statusCode) && request == null) {
+ context.yield();
+ }
+
+ if (isSuccess(statusCode)) {
+ if (request != null) {
+ session.transfer(request, ORIGINAL);
+ }
+ if (response != null) {
+ session.transfer(response, SUCCESS);
+ }
+ } else {
+ if (request != null) {
+ session.transfer(request, FAILURE);
+ }
+ }
+ }
+
+ private boolean isSuccess(int statusCode) {
+ return statusCode >= 200 && statusCode < 300;
+ }
+
+ private Map<String, String> createCommonAttributes(URI uri, HttpResponseEntity httpResponseEntity, long elapsedTime) {
+ Map<String, String> attributes = new HashMap<>();
+ attributes.put(STATUS_CODE, String.valueOf(httpResponseEntity.statusCode()));
+ attributes.put(REQUEST_URL, uri.toString());
+ attributes.put(REQUEST_DURATION, Long.toString(elapsedTime));
+ attributes.put(TRANSACTION_ID, UUID.randomUUID().toString());
+ return attributes;
+ }
+
+ private Map<String, String> setMimeType(Map<String, String> commonAttributes, HttpResponseEntity httpResponseEntity) {
+ Map<String, String> attributes = commonAttributes;
+ Optional<String> contentType = httpResponseEntity.headers().getFirstHeader(HEADER_CONTENT_TYPE);
+ if (contentType.isPresent()) {
+ attributes = new HashMap<>(commonAttributes);
+ attributes.put(CoreAttributes.MIME_TYPE.key(), contentType.get());
+ }
+ return attributes;
+ }
+
+ protected enum HashAlgorithm {
+ SHA256("SHA-256"),
+ SHA512("SHA-512");
+
+ private final String name;
+
+ HashAlgorithm(String name) {
+ this.name = name;
+ }
+
+ private String getName() {
+ return name;
+ }
+
+ private static String[] getNames() {
+ return Arrays.stream(HashAlgorithm.values()).map(HashAlgorithm::getName).toArray(String[]::new);
+ }
+ }
+
+ private class TransformResult {
Review Comment:
This class could be declared `static` since it does not reference any instance variables of the parent class.
##########
nifi-nar-bundles/nifi-workday-bundle/nifi-workday-processors/src/main/java/org/apache/nifi/processors/workday/GetWorkdayReport.java:
##########
@@ -0,0 +1,466 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.processors.workday;
+
+import static org.apache.nifi.expression.ExpressionLanguageScope.FLOWFILE_ATTRIBUTES;
+import static org.apache.nifi.expression.ExpressionLanguageScope.NONE;
+import static org.apache.nifi.processor.util.StandardValidators.NON_BLANK_VALIDATOR;
+import static org.apache.nifi.processor.util.StandardValidators.URL_VALIDATOR;
+
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Base64;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import org.apache.nifi.annotation.behavior.EventDriven;
+import org.apache.nifi.annotation.behavior.InputRequirement;
+import org.apache.nifi.annotation.behavior.InputRequirement.Requirement;
+import org.apache.nifi.annotation.behavior.SideEffectFree;
+import org.apache.nifi.annotation.behavior.SupportsBatching;
+import org.apache.nifi.annotation.behavior.WritesAttribute;
+import org.apache.nifi.annotation.behavior.WritesAttributes;
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.annotation.lifecycle.OnScheduled;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.ValidationContext;
+import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.flowfile.attributes.CoreAttributes;
+import org.apache.nifi.processor.AbstractProcessor;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.processor.util.StandardValidators;
+import org.apache.nifi.record.path.RecordPath;
+import org.apache.nifi.record.path.RecordPathResult;
+import org.apache.nifi.schema.access.SchemaNotFoundException;
+import org.apache.nifi.serialization.MalformedRecordException;
+import org.apache.nifi.serialization.RecordReader;
+import org.apache.nifi.serialization.RecordReaderFactory;
+import org.apache.nifi.serialization.RecordSetWriter;
+import org.apache.nifi.serialization.RecordSetWriterFactory;
+import org.apache.nifi.serialization.record.Record;
+import org.apache.nifi.serialization.record.RecordFieldType;
+import org.apache.nifi.serialization.record.RecordSchema;
+import org.apache.nifi.web.client.api.HttpResponseEntity;
+import org.apache.nifi.web.client.api.WebClientService;
+import org.apache.nifi.web.client.provider.api.WebClientServiceProvider;
+
+@Tags({"Workday", "report"})
+@InputRequirement(Requirement.INPUT_ALLOWED)
+@CapabilityDescription("A processor which can interact with a configurable Workday Report. The processor can forward the content without modification, or you can transform it by"
+ + " providing the specific Record Reader and Record Writer services based on your needs. You can also hash, or remove fields using the input parameters and schemes in the Record Writer. "
+ + "Supported Workday report formats are: csv, simplexml, json")
+@EventDriven
+@SideEffectFree
+@SupportsBatching
+@WritesAttributes({
+ @WritesAttribute(attribute = GetWorkdayReport.GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS, description = "The Java exception class raised when the processor fails"),
+ @WritesAttribute(attribute = GetWorkdayReport.GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE, description = "The Java exception message raised when the processor fails"),
+ @WritesAttribute(attribute = "mime.type", description = "Sets the mime.type attribute to the MIME Type specified by the Source / Record Writer"),
+ @WritesAttribute(attribute= GetWorkdayReport.RECORD_COUNT, description = "The number of records in an outgoing FlowFile. This is only populated on the 'success' relationship "
+ + "when Record Reader and Writer is set.")})
+public class GetWorkdayReport extends AbstractProcessor {
+
+ protected static final String STATUS_CODE = "getworkdayreport.status.code";
+ protected static final String REQUEST_URL = "getworkdayreport.request.url";
+ protected static final String REQUEST_DURATION = "getworkdayreport.request.duration";
+ protected static final String TRANSACTION_ID = "getworkdayreport.tx.id";
+ protected static final String GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS = "getworkdayreport.java.exception.class";
+ protected static final String GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE = "getworkdayreport.java.exception.message";
+ protected static final String RECORD_COUNT = "record.count";
+ protected static final String BASIC_PREFIX = "Basic ";
+ protected static final String COLUMNS_TO_HASH_SEPARATOR = ",";
+ protected static final String HEADER_AUTHORIZATION = "Authorization";
+ protected static final String HEADER_CONTENT_TYPE = "Content-Type";
+ protected static final String USERNAME_PASSWORD_SEPARATOR = ":";
+
+ protected static final PropertyDescriptor REPORT_URL = new PropertyDescriptor.Builder()
+ .name("Workday Report URL")
+ .displayName("Workday Report URL")
+ .description("HTTP remote URL of Workday report including a scheme of http or https, as well as a hostname or IP address with optional port and path elements.")
+ .required(true)
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .addValidator(URL_VALIDATOR)
+ .build();
+
+ protected static final PropertyDescriptor WORKDAY_USERNAME = new PropertyDescriptor.Builder()
+ .name("Workday Username")
+ .displayName("Workday Username")
+ .description("The username provided for authentication of Workday requests. Encoded using Base64 for HTTP Basic Authentication as described in RFC 7617.")
+ .required(true)
+ .addValidator(StandardValidators.createRegexMatchingValidator(Pattern.compile("^[\\x20-\\x39\\x3b-\\x7e\\x80-\\xff]+$")))
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .build();
+
+ protected static final PropertyDescriptor WORKDAY_PASSWORD = new PropertyDescriptor.Builder()
+ .name("Workday Password")
+ .displayName("Workday Password")
+ .description("The password provided for authentication of Workday requests. Encoded using Base64 for HTTP Basic Authentication as described in RFC 7617.")
+ .required(true)
+ .sensitive(true)
+ .addValidator(StandardValidators.createRegexMatchingValidator(Pattern.compile("^[\\x20-\\x7e\\x80-\\xff]+$")))
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .build();
+
+ protected static final PropertyDescriptor WEB_CLIENT_SERVICE = new PropertyDescriptor.Builder()
+ .name("Web Client Service Provider")
+ .description("Web client which is used to communicate with the Workday API.")
+ .required(true)
+ .identifiesControllerService(WebClientServiceProvider.class)
+ .build();
+
+ protected static final PropertyDescriptor FIELDS_TO_HASH = new PropertyDescriptor.Builder()
+ .name("Hashed Fields")
+ .displayName("Hashed Fields")
+ .description("Comma separated record paths to be replaced with their corresponding hash.")
+ .required(false)
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .addValidator(NON_BLANK_VALIDATOR)
+ .build();
+
+ protected static final PropertyDescriptor HASHING_ALGORITHM = new PropertyDescriptor.Builder()
+ .name("Hashing Algorithm")
+ .displayName("Hashing Algorithm")
+ .description("Determines what hashing algorithm should be used to perform the hashing function.")
+ .required(true)
+ .allowableValues(HashAlgorithm.getNames())
+ .defaultValue(HashAlgorithm.SHA256.getName())
+ .addValidator(NON_BLANK_VALIDATOR)
+ .expressionLanguageSupported(NONE)
+ .dependsOn(FIELDS_TO_HASH)
+ .build();
+
+ protected static final PropertyDescriptor RECORD_READER_FACTORY = new PropertyDescriptor.Builder()
+ .name("record-reader")
+ .displayName("Record Reader")
+ .description("Specifies the Controller Service to use for parsing incoming data and determining the data's schema.")
+ .identifiesControllerService(RecordReaderFactory.class)
+ .required(false)
+ .build();
+
+ protected static final PropertyDescriptor RECORD_WRITER_FACTORY = new PropertyDescriptor.Builder()
+ .name("record-writer")
+ .displayName("Record Writer")
+ .description("The Record Writer to use for serializing Records to an output FlowFile.")
+ .identifiesControllerService(RecordSetWriterFactory.class)
+ .dependsOn(RECORD_READER_FACTORY)
+ .required(true)
+ .build();
+
+ protected static final Relationship ORIGINAL = new Relationship.Builder()
+ .name("original")
+ .description("Request FlowFiles transferred when receiving HTTP responses with a status code between 200 and 299.")
+ .build();
+
+ protected static final Relationship FAILURE = new Relationship.Builder()
+ .name("failure")
+ .description("Request FlowFiles transferred when receiving socket communication errors.")
+ .build();
+
+ protected static final Relationship SUCCESS = new Relationship.Builder()
+ .name("success")
+ .description("Response FlowFiles transferred when receiving HTTP responses with a status code between 200 and 299.")
+ .build();
+
+ protected static final Set<Relationship> RELATIONSHIPS = Collections.unmodifiableSet(new HashSet<>(Arrays.asList(ORIGINAL, SUCCESS, FAILURE)));
+ protected static final List<PropertyDescriptor> PROPERTIES = Collections.unmodifiableList(Arrays.asList(
+ REPORT_URL,
+ WORKDAY_USERNAME,
+ WORKDAY_PASSWORD,
+ WEB_CLIENT_SERVICE,
+ FIELDS_TO_HASH,
+ HASHING_ALGORITHM,
+ RECORD_READER_FACTORY,
+ RECORD_WRITER_FACTORY
+ ));
+
+ private final AtomicReference<WebClientService> webClientReference = new AtomicReference<>();
+ private final AtomicReference<RecordReaderFactory> recordReaderFactoryReference = new AtomicReference<>();
+ private final AtomicReference<RecordSetWriterFactory> recordSetWriterFactoryReference = new AtomicReference<>();
+
+ @Override
+ protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
+ return PROPERTIES;
+ }
+
+ @Override
+ public Set<Relationship> getRelationships() {
+ return RELATIONSHIPS;
+ }
+
+ @OnScheduled
+ public void setUpClient(final ProcessContext context) {
+ WebClientServiceProvider standardWebClientServiceProvider = context.getProperty(WEB_CLIENT_SERVICE).asControllerService(WebClientServiceProvider.class);
+ RecordReaderFactory recordReaderFactory = context.getProperty(RECORD_READER_FACTORY).asControllerService(RecordReaderFactory.class);
+ RecordSetWriterFactory recordSetWriterFactory = context.getProperty(RECORD_WRITER_FACTORY).asControllerService(RecordSetWriterFactory.class);
+ WebClientService webClientService = standardWebClientServiceProvider.getWebClientService();
+ webClientReference.set(webClientService);
+ recordReaderFactoryReference.set(recordReaderFactory);
+ recordSetWriterFactoryReference.set(recordSetWriterFactory);
+ }
+
+ @Override
+ public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException {
+ FlowFile flowFile = session.get();
+
+ if (skipExecution(context, flowFile)) {
+ return;
+ }
+
+ FlowFile responseFlowFile = null;
+
+ try {
+ WebClientService webClientService = webClientReference.get();
+ URI uri = new URI(context.getProperty(REPORT_URL).evaluateAttributeExpressions(flowFile).getValue().trim());
+ long startNanos = System.nanoTime();
+ String authorization = createAuthorizationHeader(context, flowFile);
+
+ try(HttpResponseEntity httpResponseEntity = webClientService.get()
+ .uri(uri)
+ .header(HEADER_AUTHORIZATION, authorization)
+ .retrieve()) {
+ responseFlowFile = createResponseFlowFile(flowFile, session, context, httpResponseEntity);
+ long elapsedTime = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos);
+ Map<String, String> commonAttributes = createCommonAttributes(uri, httpResponseEntity, elapsedTime);
+
+ if (flowFile != null) {
+ flowFile = session.putAllAttributes(flowFile, setMimeType(commonAttributes, httpResponseEntity));
+ }
+ if (responseFlowFile != null) {
+ responseFlowFile = session.putAllAttributes(responseFlowFile, commonAttributes);
+ if (flowFile == null) {
+ session.getProvenanceReporter().receive(responseFlowFile, uri.toString(), elapsedTime);
+ } else {
+ session.getProvenanceReporter().fetch(responseFlowFile, uri.toString(), elapsedTime);
+ }
+ }
+
+ route(flowFile, responseFlowFile, session, context, httpResponseEntity.statusCode());
+ }
+ } catch (Exception e) {
+ if (flowFile == null) {
+ getLogger().error("Request Processing failed", e);
+ context.yield();
+ } else {
+ getLogger().error("Request Processing failed: {}", flowFile, e);
+ session.penalize(flowFile);
+ flowFile = session.putAttribute(flowFile, GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS, e.getClass().getSimpleName());
+ flowFile = session.putAttribute(flowFile, GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE, e.getMessage());
+ session.transfer(flowFile, FAILURE);
+ }
+
+ if (responseFlowFile != null) {
+ session.remove(responseFlowFile);
+ }
+ }
+ }
+
+ @Override
+ protected Collection<ValidationResult> customValidate(ValidationContext validationContext) {
+ List<ValidationResult> results = new ArrayList<>(super.customValidate(validationContext));
+ if (validationContext.getProperty(FIELDS_TO_HASH).isSet() && !validationContext.getProperty(RECORD_READER_FACTORY).isSet()) {
+ results.add(new ValidationResult.Builder()
+ .valid(false)
+ .explanation("Record-Reader and Record-Writer must be set if you would like to hash specific fields")
+ .subject("Workday report configuration")
+ .build());
+ }
+ return results;
+ }
+
+ /*
+ * If we have no FlowFile, and all incoming connections are self-loops then we can continue on.
+ * However, if we have no FlowFile and we have connections coming from other Processors, then
+ * we know that we should run only if we have a FlowFile.
+ */
+ private boolean skipExecution(ProcessContext context, FlowFile flowfile) {
+ return context.hasIncomingConnection() && flowfile == null && context.hasNonLoopConnection();
+ }
+
+ private FlowFile createResponseFlowFile(FlowFile flowfile, ProcessSession session, ProcessContext context, HttpResponseEntity httpResponseEntity)
+ throws IOException, SchemaNotFoundException, MalformedRecordException {
+ FlowFile responseFlowFile = null;
+ String hashingAlgorithm = context.getProperty(HASHING_ALGORITHM).getValue();
+ Set<String> columnsToHash = Optional.ofNullable(
+ context.getProperty(FIELDS_TO_HASH)
+ .evaluateAttributeExpressions(flowfile)
+ .getValue())
+ .map(String::trim)
+ .map(columns -> columns.split(COLUMNS_TO_HASH_SEPARATOR))
+ .map(Arrays::stream)
+ .map(columns -> columns.collect(Collectors.toSet()))
+ .orElse(Collections.emptySet());
+ try {
+ if (isSuccess(httpResponseEntity.statusCode())) {
+ responseFlowFile = flowfile == null ? session.create() : session.create(flowfile);
+ InputStream responseBodyStream = httpResponseEntity.body();
+ if (recordReaderFactoryReference.get() != null) {
+ TransformResult transformResult = transformRecords(session, flowfile, responseFlowFile, hashingAlgorithm, columnsToHash, responseBodyStream);
+ Map<String, String> attributes = new HashMap<>();
+ attributes.put(RECORD_COUNT, String.valueOf(transformResult.getNumberOfRecords()));
+ attributes.put(CoreAttributes.MIME_TYPE.key(), transformResult.getMimeType());
+ responseFlowFile = session.putAllAttributes(responseFlowFile, attributes);
+ } else {
+ responseFlowFile = session.importFrom(responseBodyStream, responseFlowFile);
+ Optional<String> mimeType = httpResponseEntity.headers().getFirstHeader(HEADER_CONTENT_TYPE);
+ if (mimeType.isPresent()) {
+ responseFlowFile = session.putAttribute(responseFlowFile, CoreAttributes.MIME_TYPE.key(), mimeType.get());
+ }
+ }
+ }
+ } catch (Exception e) {
+ session.remove(responseFlowFile);
+ throw e;
+ }
+ return responseFlowFile;
+ }
+
+ private String createAuthorizationHeader(ProcessContext context, FlowFile flowfile) {
+ String userName = context.getProperty(WORKDAY_USERNAME).evaluateAttributeExpressions(flowfile).getValue();
+ String password = context.getProperty(WORKDAY_PASSWORD).evaluateAttributeExpressions(flowfile).getValue();
+ String base64Credential = Base64.getEncoder().encodeToString((userName + USERNAME_PASSWORD_SEPARATOR + password).getBytes(StandardCharsets.UTF_8));
+ return BASIC_PREFIX + base64Credential;
+ }
+
+ private TransformResult transformRecords(ProcessSession session, FlowFile flowfile, FlowFile responseFlowFile, String hashingAlgorithm, Set<String> columnsToHash,
+ InputStream responseBodyStream) throws IOException, SchemaNotFoundException, MalformedRecordException {
+ int numberOfRecords = 0;
+ String mimeType = null;
+ try (RecordReader reader = recordReaderFactoryReference.get().createRecordReader(flowfile,
+ new BufferedInputStream(responseBodyStream), getLogger())) {
+ RecordSchema schema = recordSetWriterFactoryReference.get()
+ .getSchema(flowfile == null ? Collections.emptyMap() : flowfile.getAttributes(), reader.getSchema());
+ try (OutputStream responseStream = session.write(responseFlowFile);
+ RecordSetWriter recordSetWriter = recordSetWriterFactoryReference.get().createWriter(getLogger(), schema, responseStream, responseFlowFile)) {
+ mimeType = recordSetWriter.getMimeType();
+ recordSetWriter.beginRecordSet();
+ Record currentRecord;
+ // as the report can be changed independently from the flow, it's safer to ignore field types and unknown fields in the Record Reading process
+ while ((currentRecord = reader.nextRecord(false, true)) != null) {
+ for (String recordPath : columnsToHash) {
+ RecordPathResult evaluate = RecordPath.compile("hash(" + recordPath + ", '" + hashingAlgorithm + "')").evaluate(currentRecord);
+ evaluate.getSelectedFields().forEach(fieldVal -> fieldVal.updateValue(fieldVal.getValue(), RecordFieldType.STRING.getDataType()));
+ }
Review Comment:
Taking a closer look at this implementation, what is the reason for supporting this hash operation? Compiling and running the record path evaluation could take time for larger record sets. It seems like it would be better to remove this feature, which could be performed if desired in a downstream processor.
##########
nifi-nar-bundles/nifi-workday-bundle/nifi-workday-processors/src/main/java/org/apache/nifi/processors/workday/GetWorkdayReport.java:
##########
@@ -0,0 +1,466 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.processors.workday;
+
+import static org.apache.nifi.expression.ExpressionLanguageScope.FLOWFILE_ATTRIBUTES;
+import static org.apache.nifi.expression.ExpressionLanguageScope.NONE;
+import static org.apache.nifi.processor.util.StandardValidators.NON_BLANK_VALIDATOR;
+import static org.apache.nifi.processor.util.StandardValidators.URL_VALIDATOR;
+
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Base64;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import org.apache.nifi.annotation.behavior.EventDriven;
+import org.apache.nifi.annotation.behavior.InputRequirement;
+import org.apache.nifi.annotation.behavior.InputRequirement.Requirement;
+import org.apache.nifi.annotation.behavior.SideEffectFree;
+import org.apache.nifi.annotation.behavior.SupportsBatching;
+import org.apache.nifi.annotation.behavior.WritesAttribute;
+import org.apache.nifi.annotation.behavior.WritesAttributes;
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.annotation.lifecycle.OnScheduled;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.ValidationContext;
+import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.flowfile.attributes.CoreAttributes;
+import org.apache.nifi.processor.AbstractProcessor;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.processor.util.StandardValidators;
+import org.apache.nifi.record.path.RecordPath;
+import org.apache.nifi.record.path.RecordPathResult;
+import org.apache.nifi.schema.access.SchemaNotFoundException;
+import org.apache.nifi.serialization.MalformedRecordException;
+import org.apache.nifi.serialization.RecordReader;
+import org.apache.nifi.serialization.RecordReaderFactory;
+import org.apache.nifi.serialization.RecordSetWriter;
+import org.apache.nifi.serialization.RecordSetWriterFactory;
+import org.apache.nifi.serialization.record.Record;
+import org.apache.nifi.serialization.record.RecordFieldType;
+import org.apache.nifi.serialization.record.RecordSchema;
+import org.apache.nifi.web.client.api.HttpResponseEntity;
+import org.apache.nifi.web.client.api.WebClientService;
+import org.apache.nifi.web.client.provider.api.WebClientServiceProvider;
+
+@Tags({"Workday", "report"})
+@InputRequirement(Requirement.INPUT_ALLOWED)
+@CapabilityDescription("A processor which can interact with a configurable Workday Report. The processor can forward the content without modification, or you can transform it by"
+ + " providing the specific Record Reader and Record Writer services based on your needs. You can also hash, or remove fields using the input parameters and schemes in the Record Writer. "
+ + "Supported Workday report formats are: csv, simplexml, json")
+@EventDriven
+@SideEffectFree
+@SupportsBatching
+@WritesAttributes({
+ @WritesAttribute(attribute = GetWorkdayReport.GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS, description = "The Java exception class raised when the processor fails"),
+ @WritesAttribute(attribute = GetWorkdayReport.GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE, description = "The Java exception message raised when the processor fails"),
+ @WritesAttribute(attribute = "mime.type", description = "Sets the mime.type attribute to the MIME Type specified by the Source / Record Writer"),
+ @WritesAttribute(attribute= GetWorkdayReport.RECORD_COUNT, description = "The number of records in an outgoing FlowFile. This is only populated on the 'success' relationship "
+ + "when Record Reader and Writer is set.")})
+public class GetWorkdayReport extends AbstractProcessor {
+
+ protected static final String STATUS_CODE = "getworkdayreport.status.code";
+ protected static final String REQUEST_URL = "getworkdayreport.request.url";
+ protected static final String REQUEST_DURATION = "getworkdayreport.request.duration";
+ protected static final String TRANSACTION_ID = "getworkdayreport.tx.id";
+ protected static final String GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS = "getworkdayreport.java.exception.class";
+ protected static final String GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE = "getworkdayreport.java.exception.message";
+ protected static final String RECORD_COUNT = "record.count";
+ protected static final String BASIC_PREFIX = "Basic ";
+ protected static final String COLUMNS_TO_HASH_SEPARATOR = ",";
+ protected static final String HEADER_AUTHORIZATION = "Authorization";
+ protected static final String HEADER_CONTENT_TYPE = "Content-Type";
+ protected static final String USERNAME_PASSWORD_SEPARATOR = ":";
+
+ protected static final PropertyDescriptor REPORT_URL = new PropertyDescriptor.Builder()
+ .name("Workday Report URL")
+ .displayName("Workday Report URL")
+ .description("HTTP remote URL of Workday report including a scheme of http or https, as well as a hostname or IP address with optional port and path elements.")
+ .required(true)
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .addValidator(URL_VALIDATOR)
+ .build();
+
+ protected static final PropertyDescriptor WORKDAY_USERNAME = new PropertyDescriptor.Builder()
+ .name("Workday Username")
+ .displayName("Workday Username")
+ .description("The username provided for authentication of Workday requests. Encoded using Base64 for HTTP Basic Authentication as described in RFC 7617.")
+ .required(true)
+ .addValidator(StandardValidators.createRegexMatchingValidator(Pattern.compile("^[\\x20-\\x39\\x3b-\\x7e\\x80-\\xff]+$")))
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .build();
+
+ protected static final PropertyDescriptor WORKDAY_PASSWORD = new PropertyDescriptor.Builder()
+ .name("Workday Password")
+ .displayName("Workday Password")
+ .description("The password provided for authentication of Workday requests. Encoded using Base64 for HTTP Basic Authentication as described in RFC 7617.")
+ .required(true)
+ .sensitive(true)
+ .addValidator(StandardValidators.createRegexMatchingValidator(Pattern.compile("^[\\x20-\\x7e\\x80-\\xff]+$")))
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .build();
+
+ protected static final PropertyDescriptor WEB_CLIENT_SERVICE = new PropertyDescriptor.Builder()
+ .name("Web Client Service Provider")
+ .description("Web client which is used to communicate with the Workday API.")
+ .required(true)
+ .identifiesControllerService(WebClientServiceProvider.class)
+ .build();
+
+ protected static final PropertyDescriptor FIELDS_TO_HASH = new PropertyDescriptor.Builder()
+ .name("Hashed Fields")
+ .displayName("Hashed Fields")
+ .description("Comma separated record paths to be replaced with their corresponding hash.")
+ .required(false)
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .addValidator(NON_BLANK_VALIDATOR)
+ .build();
+
+ protected static final PropertyDescriptor HASHING_ALGORITHM = new PropertyDescriptor.Builder()
+ .name("Hashing Algorithm")
+ .displayName("Hashing Algorithm")
+ .description("Determines what hashing algorithm should be used to perform the hashing function.")
+ .required(true)
+ .allowableValues(HashAlgorithm.getNames())
+ .defaultValue(HashAlgorithm.SHA256.getName())
+ .addValidator(NON_BLANK_VALIDATOR)
+ .expressionLanguageSupported(NONE)
+ .dependsOn(FIELDS_TO_HASH)
+ .build();
+
+ protected static final PropertyDescriptor RECORD_READER_FACTORY = new PropertyDescriptor.Builder()
+ .name("record-reader")
+ .displayName("Record Reader")
+ .description("Specifies the Controller Service to use for parsing incoming data and determining the data's schema.")
+ .identifiesControllerService(RecordReaderFactory.class)
+ .required(false)
+ .build();
+
+ protected static final PropertyDescriptor RECORD_WRITER_FACTORY = new PropertyDescriptor.Builder()
+ .name("record-writer")
+ .displayName("Record Writer")
+ .description("The Record Writer to use for serializing Records to an output FlowFile.")
+ .identifiesControllerService(RecordSetWriterFactory.class)
+ .dependsOn(RECORD_READER_FACTORY)
+ .required(true)
+ .build();
+
+ protected static final Relationship ORIGINAL = new Relationship.Builder()
+ .name("original")
+ .description("Request FlowFiles transferred when receiving HTTP responses with a status code between 200 and 299.")
+ .build();
+
+ protected static final Relationship FAILURE = new Relationship.Builder()
+ .name("failure")
+ .description("Request FlowFiles transferred when receiving socket communication errors.")
+ .build();
+
+ protected static final Relationship SUCCESS = new Relationship.Builder()
+ .name("success")
+ .description("Response FlowFiles transferred when receiving HTTP responses with a status code between 200 and 299.")
+ .build();
+
+ protected static final Set<Relationship> RELATIONSHIPS = Collections.unmodifiableSet(new HashSet<>(Arrays.asList(ORIGINAL, SUCCESS, FAILURE)));
+ protected static final List<PropertyDescriptor> PROPERTIES = Collections.unmodifiableList(Arrays.asList(
+ REPORT_URL,
+ WORKDAY_USERNAME,
+ WORKDAY_PASSWORD,
+ WEB_CLIENT_SERVICE,
+ FIELDS_TO_HASH,
+ HASHING_ALGORITHM,
+ RECORD_READER_FACTORY,
+ RECORD_WRITER_FACTORY
+ ));
+
+ private final AtomicReference<WebClientService> webClientReference = new AtomicReference<>();
+ private final AtomicReference<RecordReaderFactory> recordReaderFactoryReference = new AtomicReference<>();
+ private final AtomicReference<RecordSetWriterFactory> recordSetWriterFactoryReference = new AtomicReference<>();
+
+ @Override
+ protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
+ return PROPERTIES;
+ }
+
+ @Override
+ public Set<Relationship> getRelationships() {
+ return RELATIONSHIPS;
+ }
+
+ @OnScheduled
+ public void setUpClient(final ProcessContext context) {
+ WebClientServiceProvider standardWebClientServiceProvider = context.getProperty(WEB_CLIENT_SERVICE).asControllerService(WebClientServiceProvider.class);
+ RecordReaderFactory recordReaderFactory = context.getProperty(RECORD_READER_FACTORY).asControllerService(RecordReaderFactory.class);
+ RecordSetWriterFactory recordSetWriterFactory = context.getProperty(RECORD_WRITER_FACTORY).asControllerService(RecordSetWriterFactory.class);
+ WebClientService webClientService = standardWebClientServiceProvider.getWebClientService();
+ webClientReference.set(webClientService);
+ recordReaderFactoryReference.set(recordReaderFactory);
+ recordSetWriterFactoryReference.set(recordSetWriterFactory);
+ }
+
+ @Override
+ public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException {
+ FlowFile flowFile = session.get();
+
+ if (skipExecution(context, flowFile)) {
+ return;
+ }
+
+ FlowFile responseFlowFile = null;
+
+ try {
+ WebClientService webClientService = webClientReference.get();
+ URI uri = new URI(context.getProperty(REPORT_URL).evaluateAttributeExpressions(flowFile).getValue().trim());
+ long startNanos = System.nanoTime();
+ String authorization = createAuthorizationHeader(context, flowFile);
+
+ try(HttpResponseEntity httpResponseEntity = webClientService.get()
+ .uri(uri)
+ .header(HEADER_AUTHORIZATION, authorization)
+ .retrieve()) {
+ responseFlowFile = createResponseFlowFile(flowFile, session, context, httpResponseEntity);
+ long elapsedTime = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos);
+ Map<String, String> commonAttributes = createCommonAttributes(uri, httpResponseEntity, elapsedTime);
+
+ if (flowFile != null) {
+ flowFile = session.putAllAttributes(flowFile, setMimeType(commonAttributes, httpResponseEntity));
+ }
+ if (responseFlowFile != null) {
+ responseFlowFile = session.putAllAttributes(responseFlowFile, commonAttributes);
+ if (flowFile == null) {
+ session.getProvenanceReporter().receive(responseFlowFile, uri.toString(), elapsedTime);
+ } else {
+ session.getProvenanceReporter().fetch(responseFlowFile, uri.toString(), elapsedTime);
+ }
+ }
+
+ route(flowFile, responseFlowFile, session, context, httpResponseEntity.statusCode());
+ }
+ } catch (Exception e) {
+ if (flowFile == null) {
+ getLogger().error("Request Processing failed", e);
+ context.yield();
+ } else {
+ getLogger().error("Request Processing failed: {}", flowFile, e);
+ session.penalize(flowFile);
+ flowFile = session.putAttribute(flowFile, GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS, e.getClass().getSimpleName());
+ flowFile = session.putAttribute(flowFile, GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE, e.getMessage());
+ session.transfer(flowFile, FAILURE);
+ }
+
+ if (responseFlowFile != null) {
+ session.remove(responseFlowFile);
+ }
+ }
+ }
+
+ @Override
+ protected Collection<ValidationResult> customValidate(ValidationContext validationContext) {
+ List<ValidationResult> results = new ArrayList<>(super.customValidate(validationContext));
+ if (validationContext.getProperty(FIELDS_TO_HASH).isSet() && !validationContext.getProperty(RECORD_READER_FACTORY).isSet()) {
+ results.add(new ValidationResult.Builder()
+ .valid(false)
+ .explanation("Record-Reader and Record-Writer must be set if you would like to hash specific fields")
+ .subject("Workday report configuration")
+ .build());
+ }
+ return results;
+ }
+
+ /*
+ * If we have no FlowFile, and all incoming connections are self-loops then we can continue on.
+ * However, if we have no FlowFile and we have connections coming from other Processors, then
+ * we know that we should run only if we have a FlowFile.
+ */
+ private boolean skipExecution(ProcessContext context, FlowFile flowfile) {
+ return context.hasIncomingConnection() && flowfile == null && context.hasNonLoopConnection();
+ }
+
+ private FlowFile createResponseFlowFile(FlowFile flowfile, ProcessSession session, ProcessContext context, HttpResponseEntity httpResponseEntity)
+ throws IOException, SchemaNotFoundException, MalformedRecordException {
+ FlowFile responseFlowFile = null;
+ String hashingAlgorithm = context.getProperty(HASHING_ALGORITHM).getValue();
+ Set<String> columnsToHash = Optional.ofNullable(
+ context.getProperty(FIELDS_TO_HASH)
+ .evaluateAttributeExpressions(flowfile)
+ .getValue())
+ .map(String::trim)
+ .map(columns -> columns.split(COLUMNS_TO_HASH_SEPARATOR))
+ .map(Arrays::stream)
+ .map(columns -> columns.collect(Collectors.toSet()))
+ .orElse(Collections.emptySet());
+ try {
+ if (isSuccess(httpResponseEntity.statusCode())) {
+ responseFlowFile = flowfile == null ? session.create() : session.create(flowfile);
+ InputStream responseBodyStream = httpResponseEntity.body();
+ if (recordReaderFactoryReference.get() != null) {
+ TransformResult transformResult = transformRecords(session, flowfile, responseFlowFile, hashingAlgorithm, columnsToHash, responseBodyStream);
+ Map<String, String> attributes = new HashMap<>();
+ attributes.put(RECORD_COUNT, String.valueOf(transformResult.getNumberOfRecords()));
+ attributes.put(CoreAttributes.MIME_TYPE.key(), transformResult.getMimeType());
+ responseFlowFile = session.putAllAttributes(responseFlowFile, attributes);
+ } else {
+ responseFlowFile = session.importFrom(responseBodyStream, responseFlowFile);
+ Optional<String> mimeType = httpResponseEntity.headers().getFirstHeader(HEADER_CONTENT_TYPE);
+ if (mimeType.isPresent()) {
+ responseFlowFile = session.putAttribute(responseFlowFile, CoreAttributes.MIME_TYPE.key(), mimeType.get());
+ }
+ }
+ }
+ } catch (Exception e) {
+ session.remove(responseFlowFile);
+ throw e;
+ }
+ return responseFlowFile;
+ }
+
+ private String createAuthorizationHeader(ProcessContext context, FlowFile flowfile) {
+ String userName = context.getProperty(WORKDAY_USERNAME).evaluateAttributeExpressions(flowfile).getValue();
+ String password = context.getProperty(WORKDAY_PASSWORD).evaluateAttributeExpressions(flowfile).getValue();
+ String base64Credential = Base64.getEncoder().encodeToString((userName + USERNAME_PASSWORD_SEPARATOR + password).getBytes(StandardCharsets.UTF_8));
+ return BASIC_PREFIX + base64Credential;
+ }
+
+ private TransformResult transformRecords(ProcessSession session, FlowFile flowfile, FlowFile responseFlowFile, String hashingAlgorithm, Set<String> columnsToHash,
+ InputStream responseBodyStream) throws IOException, SchemaNotFoundException, MalformedRecordException {
+ int numberOfRecords = 0;
+ String mimeType = null;
Review Comment:
The `null` initialized value is not necessary:
```suggestion
String mimeType;
```
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@nifi.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [nifi] ferencerdei commented on a diff in pull request #6376: NIFI-10455 Get workday report processor
Posted by GitBox <gi...@apache.org>.
ferencerdei commented on code in PR #6376:
URL: https://github.com/apache/nifi/pull/6376#discussion_r968135355
##########
nifi-nar-bundles/nifi-workday-bundle/nifi-workday-processors/src/main/java/org/apache/nifi/processors/workday/GetWorkdayReport.java:
##########
@@ -0,0 +1,432 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.processors.workday;
+
+import static org.apache.nifi.expression.ExpressionLanguageScope.FLOWFILE_ATTRIBUTES;
+import static org.apache.nifi.expression.ExpressionLanguageScope.NONE;
+import static org.apache.nifi.processor.util.StandardValidators.NON_BLANK_VALIDATOR;
+import static org.apache.nifi.processor.util.StandardValidators.URL_VALIDATOR;
+
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Base64;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import org.apache.nifi.annotation.behavior.EventDriven;
+import org.apache.nifi.annotation.behavior.InputRequirement;
+import org.apache.nifi.annotation.behavior.InputRequirement.Requirement;
+import org.apache.nifi.annotation.behavior.SideEffectFree;
+import org.apache.nifi.annotation.behavior.SupportsBatching;
+import org.apache.nifi.annotation.behavior.WritesAttribute;
+import org.apache.nifi.annotation.behavior.WritesAttributes;
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.annotation.lifecycle.OnScheduled;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.ValidationContext;
+import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.flowfile.attributes.CoreAttributes;
+import org.apache.nifi.logging.ComponentLog;
+import org.apache.nifi.processor.AbstractProcessor;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.processor.util.StandardValidators;
+import org.apache.nifi.record.path.RecordPath;
+import org.apache.nifi.record.path.RecordPathResult;
+import org.apache.nifi.schema.access.SchemaNotFoundException;
+import org.apache.nifi.security.util.crypto.HashAlgorithm;
+import org.apache.nifi.security.util.crypto.HashService;
+import org.apache.nifi.serialization.MalformedRecordException;
+import org.apache.nifi.serialization.RecordReader;
+import org.apache.nifi.serialization.RecordReaderFactory;
+import org.apache.nifi.serialization.RecordSetWriter;
+import org.apache.nifi.serialization.RecordSetWriterFactory;
+import org.apache.nifi.serialization.record.Record;
+import org.apache.nifi.serialization.record.RecordFieldType;
+import org.apache.nifi.serialization.record.RecordSchema;
+import org.apache.nifi.web.client.api.HttpResponseEntity;
+import org.apache.nifi.web.client.api.WebClientService;
+import org.apache.nifi.web.client.provider.api.WebClientServiceProvider;
+
+@Tags({"Workday", "report", "get"})
+@InputRequirement(Requirement.INPUT_ALLOWED)
+@CapabilityDescription("A processor which can interact with a configurable Workday Report. The processor can forward the content without modification, or you can transform it by"
+ + " providing the specific Record Reader and Record Writer services based on your needs. You can also hash, or remove fields using the input parameters and schemes in the Record writer. "
+ + "Supported Workday report formats are: csv, simplexml, json")
+@EventDriven
+@SideEffectFree
+@SupportsBatching
+@WritesAttributes({
+ @WritesAttribute(attribute = GetWorkdayReport.GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS, description = "The Java exception class raised when the processor fails"),
+ @WritesAttribute(attribute = GetWorkdayReport.GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE, description = "The Java exception message raised when the processor fails"),
+ @WritesAttribute(attribute = "mime.type", description = "Sets the mime.type attribute to the MIME Type specified by the Source / Record Writer"),
+ @WritesAttribute(attribute= GetWorkdayReport.RECORD_COUNT, description = "The number of records in an outgoing FlowFile. This is only populated on the 'success' relationship "
+ + "when Record Reader and Writer is set.")})
+public class GetWorkdayReport extends AbstractProcessor {
+
+ protected static final String STATUS_CODE = "getworkdayreport.status.code";
+ protected static final String REQUEST_URL = "getworkdayreport.request.url";
+ protected static final String REQUEST_DURATION = "getworkdayreport.request.duration";
+ protected static final String TRANSACTION_ID = "getworkdayreport.tx.id";
+ protected static final String GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS = "getworkdayreport.java.exception.class";
+ protected static final String GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE = "getworkdayreport.java.exception.message";
+ protected static final String RECORD_COUNT = "record.count";
+ protected static final String BASIC_PREFIX = "Basic ";
+ protected static final String COLUMNS_TO_HASH_SEPARATOR = ",";
+ protected static final String HEADER_AUTHORIZATION = "Authorization";
+ protected static final String HEADER_CONTENT_TYPE = "Content-Type";
+ protected static final String USERNAME_PASSWORD_SEPARATOR = ":";
+
+ protected static final PropertyDescriptor REPORT_URL = new PropertyDescriptor.Builder()
+ .name("Workday report URL")
+ .displayName("Workday report URL")
+ .description("HTTP remote URL of Workday report including a scheme of http or https, as well as a hostname or IP address with optional port and path elements.")
+ .required(true)
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .addValidator(URL_VALIDATOR)
+ .build();
+
+ protected static final PropertyDescriptor WORKDAY_USERNAME = new PropertyDescriptor.Builder()
+ .name("Workday Username")
+ .displayName("Workday Username")
+ .description("The username provided for authentication of Workday requests. Encoded using Base64 for HTTP Basic Authentication as described in RFC 7617.")
+ .required(true)
+ .addValidator(StandardValidators.createRegexMatchingValidator(Pattern.compile("^[\\x20-\\x39\\x3b-\\x7e\\x80-\\xff]+$")))
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .build();
+
+ protected static final PropertyDescriptor WORKDAY_PASSWORD = new PropertyDescriptor.Builder()
+ .name("Workday Password")
+ .displayName("Workday Password")
+ .description("The password provided for authentication of Workday requests. Encoded using Base64 for HTTP Basic Authentication as described in RFC 7617.")
+ .required(true)
+ .sensitive(true)
+ .addValidator(StandardValidators.createRegexMatchingValidator(Pattern.compile("^[\\x20-\\x7e\\x80-\\xff]+$")))
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .build();
+
+ protected static final PropertyDescriptor WEB_CLIENT_SERVICE = new PropertyDescriptor.Builder()
+ .name("Standard Web Client Service")
+ .description("Web client which is used to communicate with the Workday API.")
+ .required(true)
+ .identifiesControllerService(WebClientServiceProvider.class)
+ .build();
+
+ protected static final PropertyDescriptor FIELDS_TO_HASH = new PropertyDescriptor.Builder()
+ .name("Fields to hash")
+ .displayName("Fields to hash")
+ .description("Comma separated record paths to replace with hash.")
+ .required(false)
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .addValidator(NON_BLANK_VALIDATOR)
+ .build();
+
+ protected static final PropertyDescriptor HASHING_ALGORITHM = new PropertyDescriptor.Builder()
+ .name("Hashing algorithm")
+ .displayName("Hashing algorithm")
+ .description("Determines what hashing algorithm should be used to perform the hashing function.")
+ .required(true)
+ .allowableValues(HashService.buildHashAlgorithmAllowableValues())
+ .defaultValue(HashAlgorithm.SHA256.getName())
+ .addValidator(NON_BLANK_VALIDATOR)
+ .expressionLanguageSupported(NONE)
+ .dependsOn(FIELDS_TO_HASH)
+ .build();
+
+ protected static final PropertyDescriptor RECORD_READER_FACTORY = new PropertyDescriptor.Builder()
+ .name("record-reader")
+ .displayName("Record Reader")
+ .description("Specifies the Controller Service to use for parsing incoming data and determining the data's schema.")
+ .identifiesControllerService(RecordReaderFactory.class)
+ .required(false)
+ .build();
+
+ protected static final PropertyDescriptor RECORD_WRITER_FACTORY = new PropertyDescriptor.Builder()
+ .name("record-writer")
+ .displayName("Record Writer")
+ .description("The Record Writer to use for serializing Records to an output FlowFile.")
+ .identifiesControllerService(RecordSetWriterFactory.class)
+ .dependsOn(RECORD_READER_FACTORY)
+ .required(true)
+ .build();
+
+ protected static final Relationship ORIGINAL = new Relationship.Builder()
+ .name("Original")
+ .description("Request FlowFiles transferred when receiving HTTP responses with a status code between 200 and 299.")
+ .build();
+
+ protected static final Relationship FAILURE = new Relationship.Builder()
+ .name("Failure")
+ .description("Request FlowFiles transferred when receiving socket communication errors.")
+ .build();
+
+ protected static final Relationship RESPONSE = new Relationship.Builder()
+ .name("Response")
+ .description("Response FlowFiles transferred when receiving HTTP responses with a status code between 200 and 299.")
+ .build();
+
+ protected static final Set<Relationship> RELATIONSHIPS = Collections.unmodifiableSet(new HashSet<>(Arrays.asList(ORIGINAL, RESPONSE, FAILURE)));
+ protected static final List<PropertyDescriptor> PROPERTIES = Collections.unmodifiableList(Arrays.asList(REPORT_URL, WORKDAY_USERNAME, WORKDAY_PASSWORD, WEB_CLIENT_SERVICE,
+ FIELDS_TO_HASH, HASHING_ALGORITHM, RECORD_READER_FACTORY, RECORD_WRITER_FACTORY));
+
+ private final AtomicReference<WebClientService> webClientAtomicReference = new AtomicReference<>();
+ private final AtomicReference<RecordReaderFactory> recordReaderFactoryAtomicReference = new AtomicReference<>();
+ private final AtomicReference<RecordSetWriterFactory> recordSetWriterFactoryAtomicReference = new AtomicReference<>();
+
+ @Override
+ protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
+ return PROPERTIES;
+ }
+
+ @Override
+ public Set<Relationship> getRelationships() {
+ return RELATIONSHIPS;
+ }
+
+ @OnScheduled
+ public void setUpClient(final ProcessContext context) {
+ WebClientServiceProvider standardWebClientServiceProvider = context.getProperty(WEB_CLIENT_SERVICE).asControllerService(WebClientServiceProvider.class);
+ RecordReaderFactory recordReaderFactory = context.getProperty(RECORD_READER_FACTORY).asControllerService(RecordReaderFactory.class);
+ RecordSetWriterFactory recordSetWriterFactory = context.getProperty(RECORD_WRITER_FACTORY).asControllerService(RecordSetWriterFactory.class);
+ WebClientService webClientService = standardWebClientServiceProvider.getWebClientService();
+ webClientAtomicReference.set(webClientService);
+ recordReaderFactoryAtomicReference.set(recordReaderFactory);
+ recordSetWriterFactoryAtomicReference.set(recordSetWriterFactory);
+ }
+
+ @Override
+ public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException {
+ FlowFile flowfile = session.get();
+
+ if (skipExecution(context, flowfile)) {
+ return;
+ }
+
+ ComponentLog logger = getLogger();
+ FlowFile responseFlowFile = null;
+
+ try {
+ WebClientService webClientService = webClientAtomicReference.get();
+ URI uri = new URI(context.getProperty(REPORT_URL).evaluateAttributeExpressions(flowfile).getValue().trim());
+ long startNanos = System.nanoTime();
+ String authorization = createAuthorizationHeader(context, flowfile);
+
+ try(HttpResponseEntity httpResponseEntity = webClientService.get().uri(uri).header(HEADER_AUTHORIZATION, authorization).retrieve()) {
+ responseFlowFile = createResponseFlowFile(flowfile, session, context, httpResponseEntity);
+ long elapsedTime = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos);
+ Map<String, String> commonAttributes = createCommonAttributes(uri, httpResponseEntity, elapsedTime);
+
+ if (flowfile != null) {
+ flowfile = session.putAllAttributes(flowfile, decorateWithMimeAttribute(commonAttributes, httpResponseEntity));
+ }
+ if (responseFlowFile != null) {
+ responseFlowFile = session.putAllAttributes(responseFlowFile, commonAttributes);
+ if (flowfile != null) {
+ session.getProvenanceReporter().fetch(responseFlowFile, uri.toString(), elapsedTime);
+ } else {
+ session.getProvenanceReporter().receive(responseFlowFile, uri.toString(), elapsedTime);
+ }
+ }
+
+ route(flowfile, responseFlowFile, session, context, httpResponseEntity.statusCode());
+ }
+ } catch (Exception e) {
+ if (flowfile == null) {
+ logger.error("Request Processing failed", e);
+ context.yield();
+ } else {
+ logger.error("Request Processing failed: {}", flowfile, e);
+ session.penalize(flowfile);
+ flowfile = session.putAttribute(flowfile, GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS, e.getClass().getName());
+ flowfile = session.putAttribute(flowfile, GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE, e.getMessage());
+ session.transfer(flowfile, FAILURE);
+ }
+
+ if (responseFlowFile != null) {
+ session.remove(responseFlowFile);
+ }
+ }
+ }
+
+ @Override
+ protected Collection<ValidationResult> customValidate(ValidationContext validationContext) {
+ List<ValidationResult> results = new ArrayList<>(super.customValidate(validationContext));
+ if (validationContext.getProperty(FIELDS_TO_HASH).isSet() && !validationContext.getProperty(RECORD_READER_FACTORY).isSet()) {
+ results.add(new ValidationResult.Builder()
+ .valid(false)
+ .explanation("Record-Reader and Record-Writer must be set if you would like to hash specific fields")
+ .subject("Workday report configuration")
+ .build());
+ }
+ return results;
+ }
+
+ /*
+ * If we have no FlowFile, and all incoming connections are self-loops then we can continue on.
+ * However, if we have no FlowFile and we have connections coming from other Processors, then
+ * we know that we should run only if we have a FlowFile.
+ */
+ private boolean skipExecution(ProcessContext context, FlowFile flowfile) {
+ return context.hasIncomingConnection() && flowfile == null && context.hasNonLoopConnection();
+ }
+
+ private FlowFile createResponseFlowFile(FlowFile flowfile, ProcessSession session, ProcessContext context, HttpResponseEntity httpResponseEntity)
+ throws IOException, SchemaNotFoundException, MalformedRecordException {
+ FlowFile responseFlowFile = null;
+ String hashingAlgorithm = context.getProperty(HASHING_ALGORITHM).getValue();
+ Set<String> columnsToHash = Optional.ofNullable(context.getProperty(FIELDS_TO_HASH).evaluateAttributeExpressions(flowfile).getValue()).map(String::trim)
+ .map(columns -> columns.split(COLUMNS_TO_HASH_SEPARATOR)).map(Arrays::stream).map(columns -> columns.collect(Collectors.toSet())).orElse(Collections.emptySet());
+ try {
+ if (isSuccess(httpResponseEntity.statusCode())) {
+ responseFlowFile = flowfile != null ? session.create(flowfile) : session.create();
+ InputStream responseBodyStream = httpResponseEntity.body();
+ if (recordReaderFactoryAtomicReference.get() != null) {
+ TransformResult transformResult = transformRecords(session, flowfile, responseFlowFile, hashingAlgorithm, columnsToHash, responseBodyStream);
+ Map<String, String> attributes = new HashMap<>();
+ attributes.put(RECORD_COUNT, String.valueOf(transformResult.getNumberOfRecords()));
+ attributes.put(CoreAttributes.MIME_TYPE.key(), transformResult.getMimeType());
+ responseFlowFile = session.putAllAttributes(responseFlowFile, attributes);
+ } else {
+ responseFlowFile = session.importFrom(responseBodyStream, responseFlowFile);
+ Optional<String> mimeType = httpResponseEntity.headers().getFirstHeader(HEADER_CONTENT_TYPE);
+ if (mimeType.isPresent()) {
+ responseFlowFile = session.putAttribute(responseFlowFile, CoreAttributes.MIME_TYPE.key(), mimeType.get());
+ }
+ }
+ }
+ } catch (Exception e) {
+ session.remove(responseFlowFile);
+ throw e;
+ }
+ return responseFlowFile;
+ }
+
+ private String createAuthorizationHeader(ProcessContext context, FlowFile flowfile) {
+ String userName = context.getProperty(WORKDAY_USERNAME).evaluateAttributeExpressions(flowfile).getValue();
+ String password = context.getProperty(WORKDAY_PASSWORD).evaluateAttributeExpressions(flowfile).getValue();
+ String base64Credential = Base64.getEncoder().encodeToString((userName + USERNAME_PASSWORD_SEPARATOR + password).getBytes(StandardCharsets.ISO_8859_1));
+ return BASIC_PREFIX + base64Credential;
+ }
+
+ private TransformResult transformRecords(ProcessSession session, FlowFile flowfile, FlowFile responseFlowFile, String hashingAlgorithm, Set<String> columnsToHash,
+ InputStream responseBodyStream) throws IOException, SchemaNotFoundException, MalformedRecordException {
+ int numberOfRecords = 0;
+ String mimeType = null;
+ try (RecordReader reader = recordReaderFactoryAtomicReference.get().createRecordReader(flowfile,
+ new BufferedInputStream(responseBodyStream), getLogger())) {
+ RecordSchema schema = recordSetWriterFactoryAtomicReference.get()
+ .getSchema(flowfile == null ? Collections.emptyMap() : flowfile.getAttributes(), reader.getSchema());
+ try (OutputStream responseStream = session.write(responseFlowFile);
+ RecordSetWriter recordSetWriter = recordSetWriterFactoryAtomicReference.get().createWriter(getLogger(), schema, responseStream, responseFlowFile)) {
+ mimeType = recordSetWriter.getMimeType();
+ recordSetWriter.beginRecordSet();
+ Record currentRecord;
+ while ((currentRecord = reader.nextRecord(false, true)) != null) {
Review Comment:
added a comment which explains the parameters. I would like to keep the processor parameters as simple as possible so decided to make this not customizable
##########
nifi-nar-bundles/nifi-workday-bundle/nifi-workday-processors/src/main/java/org/apache/nifi/processors/workday/GetWorkdayReport.java:
##########
@@ -0,0 +1,432 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.processors.workday;
+
+import static org.apache.nifi.expression.ExpressionLanguageScope.FLOWFILE_ATTRIBUTES;
+import static org.apache.nifi.expression.ExpressionLanguageScope.NONE;
+import static org.apache.nifi.processor.util.StandardValidators.NON_BLANK_VALIDATOR;
+import static org.apache.nifi.processor.util.StandardValidators.URL_VALIDATOR;
+
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Base64;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import org.apache.nifi.annotation.behavior.EventDriven;
+import org.apache.nifi.annotation.behavior.InputRequirement;
+import org.apache.nifi.annotation.behavior.InputRequirement.Requirement;
+import org.apache.nifi.annotation.behavior.SideEffectFree;
+import org.apache.nifi.annotation.behavior.SupportsBatching;
+import org.apache.nifi.annotation.behavior.WritesAttribute;
+import org.apache.nifi.annotation.behavior.WritesAttributes;
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.annotation.lifecycle.OnScheduled;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.ValidationContext;
+import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.flowfile.attributes.CoreAttributes;
+import org.apache.nifi.logging.ComponentLog;
+import org.apache.nifi.processor.AbstractProcessor;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.processor.util.StandardValidators;
+import org.apache.nifi.record.path.RecordPath;
+import org.apache.nifi.record.path.RecordPathResult;
+import org.apache.nifi.schema.access.SchemaNotFoundException;
+import org.apache.nifi.security.util.crypto.HashAlgorithm;
+import org.apache.nifi.security.util.crypto.HashService;
+import org.apache.nifi.serialization.MalformedRecordException;
+import org.apache.nifi.serialization.RecordReader;
+import org.apache.nifi.serialization.RecordReaderFactory;
+import org.apache.nifi.serialization.RecordSetWriter;
+import org.apache.nifi.serialization.RecordSetWriterFactory;
+import org.apache.nifi.serialization.record.Record;
+import org.apache.nifi.serialization.record.RecordFieldType;
+import org.apache.nifi.serialization.record.RecordSchema;
+import org.apache.nifi.web.client.api.HttpResponseEntity;
+import org.apache.nifi.web.client.api.WebClientService;
+import org.apache.nifi.web.client.provider.api.WebClientServiceProvider;
+
+@Tags({"Workday", "report", "get"})
+@InputRequirement(Requirement.INPUT_ALLOWED)
+@CapabilityDescription("A processor which can interact with a configurable Workday Report. The processor can forward the content without modification, or you can transform it by"
+ + " providing the specific Record Reader and Record Writer services based on your needs. You can also hash, or remove fields using the input parameters and schemes in the Record writer. "
+ + "Supported Workday report formats are: csv, simplexml, json")
+@EventDriven
+@SideEffectFree
+@SupportsBatching
+@WritesAttributes({
+ @WritesAttribute(attribute = GetWorkdayReport.GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS, description = "The Java exception class raised when the processor fails"),
+ @WritesAttribute(attribute = GetWorkdayReport.GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE, description = "The Java exception message raised when the processor fails"),
+ @WritesAttribute(attribute = "mime.type", description = "Sets the mime.type attribute to the MIME Type specified by the Source / Record Writer"),
+ @WritesAttribute(attribute= GetWorkdayReport.RECORD_COUNT, description = "The number of records in an outgoing FlowFile. This is only populated on the 'success' relationship "
+ + "when Record Reader and Writer is set.")})
+public class GetWorkdayReport extends AbstractProcessor {
+
+ protected static final String STATUS_CODE = "getworkdayreport.status.code";
+ protected static final String REQUEST_URL = "getworkdayreport.request.url";
+ protected static final String REQUEST_DURATION = "getworkdayreport.request.duration";
+ protected static final String TRANSACTION_ID = "getworkdayreport.tx.id";
+ protected static final String GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS = "getworkdayreport.java.exception.class";
+ protected static final String GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE = "getworkdayreport.java.exception.message";
+ protected static final String RECORD_COUNT = "record.count";
+ protected static final String BASIC_PREFIX = "Basic ";
+ protected static final String COLUMNS_TO_HASH_SEPARATOR = ",";
+ protected static final String HEADER_AUTHORIZATION = "Authorization";
+ protected static final String HEADER_CONTENT_TYPE = "Content-Type";
+ protected static final String USERNAME_PASSWORD_SEPARATOR = ":";
+
+ protected static final PropertyDescriptor REPORT_URL = new PropertyDescriptor.Builder()
+ .name("Workday report URL")
+ .displayName("Workday report URL")
+ .description("HTTP remote URL of Workday report including a scheme of http or https, as well as a hostname or IP address with optional port and path elements.")
+ .required(true)
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .addValidator(URL_VALIDATOR)
+ .build();
+
+ protected static final PropertyDescriptor WORKDAY_USERNAME = new PropertyDescriptor.Builder()
+ .name("Workday Username")
+ .displayName("Workday Username")
+ .description("The username provided for authentication of Workday requests. Encoded using Base64 for HTTP Basic Authentication as described in RFC 7617.")
+ .required(true)
+ .addValidator(StandardValidators.createRegexMatchingValidator(Pattern.compile("^[\\x20-\\x39\\x3b-\\x7e\\x80-\\xff]+$")))
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .build();
+
+ protected static final PropertyDescriptor WORKDAY_PASSWORD = new PropertyDescriptor.Builder()
+ .name("Workday Password")
+ .displayName("Workday Password")
+ .description("The password provided for authentication of Workday requests. Encoded using Base64 for HTTP Basic Authentication as described in RFC 7617.")
+ .required(true)
+ .sensitive(true)
+ .addValidator(StandardValidators.createRegexMatchingValidator(Pattern.compile("^[\\x20-\\x7e\\x80-\\xff]+$")))
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .build();
+
+ protected static final PropertyDescriptor WEB_CLIENT_SERVICE = new PropertyDescriptor.Builder()
+ .name("Standard Web Client Service")
+ .description("Web client which is used to communicate with the Workday API.")
+ .required(true)
+ .identifiesControllerService(WebClientServiceProvider.class)
+ .build();
+
+ protected static final PropertyDescriptor FIELDS_TO_HASH = new PropertyDescriptor.Builder()
+ .name("Fields to hash")
+ .displayName("Fields to hash")
+ .description("Comma separated record paths to replace with hash.")
+ .required(false)
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .addValidator(NON_BLANK_VALIDATOR)
+ .build();
+
+ protected static final PropertyDescriptor HASHING_ALGORITHM = new PropertyDescriptor.Builder()
+ .name("Hashing algorithm")
+ .displayName("Hashing algorithm")
+ .description("Determines what hashing algorithm should be used to perform the hashing function.")
+ .required(true)
+ .allowableValues(HashService.buildHashAlgorithmAllowableValues())
+ .defaultValue(HashAlgorithm.SHA256.getName())
+ .addValidator(NON_BLANK_VALIDATOR)
+ .expressionLanguageSupported(NONE)
+ .dependsOn(FIELDS_TO_HASH)
+ .build();
+
+ protected static final PropertyDescriptor RECORD_READER_FACTORY = new PropertyDescriptor.Builder()
+ .name("record-reader")
+ .displayName("Record Reader")
+ .description("Specifies the Controller Service to use for parsing incoming data and determining the data's schema.")
+ .identifiesControllerService(RecordReaderFactory.class)
+ .required(false)
+ .build();
+
+ protected static final PropertyDescriptor RECORD_WRITER_FACTORY = new PropertyDescriptor.Builder()
+ .name("record-writer")
+ .displayName("Record Writer")
+ .description("The Record Writer to use for serializing Records to an output FlowFile.")
+ .identifiesControllerService(RecordSetWriterFactory.class)
+ .dependsOn(RECORD_READER_FACTORY)
+ .required(true)
+ .build();
+
+ protected static final Relationship ORIGINAL = new Relationship.Builder()
+ .name("Original")
+ .description("Request FlowFiles transferred when receiving HTTP responses with a status code between 200 and 299.")
+ .build();
+
+ protected static final Relationship FAILURE = new Relationship.Builder()
+ .name("Failure")
+ .description("Request FlowFiles transferred when receiving socket communication errors.")
+ .build();
+
+ protected static final Relationship RESPONSE = new Relationship.Builder()
+ .name("Response")
+ .description("Response FlowFiles transferred when receiving HTTP responses with a status code between 200 and 299.")
+ .build();
+
+ protected static final Set<Relationship> RELATIONSHIPS = Collections.unmodifiableSet(new HashSet<>(Arrays.asList(ORIGINAL, RESPONSE, FAILURE)));
+ protected static final List<PropertyDescriptor> PROPERTIES = Collections.unmodifiableList(Arrays.asList(REPORT_URL, WORKDAY_USERNAME, WORKDAY_PASSWORD, WEB_CLIENT_SERVICE,
+ FIELDS_TO_HASH, HASHING_ALGORITHM, RECORD_READER_FACTORY, RECORD_WRITER_FACTORY));
+
+ private final AtomicReference<WebClientService> webClientAtomicReference = new AtomicReference<>();
+ private final AtomicReference<RecordReaderFactory> recordReaderFactoryAtomicReference = new AtomicReference<>();
+ private final AtomicReference<RecordSetWriterFactory> recordSetWriterFactoryAtomicReference = new AtomicReference<>();
+
+ @Override
+ protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
+ return PROPERTIES;
+ }
+
+ @Override
+ public Set<Relationship> getRelationships() {
+ return RELATIONSHIPS;
+ }
+
+ @OnScheduled
+ public void setUpClient(final ProcessContext context) {
+ WebClientServiceProvider standardWebClientServiceProvider = context.getProperty(WEB_CLIENT_SERVICE).asControllerService(WebClientServiceProvider.class);
+ RecordReaderFactory recordReaderFactory = context.getProperty(RECORD_READER_FACTORY).asControllerService(RecordReaderFactory.class);
+ RecordSetWriterFactory recordSetWriterFactory = context.getProperty(RECORD_WRITER_FACTORY).asControllerService(RecordSetWriterFactory.class);
+ WebClientService webClientService = standardWebClientServiceProvider.getWebClientService();
+ webClientAtomicReference.set(webClientService);
+ recordReaderFactoryAtomicReference.set(recordReaderFactory);
+ recordSetWriterFactoryAtomicReference.set(recordSetWriterFactory);
+ }
+
+ @Override
+ public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException {
+ FlowFile flowfile = session.get();
+
+ if (skipExecution(context, flowfile)) {
+ return;
+ }
+
+ ComponentLog logger = getLogger();
+ FlowFile responseFlowFile = null;
+
+ try {
+ WebClientService webClientService = webClientAtomicReference.get();
+ URI uri = new URI(context.getProperty(REPORT_URL).evaluateAttributeExpressions(flowfile).getValue().trim());
+ long startNanos = System.nanoTime();
+ String authorization = createAuthorizationHeader(context, flowfile);
+
+ try(HttpResponseEntity httpResponseEntity = webClientService.get().uri(uri).header(HEADER_AUTHORIZATION, authorization).retrieve()) {
+ responseFlowFile = createResponseFlowFile(flowfile, session, context, httpResponseEntity);
+ long elapsedTime = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos);
+ Map<String, String> commonAttributes = createCommonAttributes(uri, httpResponseEntity, elapsedTime);
+
+ if (flowfile != null) {
+ flowfile = session.putAllAttributes(flowfile, decorateWithMimeAttribute(commonAttributes, httpResponseEntity));
+ }
+ if (responseFlowFile != null) {
+ responseFlowFile = session.putAllAttributes(responseFlowFile, commonAttributes);
+ if (flowfile != null) {
+ session.getProvenanceReporter().fetch(responseFlowFile, uri.toString(), elapsedTime);
+ } else {
+ session.getProvenanceReporter().receive(responseFlowFile, uri.toString(), elapsedTime);
+ }
+ }
+
+ route(flowfile, responseFlowFile, session, context, httpResponseEntity.statusCode());
+ }
+ } catch (Exception e) {
+ if (flowfile == null) {
+ logger.error("Request Processing failed", e);
+ context.yield();
+ } else {
+ logger.error("Request Processing failed: {}", flowfile, e);
+ session.penalize(flowfile);
+ flowfile = session.putAttribute(flowfile, GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS, e.getClass().getName());
+ flowfile = session.putAttribute(flowfile, GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE, e.getMessage());
+ session.transfer(flowfile, FAILURE);
+ }
+
+ if (responseFlowFile != null) {
+ session.remove(responseFlowFile);
+ }
+ }
+ }
+
+ @Override
+ protected Collection<ValidationResult> customValidate(ValidationContext validationContext) {
+ List<ValidationResult> results = new ArrayList<>(super.customValidate(validationContext));
+ if (validationContext.getProperty(FIELDS_TO_HASH).isSet() && !validationContext.getProperty(RECORD_READER_FACTORY).isSet()) {
+ results.add(new ValidationResult.Builder()
+ .valid(false)
+ .explanation("Record-Reader and Record-Writer must be set if you would like to hash specific fields")
+ .subject("Workday report configuration")
+ .build());
+ }
+ return results;
+ }
+
+ /*
+ * If we have no FlowFile, and all incoming connections are self-loops then we can continue on.
+ * However, if we have no FlowFile and we have connections coming from other Processors, then
+ * we know that we should run only if we have a FlowFile.
+ */
+ private boolean skipExecution(ProcessContext context, FlowFile flowfile) {
+ return context.hasIncomingConnection() && flowfile == null && context.hasNonLoopConnection();
+ }
+
+ private FlowFile createResponseFlowFile(FlowFile flowfile, ProcessSession session, ProcessContext context, HttpResponseEntity httpResponseEntity)
+ throws IOException, SchemaNotFoundException, MalformedRecordException {
+ FlowFile responseFlowFile = null;
+ String hashingAlgorithm = context.getProperty(HASHING_ALGORITHM).getValue();
+ Set<String> columnsToHash = Optional.ofNullable(context.getProperty(FIELDS_TO_HASH).evaluateAttributeExpressions(flowfile).getValue()).map(String::trim)
+ .map(columns -> columns.split(COLUMNS_TO_HASH_SEPARATOR)).map(Arrays::stream).map(columns -> columns.collect(Collectors.toSet())).orElse(Collections.emptySet());
+ try {
+ if (isSuccess(httpResponseEntity.statusCode())) {
+ responseFlowFile = flowfile != null ? session.create(flowfile) : session.create();
+ InputStream responseBodyStream = httpResponseEntity.body();
+ if (recordReaderFactoryAtomicReference.get() != null) {
+ TransformResult transformResult = transformRecords(session, flowfile, responseFlowFile, hashingAlgorithm, columnsToHash, responseBodyStream);
+ Map<String, String> attributes = new HashMap<>();
+ attributes.put(RECORD_COUNT, String.valueOf(transformResult.getNumberOfRecords()));
+ attributes.put(CoreAttributes.MIME_TYPE.key(), transformResult.getMimeType());
+ responseFlowFile = session.putAllAttributes(responseFlowFile, attributes);
+ } else {
+ responseFlowFile = session.importFrom(responseBodyStream, responseFlowFile);
+ Optional<String> mimeType = httpResponseEntity.headers().getFirstHeader(HEADER_CONTENT_TYPE);
+ if (mimeType.isPresent()) {
+ responseFlowFile = session.putAttribute(responseFlowFile, CoreAttributes.MIME_TYPE.key(), mimeType.get());
+ }
+ }
+ }
+ } catch (Exception e) {
+ session.remove(responseFlowFile);
+ throw e;
+ }
+ return responseFlowFile;
+ }
+
+ private String createAuthorizationHeader(ProcessContext context, FlowFile flowfile) {
+ String userName = context.getProperty(WORKDAY_USERNAME).evaluateAttributeExpressions(flowfile).getValue();
+ String password = context.getProperty(WORKDAY_PASSWORD).evaluateAttributeExpressions(flowfile).getValue();
+ String base64Credential = Base64.getEncoder().encodeToString((userName + USERNAME_PASSWORD_SEPARATOR + password).getBytes(StandardCharsets.ISO_8859_1));
+ return BASIC_PREFIX + base64Credential;
+ }
+
+ private TransformResult transformRecords(ProcessSession session, FlowFile flowfile, FlowFile responseFlowFile, String hashingAlgorithm, Set<String> columnsToHash,
+ InputStream responseBodyStream) throws IOException, SchemaNotFoundException, MalformedRecordException {
+ int numberOfRecords = 0;
+ String mimeType = null;
+ try (RecordReader reader = recordReaderFactoryAtomicReference.get().createRecordReader(flowfile,
+ new BufferedInputStream(responseBodyStream), getLogger())) {
+ RecordSchema schema = recordSetWriterFactoryAtomicReference.get()
+ .getSchema(flowfile == null ? Collections.emptyMap() : flowfile.getAttributes(), reader.getSchema());
+ try (OutputStream responseStream = session.write(responseFlowFile);
+ RecordSetWriter recordSetWriter = recordSetWriterFactoryAtomicReference.get().createWriter(getLogger(), schema, responseStream, responseFlowFile)) {
+ mimeType = recordSetWriter.getMimeType();
+ recordSetWriter.beginRecordSet();
+ Record currentRecord;
+ while ((currentRecord = reader.nextRecord(false, true)) != null) {
+ for (String recordPath : columnsToHash) {
+ RecordPathResult evaluate = RecordPath.compile("hash(" + recordPath + ", '" + hashingAlgorithm + "')").evaluate(currentRecord);
+ evaluate.getSelectedFields().forEach(fieldVal -> fieldVal.updateValue(fieldVal.getValue(), RecordFieldType.STRING.getDataType()));
+ }
+ currentRecord.incorporateInactiveFields();
+ recordSetWriter.write(currentRecord);
+ numberOfRecords++;
+ }
+ }
+ }
+ return new TransformResult(numberOfRecords, mimeType);
+ }
+
+ private void route(FlowFile request, FlowFile response, ProcessSession session, ProcessContext context, int statusCode) {
+ if (!isSuccess(statusCode) && request == null) {
+ context.yield();
+ }
+
+ if (isSuccess(statusCode)) {
+ if (request != null) {
+ session.transfer(request, ORIGINAL);
+ }
+ if (response != null) {
+ session.transfer(response, RESPONSE);
+ }
+ } else {
+ if (request != null) {
+ session.transfer(request, FAILURE);
+ }
+ }
+ }
+
+ private boolean isSuccess(int statusCode) {
+ return statusCode / 100 == 2;
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@nifi.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [nifi] ferencerdei commented on a diff in pull request #6376: NIFI-10455 Get workday report processor
Posted by GitBox <gi...@apache.org>.
ferencerdei commented on code in PR #6376:
URL: https://github.com/apache/nifi/pull/6376#discussion_r968134167
##########
nifi-nar-bundles/nifi-workday-bundle/nifi-workday-processors/src/main/java/org/apache/nifi/processors/workday/GetWorkdayReport.java:
##########
@@ -0,0 +1,432 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.processors.workday;
+
+import static org.apache.nifi.expression.ExpressionLanguageScope.FLOWFILE_ATTRIBUTES;
+import static org.apache.nifi.expression.ExpressionLanguageScope.NONE;
+import static org.apache.nifi.processor.util.StandardValidators.NON_BLANK_VALIDATOR;
+import static org.apache.nifi.processor.util.StandardValidators.URL_VALIDATOR;
+
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Base64;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import org.apache.nifi.annotation.behavior.EventDriven;
+import org.apache.nifi.annotation.behavior.InputRequirement;
+import org.apache.nifi.annotation.behavior.InputRequirement.Requirement;
+import org.apache.nifi.annotation.behavior.SideEffectFree;
+import org.apache.nifi.annotation.behavior.SupportsBatching;
+import org.apache.nifi.annotation.behavior.WritesAttribute;
+import org.apache.nifi.annotation.behavior.WritesAttributes;
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.annotation.lifecycle.OnScheduled;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.ValidationContext;
+import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.flowfile.attributes.CoreAttributes;
+import org.apache.nifi.logging.ComponentLog;
+import org.apache.nifi.processor.AbstractProcessor;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.processor.util.StandardValidators;
+import org.apache.nifi.record.path.RecordPath;
+import org.apache.nifi.record.path.RecordPathResult;
+import org.apache.nifi.schema.access.SchemaNotFoundException;
+import org.apache.nifi.security.util.crypto.HashAlgorithm;
+import org.apache.nifi.security.util.crypto.HashService;
+import org.apache.nifi.serialization.MalformedRecordException;
+import org.apache.nifi.serialization.RecordReader;
+import org.apache.nifi.serialization.RecordReaderFactory;
+import org.apache.nifi.serialization.RecordSetWriter;
+import org.apache.nifi.serialization.RecordSetWriterFactory;
+import org.apache.nifi.serialization.record.Record;
+import org.apache.nifi.serialization.record.RecordFieldType;
+import org.apache.nifi.serialization.record.RecordSchema;
+import org.apache.nifi.web.client.api.HttpResponseEntity;
+import org.apache.nifi.web.client.api.WebClientService;
+import org.apache.nifi.web.client.provider.api.WebClientServiceProvider;
+
+@Tags({"Workday", "report", "get"})
+@InputRequirement(Requirement.INPUT_ALLOWED)
+@CapabilityDescription("A processor which can interact with a configurable Workday Report. The processor can forward the content without modification, or you can transform it by"
+ + " providing the specific Record Reader and Record Writer services based on your needs. You can also hash, or remove fields using the input parameters and schemes in the Record writer. "
+ + "Supported Workday report formats are: csv, simplexml, json")
+@EventDriven
+@SideEffectFree
+@SupportsBatching
+@WritesAttributes({
+ @WritesAttribute(attribute = GetWorkdayReport.GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS, description = "The Java exception class raised when the processor fails"),
+ @WritesAttribute(attribute = GetWorkdayReport.GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE, description = "The Java exception message raised when the processor fails"),
+ @WritesAttribute(attribute = "mime.type", description = "Sets the mime.type attribute to the MIME Type specified by the Source / Record Writer"),
+ @WritesAttribute(attribute= GetWorkdayReport.RECORD_COUNT, description = "The number of records in an outgoing FlowFile. This is only populated on the 'success' relationship "
+ + "when Record Reader and Writer is set.")})
+public class GetWorkdayReport extends AbstractProcessor {
+
+ protected static final String STATUS_CODE = "getworkdayreport.status.code";
+ protected static final String REQUEST_URL = "getworkdayreport.request.url";
+ protected static final String REQUEST_DURATION = "getworkdayreport.request.duration";
+ protected static final String TRANSACTION_ID = "getworkdayreport.tx.id";
+ protected static final String GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS = "getworkdayreport.java.exception.class";
+ protected static final String GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE = "getworkdayreport.java.exception.message";
+ protected static final String RECORD_COUNT = "record.count";
+ protected static final String BASIC_PREFIX = "Basic ";
+ protected static final String COLUMNS_TO_HASH_SEPARATOR = ",";
+ protected static final String HEADER_AUTHORIZATION = "Authorization";
+ protected static final String HEADER_CONTENT_TYPE = "Content-Type";
+ protected static final String USERNAME_PASSWORD_SEPARATOR = ":";
+
+ protected static final PropertyDescriptor REPORT_URL = new PropertyDescriptor.Builder()
+ .name("Workday report URL")
+ .displayName("Workday report URL")
+ .description("HTTP remote URL of Workday report including a scheme of http or https, as well as a hostname or IP address with optional port and path elements.")
+ .required(true)
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .addValidator(URL_VALIDATOR)
+ .build();
+
+ protected static final PropertyDescriptor WORKDAY_USERNAME = new PropertyDescriptor.Builder()
+ .name("Workday Username")
+ .displayName("Workday Username")
+ .description("The username provided for authentication of Workday requests. Encoded using Base64 for HTTP Basic Authentication as described in RFC 7617.")
+ .required(true)
+ .addValidator(StandardValidators.createRegexMatchingValidator(Pattern.compile("^[\\x20-\\x39\\x3b-\\x7e\\x80-\\xff]+$")))
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .build();
+
+ protected static final PropertyDescriptor WORKDAY_PASSWORD = new PropertyDescriptor.Builder()
+ .name("Workday Password")
+ .displayName("Workday Password")
+ .description("The password provided for authentication of Workday requests. Encoded using Base64 for HTTP Basic Authentication as described in RFC 7617.")
+ .required(true)
+ .sensitive(true)
+ .addValidator(StandardValidators.createRegexMatchingValidator(Pattern.compile("^[\\x20-\\x7e\\x80-\\xff]+$")))
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .build();
+
+ protected static final PropertyDescriptor WEB_CLIENT_SERVICE = new PropertyDescriptor.Builder()
+ .name("Standard Web Client Service")
+ .description("Web client which is used to communicate with the Workday API.")
+ .required(true)
+ .identifiesControllerService(WebClientServiceProvider.class)
+ .build();
+
+ protected static final PropertyDescriptor FIELDS_TO_HASH = new PropertyDescriptor.Builder()
+ .name("Fields to hash")
+ .displayName("Fields to hash")
+ .description("Comma separated record paths to replace with hash.")
+ .required(false)
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .addValidator(NON_BLANK_VALIDATOR)
+ .build();
+
+ protected static final PropertyDescriptor HASHING_ALGORITHM = new PropertyDescriptor.Builder()
+ .name("Hashing algorithm")
+ .displayName("Hashing algorithm")
+ .description("Determines what hashing algorithm should be used to perform the hashing function.")
+ .required(true)
+ .allowableValues(HashService.buildHashAlgorithmAllowableValues())
Review Comment:
done
##########
nifi-nar-bundles/nifi-workday-bundle/nifi-workday-processors/src/main/java/org/apache/nifi/processors/workday/GetWorkdayReport.java:
##########
@@ -0,0 +1,432 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.processors.workday;
+
+import static org.apache.nifi.expression.ExpressionLanguageScope.FLOWFILE_ATTRIBUTES;
+import static org.apache.nifi.expression.ExpressionLanguageScope.NONE;
+import static org.apache.nifi.processor.util.StandardValidators.NON_BLANK_VALIDATOR;
+import static org.apache.nifi.processor.util.StandardValidators.URL_VALIDATOR;
+
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Base64;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import org.apache.nifi.annotation.behavior.EventDriven;
+import org.apache.nifi.annotation.behavior.InputRequirement;
+import org.apache.nifi.annotation.behavior.InputRequirement.Requirement;
+import org.apache.nifi.annotation.behavior.SideEffectFree;
+import org.apache.nifi.annotation.behavior.SupportsBatching;
+import org.apache.nifi.annotation.behavior.WritesAttribute;
+import org.apache.nifi.annotation.behavior.WritesAttributes;
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.annotation.lifecycle.OnScheduled;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.ValidationContext;
+import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.flowfile.attributes.CoreAttributes;
+import org.apache.nifi.logging.ComponentLog;
+import org.apache.nifi.processor.AbstractProcessor;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.processor.util.StandardValidators;
+import org.apache.nifi.record.path.RecordPath;
+import org.apache.nifi.record.path.RecordPathResult;
+import org.apache.nifi.schema.access.SchemaNotFoundException;
+import org.apache.nifi.security.util.crypto.HashAlgorithm;
+import org.apache.nifi.security.util.crypto.HashService;
+import org.apache.nifi.serialization.MalformedRecordException;
+import org.apache.nifi.serialization.RecordReader;
+import org.apache.nifi.serialization.RecordReaderFactory;
+import org.apache.nifi.serialization.RecordSetWriter;
+import org.apache.nifi.serialization.RecordSetWriterFactory;
+import org.apache.nifi.serialization.record.Record;
+import org.apache.nifi.serialization.record.RecordFieldType;
+import org.apache.nifi.serialization.record.RecordSchema;
+import org.apache.nifi.web.client.api.HttpResponseEntity;
+import org.apache.nifi.web.client.api.WebClientService;
+import org.apache.nifi.web.client.provider.api.WebClientServiceProvider;
+
+@Tags({"Workday", "report", "get"})
+@InputRequirement(Requirement.INPUT_ALLOWED)
+@CapabilityDescription("A processor which can interact with a configurable Workday Report. The processor can forward the content without modification, or you can transform it by"
+ + " providing the specific Record Reader and Record Writer services based on your needs. You can also hash, or remove fields using the input parameters and schemes in the Record writer. "
+ + "Supported Workday report formats are: csv, simplexml, json")
+@EventDriven
+@SideEffectFree
+@SupportsBatching
+@WritesAttributes({
+ @WritesAttribute(attribute = GetWorkdayReport.GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS, description = "The Java exception class raised when the processor fails"),
+ @WritesAttribute(attribute = GetWorkdayReport.GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE, description = "The Java exception message raised when the processor fails"),
+ @WritesAttribute(attribute = "mime.type", description = "Sets the mime.type attribute to the MIME Type specified by the Source / Record Writer"),
+ @WritesAttribute(attribute= GetWorkdayReport.RECORD_COUNT, description = "The number of records in an outgoing FlowFile. This is only populated on the 'success' relationship "
+ + "when Record Reader and Writer is set.")})
+public class GetWorkdayReport extends AbstractProcessor {
+
+ protected static final String STATUS_CODE = "getworkdayreport.status.code";
+ protected static final String REQUEST_URL = "getworkdayreport.request.url";
+ protected static final String REQUEST_DURATION = "getworkdayreport.request.duration";
+ protected static final String TRANSACTION_ID = "getworkdayreport.tx.id";
+ protected static final String GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS = "getworkdayreport.java.exception.class";
+ protected static final String GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE = "getworkdayreport.java.exception.message";
+ protected static final String RECORD_COUNT = "record.count";
+ protected static final String BASIC_PREFIX = "Basic ";
+ protected static final String COLUMNS_TO_HASH_SEPARATOR = ",";
+ protected static final String HEADER_AUTHORIZATION = "Authorization";
+ protected static final String HEADER_CONTENT_TYPE = "Content-Type";
+ protected static final String USERNAME_PASSWORD_SEPARATOR = ":";
+
+ protected static final PropertyDescriptor REPORT_URL = new PropertyDescriptor.Builder()
+ .name("Workday report URL")
+ .displayName("Workday report URL")
+ .description("HTTP remote URL of Workday report including a scheme of http or https, as well as a hostname or IP address with optional port and path elements.")
+ .required(true)
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .addValidator(URL_VALIDATOR)
+ .build();
+
+ protected static final PropertyDescriptor WORKDAY_USERNAME = new PropertyDescriptor.Builder()
+ .name("Workday Username")
+ .displayName("Workday Username")
+ .description("The username provided for authentication of Workday requests. Encoded using Base64 for HTTP Basic Authentication as described in RFC 7617.")
+ .required(true)
+ .addValidator(StandardValidators.createRegexMatchingValidator(Pattern.compile("^[\\x20-\\x39\\x3b-\\x7e\\x80-\\xff]+$")))
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .build();
+
+ protected static final PropertyDescriptor WORKDAY_PASSWORD = new PropertyDescriptor.Builder()
+ .name("Workday Password")
+ .displayName("Workday Password")
+ .description("The password provided for authentication of Workday requests. Encoded using Base64 for HTTP Basic Authentication as described in RFC 7617.")
+ .required(true)
+ .sensitive(true)
+ .addValidator(StandardValidators.createRegexMatchingValidator(Pattern.compile("^[\\x20-\\x7e\\x80-\\xff]+$")))
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .build();
+
+ protected static final PropertyDescriptor WEB_CLIENT_SERVICE = new PropertyDescriptor.Builder()
+ .name("Standard Web Client Service")
+ .description("Web client which is used to communicate with the Workday API.")
+ .required(true)
+ .identifiesControllerService(WebClientServiceProvider.class)
+ .build();
+
+ protected static final PropertyDescriptor FIELDS_TO_HASH = new PropertyDescriptor.Builder()
+ .name("Fields to hash")
+ .displayName("Fields to hash")
+ .description("Comma separated record paths to replace with hash.")
+ .required(false)
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .addValidator(NON_BLANK_VALIDATOR)
+ .build();
+
+ protected static final PropertyDescriptor HASHING_ALGORITHM = new PropertyDescriptor.Builder()
+ .name("Hashing algorithm")
+ .displayName("Hashing algorithm")
+ .description("Determines what hashing algorithm should be used to perform the hashing function.")
+ .required(true)
+ .allowableValues(HashService.buildHashAlgorithmAllowableValues())
+ .defaultValue(HashAlgorithm.SHA256.getName())
+ .addValidator(NON_BLANK_VALIDATOR)
+ .expressionLanguageSupported(NONE)
+ .dependsOn(FIELDS_TO_HASH)
+ .build();
+
+ protected static final PropertyDescriptor RECORD_READER_FACTORY = new PropertyDescriptor.Builder()
+ .name("record-reader")
+ .displayName("Record Reader")
+ .description("Specifies the Controller Service to use for parsing incoming data and determining the data's schema.")
+ .identifiesControllerService(RecordReaderFactory.class)
+ .required(false)
+ .build();
+
+ protected static final PropertyDescriptor RECORD_WRITER_FACTORY = new PropertyDescriptor.Builder()
+ .name("record-writer")
+ .displayName("Record Writer")
+ .description("The Record Writer to use for serializing Records to an output FlowFile.")
+ .identifiesControllerService(RecordSetWriterFactory.class)
+ .dependsOn(RECORD_READER_FACTORY)
+ .required(true)
+ .build();
+
+ protected static final Relationship ORIGINAL = new Relationship.Builder()
+ .name("Original")
+ .description("Request FlowFiles transferred when receiving HTTP responses with a status code between 200 and 299.")
+ .build();
+
+ protected static final Relationship FAILURE = new Relationship.Builder()
+ .name("Failure")
+ .description("Request FlowFiles transferred when receiving socket communication errors.")
+ .build();
+
+ protected static final Relationship RESPONSE = new Relationship.Builder()
+ .name("Response")
+ .description("Response FlowFiles transferred when receiving HTTP responses with a status code between 200 and 299.")
+ .build();
+
+ protected static final Set<Relationship> RELATIONSHIPS = Collections.unmodifiableSet(new HashSet<>(Arrays.asList(ORIGINAL, RESPONSE, FAILURE)));
+ protected static final List<PropertyDescriptor> PROPERTIES = Collections.unmodifiableList(Arrays.asList(REPORT_URL, WORKDAY_USERNAME, WORKDAY_PASSWORD, WEB_CLIENT_SERVICE,
+ FIELDS_TO_HASH, HASHING_ALGORITHM, RECORD_READER_FACTORY, RECORD_WRITER_FACTORY));
+
+ private final AtomicReference<WebClientService> webClientAtomicReference = new AtomicReference<>();
+ private final AtomicReference<RecordReaderFactory> recordReaderFactoryAtomicReference = new AtomicReference<>();
+ private final AtomicReference<RecordSetWriterFactory> recordSetWriterFactoryAtomicReference = new AtomicReference<>();
+
+ @Override
+ protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
+ return PROPERTIES;
+ }
+
+ @Override
+ public Set<Relationship> getRelationships() {
+ return RELATIONSHIPS;
+ }
+
+ @OnScheduled
+ public void setUpClient(final ProcessContext context) {
+ WebClientServiceProvider standardWebClientServiceProvider = context.getProperty(WEB_CLIENT_SERVICE).asControllerService(WebClientServiceProvider.class);
+ RecordReaderFactory recordReaderFactory = context.getProperty(RECORD_READER_FACTORY).asControllerService(RecordReaderFactory.class);
+ RecordSetWriterFactory recordSetWriterFactory = context.getProperty(RECORD_WRITER_FACTORY).asControllerService(RecordSetWriterFactory.class);
+ WebClientService webClientService = standardWebClientServiceProvider.getWebClientService();
+ webClientAtomicReference.set(webClientService);
+ recordReaderFactoryAtomicReference.set(recordReaderFactory);
+ recordSetWriterFactoryAtomicReference.set(recordSetWriterFactory);
+ }
+
+ @Override
+ public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException {
+ FlowFile flowfile = session.get();
+
+ if (skipExecution(context, flowfile)) {
+ return;
+ }
+
+ ComponentLog logger = getLogger();
+ FlowFile responseFlowFile = null;
+
+ try {
+ WebClientService webClientService = webClientAtomicReference.get();
+ URI uri = new URI(context.getProperty(REPORT_URL).evaluateAttributeExpressions(flowfile).getValue().trim());
+ long startNanos = System.nanoTime();
+ String authorization = createAuthorizationHeader(context, flowfile);
+
+ try(HttpResponseEntity httpResponseEntity = webClientService.get().uri(uri).header(HEADER_AUTHORIZATION, authorization).retrieve()) {
+ responseFlowFile = createResponseFlowFile(flowfile, session, context, httpResponseEntity);
+ long elapsedTime = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos);
+ Map<String, String> commonAttributes = createCommonAttributes(uri, httpResponseEntity, elapsedTime);
+
+ if (flowfile != null) {
+ flowfile = session.putAllAttributes(flowfile, decorateWithMimeAttribute(commonAttributes, httpResponseEntity));
+ }
+ if (responseFlowFile != null) {
+ responseFlowFile = session.putAllAttributes(responseFlowFile, commonAttributes);
+ if (flowfile != null) {
+ session.getProvenanceReporter().fetch(responseFlowFile, uri.toString(), elapsedTime);
+ } else {
+ session.getProvenanceReporter().receive(responseFlowFile, uri.toString(), elapsedTime);
+ }
+ }
+
+ route(flowfile, responseFlowFile, session, context, httpResponseEntity.statusCode());
+ }
+ } catch (Exception e) {
+ if (flowfile == null) {
+ logger.error("Request Processing failed", e);
+ context.yield();
+ } else {
+ logger.error("Request Processing failed: {}", flowfile, e);
+ session.penalize(flowfile);
+ flowfile = session.putAttribute(flowfile, GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS, e.getClass().getName());
+ flowfile = session.putAttribute(flowfile, GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE, e.getMessage());
+ session.transfer(flowfile, FAILURE);
+ }
+
+ if (responseFlowFile != null) {
+ session.remove(responseFlowFile);
+ }
+ }
+ }
+
+ @Override
+ protected Collection<ValidationResult> customValidate(ValidationContext validationContext) {
+ List<ValidationResult> results = new ArrayList<>(super.customValidate(validationContext));
+ if (validationContext.getProperty(FIELDS_TO_HASH).isSet() && !validationContext.getProperty(RECORD_READER_FACTORY).isSet()) {
+ results.add(new ValidationResult.Builder()
+ .valid(false)
+ .explanation("Record-Reader and Record-Writer must be set if you would like to hash specific fields")
+ .subject("Workday report configuration")
+ .build());
+ }
+ return results;
+ }
+
+ /*
+ * If we have no FlowFile, and all incoming connections are self-loops then we can continue on.
+ * However, if we have no FlowFile and we have connections coming from other Processors, then
+ * we know that we should run only if we have a FlowFile.
+ */
+ private boolean skipExecution(ProcessContext context, FlowFile flowfile) {
+ return context.hasIncomingConnection() && flowfile == null && context.hasNonLoopConnection();
+ }
+
+ private FlowFile createResponseFlowFile(FlowFile flowfile, ProcessSession session, ProcessContext context, HttpResponseEntity httpResponseEntity)
+ throws IOException, SchemaNotFoundException, MalformedRecordException {
+ FlowFile responseFlowFile = null;
+ String hashingAlgorithm = context.getProperty(HASHING_ALGORITHM).getValue();
+ Set<String> columnsToHash = Optional.ofNullable(context.getProperty(FIELDS_TO_HASH).evaluateAttributeExpressions(flowfile).getValue()).map(String::trim)
+ .map(columns -> columns.split(COLUMNS_TO_HASH_SEPARATOR)).map(Arrays::stream).map(columns -> columns.collect(Collectors.toSet())).orElse(Collections.emptySet());
+ try {
+ if (isSuccess(httpResponseEntity.statusCode())) {
+ responseFlowFile = flowfile != null ? session.create(flowfile) : session.create();
+ InputStream responseBodyStream = httpResponseEntity.body();
+ if (recordReaderFactoryAtomicReference.get() != null) {
+ TransformResult transformResult = transformRecords(session, flowfile, responseFlowFile, hashingAlgorithm, columnsToHash, responseBodyStream);
+ Map<String, String> attributes = new HashMap<>();
+ attributes.put(RECORD_COUNT, String.valueOf(transformResult.getNumberOfRecords()));
+ attributes.put(CoreAttributes.MIME_TYPE.key(), transformResult.getMimeType());
+ responseFlowFile = session.putAllAttributes(responseFlowFile, attributes);
+ } else {
+ responseFlowFile = session.importFrom(responseBodyStream, responseFlowFile);
+ Optional<String> mimeType = httpResponseEntity.headers().getFirstHeader(HEADER_CONTENT_TYPE);
+ if (mimeType.isPresent()) {
+ responseFlowFile = session.putAttribute(responseFlowFile, CoreAttributes.MIME_TYPE.key(), mimeType.get());
+ }
+ }
+ }
+ } catch (Exception e) {
+ session.remove(responseFlowFile);
+ throw e;
+ }
+ return responseFlowFile;
+ }
+
+ private String createAuthorizationHeader(ProcessContext context, FlowFile flowfile) {
+ String userName = context.getProperty(WORKDAY_USERNAME).evaluateAttributeExpressions(flowfile).getValue();
+ String password = context.getProperty(WORKDAY_PASSWORD).evaluateAttributeExpressions(flowfile).getValue();
+ String base64Credential = Base64.getEncoder().encodeToString((userName + USERNAME_PASSWORD_SEPARATOR + password).getBytes(StandardCharsets.ISO_8859_1));
+ return BASIC_PREFIX + base64Credential;
+ }
+
+ private TransformResult transformRecords(ProcessSession session, FlowFile flowfile, FlowFile responseFlowFile, String hashingAlgorithm, Set<String> columnsToHash,
+ InputStream responseBodyStream) throws IOException, SchemaNotFoundException, MalformedRecordException {
+ int numberOfRecords = 0;
+ String mimeType = null;
+ try (RecordReader reader = recordReaderFactoryAtomicReference.get().createRecordReader(flowfile,
+ new BufferedInputStream(responseBodyStream), getLogger())) {
+ RecordSchema schema = recordSetWriterFactoryAtomicReference.get()
+ .getSchema(flowfile == null ? Collections.emptyMap() : flowfile.getAttributes(), reader.getSchema());
+ try (OutputStream responseStream = session.write(responseFlowFile);
+ RecordSetWriter recordSetWriter = recordSetWriterFactoryAtomicReference.get().createWriter(getLogger(), schema, responseStream, responseFlowFile)) {
+ mimeType = recordSetWriter.getMimeType();
+ recordSetWriter.beginRecordSet();
+ Record currentRecord;
+ while ((currentRecord = reader.nextRecord(false, true)) != null) {
+ for (String recordPath : columnsToHash) {
+ RecordPathResult evaluate = RecordPath.compile("hash(" + recordPath + ", '" + hashingAlgorithm + "')").evaluate(currentRecord);
+ evaluate.getSelectedFields().forEach(fieldVal -> fieldVal.updateValue(fieldVal.getValue(), RecordFieldType.STRING.getDataType()));
+ }
+ currentRecord.incorporateInactiveFields();
+ recordSetWriter.write(currentRecord);
+ numberOfRecords++;
+ }
+ }
+ }
+ return new TransformResult(numberOfRecords, mimeType);
+ }
+
+ private void route(FlowFile request, FlowFile response, ProcessSession session, ProcessContext context, int statusCode) {
+ if (!isSuccess(statusCode) && request == null) {
+ context.yield();
+ }
+
+ if (isSuccess(statusCode)) {
+ if (request != null) {
+ session.transfer(request, ORIGINAL);
+ }
+ if (response != null) {
+ session.transfer(response, RESPONSE);
+ }
+ } else {
+ if (request != null) {
+ session.transfer(request, FAILURE);
+ }
+ }
+ }
+
+ private boolean isSuccess(int statusCode) {
+ return statusCode / 100 == 2;
+ }
+
+ private Map<String, String> createCommonAttributes(URI uri, HttpResponseEntity httpResponseEntity, long elapsedTime) {
+ Map<String, String> attributes = new HashMap<>();
+ attributes.put(STATUS_CODE, String.valueOf(httpResponseEntity.statusCode()));
+ attributes.put(REQUEST_URL, uri.toString());
+ attributes.put(REQUEST_DURATION, Long.toString(elapsedTime));
+ attributes.put(TRANSACTION_ID, UUID.randomUUID().toString());
+ return attributes;
+ }
+
+ private Map<String, String> decorateWithMimeAttribute(Map<String, String> commonAttributes, HttpResponseEntity httpResponseEntity) {
Review Comment:
done
##########
nifi-nar-bundles/nifi-workday-bundle/nifi-workday-processors/src/main/java/org/apache/nifi/processors/workday/GetWorkdayReport.java:
##########
@@ -0,0 +1,432 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.processors.workday;
+
+import static org.apache.nifi.expression.ExpressionLanguageScope.FLOWFILE_ATTRIBUTES;
+import static org.apache.nifi.expression.ExpressionLanguageScope.NONE;
+import static org.apache.nifi.processor.util.StandardValidators.NON_BLANK_VALIDATOR;
+import static org.apache.nifi.processor.util.StandardValidators.URL_VALIDATOR;
+
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Base64;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import org.apache.nifi.annotation.behavior.EventDriven;
+import org.apache.nifi.annotation.behavior.InputRequirement;
+import org.apache.nifi.annotation.behavior.InputRequirement.Requirement;
+import org.apache.nifi.annotation.behavior.SideEffectFree;
+import org.apache.nifi.annotation.behavior.SupportsBatching;
+import org.apache.nifi.annotation.behavior.WritesAttribute;
+import org.apache.nifi.annotation.behavior.WritesAttributes;
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.annotation.lifecycle.OnScheduled;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.ValidationContext;
+import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.flowfile.attributes.CoreAttributes;
+import org.apache.nifi.logging.ComponentLog;
+import org.apache.nifi.processor.AbstractProcessor;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.processor.util.StandardValidators;
+import org.apache.nifi.record.path.RecordPath;
+import org.apache.nifi.record.path.RecordPathResult;
+import org.apache.nifi.schema.access.SchemaNotFoundException;
+import org.apache.nifi.security.util.crypto.HashAlgorithm;
+import org.apache.nifi.security.util.crypto.HashService;
+import org.apache.nifi.serialization.MalformedRecordException;
+import org.apache.nifi.serialization.RecordReader;
+import org.apache.nifi.serialization.RecordReaderFactory;
+import org.apache.nifi.serialization.RecordSetWriter;
+import org.apache.nifi.serialization.RecordSetWriterFactory;
+import org.apache.nifi.serialization.record.Record;
+import org.apache.nifi.serialization.record.RecordFieldType;
+import org.apache.nifi.serialization.record.RecordSchema;
+import org.apache.nifi.web.client.api.HttpResponseEntity;
+import org.apache.nifi.web.client.api.WebClientService;
+import org.apache.nifi.web.client.provider.api.WebClientServiceProvider;
+
+@Tags({"Workday", "report", "get"})
Review Comment:
removed
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@nifi.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [nifi] ferencerdei commented on a diff in pull request #6376: NIFI-10455 Get workday report processor
Posted by GitBox <gi...@apache.org>.
ferencerdei commented on code in PR #6376:
URL: https://github.com/apache/nifi/pull/6376#discussion_r968135546
##########
nifi-nar-bundles/nifi-workday-bundle/nifi-workday-processors/src/main/resources/docs/org.apache.nifi.processors.workday.GetWorkdayReport/additionalDetails.html:
##########
@@ -0,0 +1,103 @@
+<!DOCTYPE html>
+<html lang="en">
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<head>
+ <meta charset="utf-8"/>
+ <title>GetWorkdayReport</title>
+ <link rel="stylesheet" href="../../../../../css/component-usage.css" type="text/css"/>
+</head>
+
+<body>
+<h2>Summary</h2>
+<p>
+ This processor acts as a client endpoint to interact with the Workday API.
+ It is capable of reading reports from Workday RaaS and transferring the content directly to the output, or you can define
+ the required Record Reader and RecordSet Writer, so you can transform the report to the required format.
+</p>
+
+<h2>Supported report formats</h2>
+
+<ul>
+ <li>csv</li>
+ <li>simplexml</li>
+ <li>json</li>
+</ul>
+
+<p>
+ In case of json source you need to set the following parameters in the JsonTreeReader:
+ <ul>
+ <li>Starting Field Strategy: Nested Field</li>
+ <li>Starting Field Name: Report_Entry</li>
+ </ul>
+</p>
+
+<p>
+ It is possible to hide specific columns from the response if you define the Writer scheme explicitly in the configuration of the RecordSet Writer.
+</p>
+
+<h2>
+ Example: Remove name2 column from the response
+</h2>
+<p>
+ Let's say we have the following record structure:
+</p>
+<code>
+ <pre>
+ RecordSet (
+ Record (
+ Field "name1" = "value1",
+ Field "name2" = 42
+ ),
+ Record (
+ Field "name1" = "value2",
+ Field "name2" = 84
+ )
+ )
+ </pre>
+</code>
+
+<p>
+ If you would like to remove the "name2" column from the response, then you need to define the following writer schema:
+</p>
+
+<code>
+ <pre>
+ {
+ "name": "test",
+ "namespace": "nifi",
+ "type": "record",
+ "fields": [
+ { "name": "name1", "type": "string" }
+ ]
+ }
+ </pre>
+</code>
+
+<h2>
+ Example: Hash selected columns
+</h2>
+
+<p>
+ If you would like to keep a column in the response, but hash the value, you can do it by defining the record path selector and a hashing algorithm.
+ <ul>
+ <li>Columns to hash: /personalId,/emailAddress</li>
+ <li>Hashing algorithm: SHA-512</li>
+ </ul>
+ The above example hashes the PersonalId and emailAddress columns.
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@nifi.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [nifi] exceptionfactory commented on a diff in pull request #6376: NIFI-10455 Get workday report processor
Posted by GitBox <gi...@apache.org>.
exceptionfactory commented on code in PR #6376:
URL: https://github.com/apache/nifi/pull/6376#discussion_r968420424
##########
nifi-nar-bundles/nifi-workday-bundle/nifi-workday-processors/pom.xml:
##########
@@ -0,0 +1,114 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<!--
+ ~ Licensed to the Apache Software Foundation (ASF) under one or more
+ ~ contributor license agreements. See the NOTICE file distributed with
+ ~ this work for additional information regarding copyright ownership.
+ ~ The ASF licenses this file to You under the Apache License, Version 2.0
+ ~ (the "License"); you may not use this file except in compliance with
+ ~ the License. You may obtain a copy of the License at
+ ~
+ ~ http://www.apache.org/licenses/LICENSE-2.0
+ ~
+ ~ Unless required by applicable law or agreed to in writing, software
+ ~ distributed under the License is distributed on an "AS IS" BASIS,
+ ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ ~ See the License for the specific language governing permissions and
+ ~ limitations under the License.
+ -->
+
+<project xmlns="http://maven.apache.org/POM/4.0.0"
+ xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance"
+ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
+ <parent>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-workday-bundle</artifactId>
+ <version>1.18.0-SNAPSHOT</version>
+ </parent>
+ <modelVersion>4.0.0</modelVersion>
+
+ <artifactId>nifi-workday-processors</artifactId>
+ <packaging>jar</packaging>
+
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-api</artifactId>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-web-client-provider-api</artifactId>
+ <version>1.18.0-SNAPSHOT</version>
+ <scope>provided</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-utils</artifactId>
+ <version>1.18.0-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-record</artifactId>
+ <version>1.18.0-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-record-path</artifactId>
+ <version>1.18.0-SNAPSHOT</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.nifi</groupId>
+ <artifactId>nifi-security-utils</artifactId>
+ <version>1.18.0-SNAPSHOT</version>
+ </dependency>
Review Comment:
Unfortunately this dependency brings in a number of unnecessary dependencies. If the only purpose for the dependency is to bring in the `HashAlgorithm` enum, recommend removing this dependency and creating a new local enum for this processor.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@nifi.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [nifi] exceptionfactory commented on a diff in pull request #6376: NIFI-10455 Get workday report processor
Posted by GitBox <gi...@apache.org>.
exceptionfactory commented on code in PR #6376:
URL: https://github.com/apache/nifi/pull/6376#discussion_r973177252
##########
nifi-nar-bundles/nifi-workday-bundle/nifi-workday-processors/src/main/java/org/apache/nifi/processors/workday/GetWorkdayReport.java:
##########
@@ -0,0 +1,466 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.processors.workday;
+
+import static org.apache.nifi.expression.ExpressionLanguageScope.FLOWFILE_ATTRIBUTES;
+import static org.apache.nifi.expression.ExpressionLanguageScope.NONE;
+import static org.apache.nifi.processor.util.StandardValidators.NON_BLANK_VALIDATOR;
+import static org.apache.nifi.processor.util.StandardValidators.URL_VALIDATOR;
+
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Base64;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import org.apache.nifi.annotation.behavior.EventDriven;
+import org.apache.nifi.annotation.behavior.InputRequirement;
+import org.apache.nifi.annotation.behavior.InputRequirement.Requirement;
+import org.apache.nifi.annotation.behavior.SideEffectFree;
+import org.apache.nifi.annotation.behavior.SupportsBatching;
+import org.apache.nifi.annotation.behavior.WritesAttribute;
+import org.apache.nifi.annotation.behavior.WritesAttributes;
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.annotation.lifecycle.OnScheduled;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.ValidationContext;
+import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.flowfile.attributes.CoreAttributes;
+import org.apache.nifi.processor.AbstractProcessor;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.processor.util.StandardValidators;
+import org.apache.nifi.record.path.RecordPath;
+import org.apache.nifi.record.path.RecordPathResult;
+import org.apache.nifi.schema.access.SchemaNotFoundException;
+import org.apache.nifi.serialization.MalformedRecordException;
+import org.apache.nifi.serialization.RecordReader;
+import org.apache.nifi.serialization.RecordReaderFactory;
+import org.apache.nifi.serialization.RecordSetWriter;
+import org.apache.nifi.serialization.RecordSetWriterFactory;
+import org.apache.nifi.serialization.record.Record;
+import org.apache.nifi.serialization.record.RecordFieldType;
+import org.apache.nifi.serialization.record.RecordSchema;
+import org.apache.nifi.web.client.api.HttpResponseEntity;
+import org.apache.nifi.web.client.api.WebClientService;
+import org.apache.nifi.web.client.provider.api.WebClientServiceProvider;
+
+@Tags({"Workday", "report"})
+@InputRequirement(Requirement.INPUT_ALLOWED)
+@CapabilityDescription("A processor which can interact with a configurable Workday Report. The processor can forward the content without modification, or you can transform it by"
+ + " providing the specific Record Reader and Record Writer services based on your needs. You can also hash, or remove fields using the input parameters and schemes in the Record Writer. "
+ + "Supported Workday report formats are: csv, simplexml, json")
+@EventDriven
+@SideEffectFree
+@SupportsBatching
+@WritesAttributes({
+ @WritesAttribute(attribute = GetWorkdayReport.GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS, description = "The Java exception class raised when the processor fails"),
+ @WritesAttribute(attribute = GetWorkdayReport.GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE, description = "The Java exception message raised when the processor fails"),
+ @WritesAttribute(attribute = "mime.type", description = "Sets the mime.type attribute to the MIME Type specified by the Source / Record Writer"),
+ @WritesAttribute(attribute= GetWorkdayReport.RECORD_COUNT, description = "The number of records in an outgoing FlowFile. This is only populated on the 'success' relationship "
+ + "when Record Reader and Writer is set.")})
+public class GetWorkdayReport extends AbstractProcessor {
+
+ protected static final String STATUS_CODE = "getworkdayreport.status.code";
+ protected static final String REQUEST_URL = "getworkdayreport.request.url";
+ protected static final String REQUEST_DURATION = "getworkdayreport.request.duration";
+ protected static final String TRANSACTION_ID = "getworkdayreport.tx.id";
+ protected static final String GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS = "getworkdayreport.java.exception.class";
+ protected static final String GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE = "getworkdayreport.java.exception.message";
+ protected static final String RECORD_COUNT = "record.count";
+ protected static final String BASIC_PREFIX = "Basic ";
+ protected static final String COLUMNS_TO_HASH_SEPARATOR = ",";
+ protected static final String HEADER_AUTHORIZATION = "Authorization";
+ protected static final String HEADER_CONTENT_TYPE = "Content-Type";
+ protected static final String USERNAME_PASSWORD_SEPARATOR = ":";
+
+ protected static final PropertyDescriptor REPORT_URL = new PropertyDescriptor.Builder()
+ .name("Workday Report URL")
+ .displayName("Workday Report URL")
+ .description("HTTP remote URL of Workday report including a scheme of http or https, as well as a hostname or IP address with optional port and path elements.")
+ .required(true)
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .addValidator(URL_VALIDATOR)
+ .build();
+
+ protected static final PropertyDescriptor WORKDAY_USERNAME = new PropertyDescriptor.Builder()
+ .name("Workday Username")
+ .displayName("Workday Username")
+ .description("The username provided for authentication of Workday requests. Encoded using Base64 for HTTP Basic Authentication as described in RFC 7617.")
+ .required(true)
+ .addValidator(StandardValidators.createRegexMatchingValidator(Pattern.compile("^[\\x20-\\x39\\x3b-\\x7e\\x80-\\xff]+$")))
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .build();
+
+ protected static final PropertyDescriptor WORKDAY_PASSWORD = new PropertyDescriptor.Builder()
+ .name("Workday Password")
+ .displayName("Workday Password")
+ .description("The password provided for authentication of Workday requests. Encoded using Base64 for HTTP Basic Authentication as described in RFC 7617.")
+ .required(true)
+ .sensitive(true)
+ .addValidator(StandardValidators.createRegexMatchingValidator(Pattern.compile("^[\\x20-\\x7e\\x80-\\xff]+$")))
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .build();
+
+ protected static final PropertyDescriptor WEB_CLIENT_SERVICE = new PropertyDescriptor.Builder()
+ .name("Web Client Service Provider")
+ .description("Web client which is used to communicate with the Workday API.")
+ .required(true)
+ .identifiesControllerService(WebClientServiceProvider.class)
+ .build();
+
+ protected static final PropertyDescriptor FIELDS_TO_HASH = new PropertyDescriptor.Builder()
+ .name("Hashed Fields")
+ .displayName("Hashed Fields")
+ .description("Comma separated record paths to be replaced with their corresponding hash.")
+ .required(false)
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .addValidator(NON_BLANK_VALIDATOR)
+ .build();
+
+ protected static final PropertyDescriptor HASHING_ALGORITHM = new PropertyDescriptor.Builder()
+ .name("Hashing Algorithm")
+ .displayName("Hashing Algorithm")
+ .description("Determines what hashing algorithm should be used to perform the hashing function.")
+ .required(true)
+ .allowableValues(HashAlgorithm.getNames())
+ .defaultValue(HashAlgorithm.SHA256.getName())
+ .addValidator(NON_BLANK_VALIDATOR)
+ .expressionLanguageSupported(NONE)
+ .dependsOn(FIELDS_TO_HASH)
+ .build();
+
+ protected static final PropertyDescriptor RECORD_READER_FACTORY = new PropertyDescriptor.Builder()
+ .name("record-reader")
+ .displayName("Record Reader")
+ .description("Specifies the Controller Service to use for parsing incoming data and determining the data's schema.")
+ .identifiesControllerService(RecordReaderFactory.class)
+ .required(false)
+ .build();
+
+ protected static final PropertyDescriptor RECORD_WRITER_FACTORY = new PropertyDescriptor.Builder()
+ .name("record-writer")
+ .displayName("Record Writer")
+ .description("The Record Writer to use for serializing Records to an output FlowFile.")
+ .identifiesControllerService(RecordSetWriterFactory.class)
+ .dependsOn(RECORD_READER_FACTORY)
+ .required(true)
+ .build();
+
+ protected static final Relationship ORIGINAL = new Relationship.Builder()
+ .name("original")
+ .description("Request FlowFiles transferred when receiving HTTP responses with a status code between 200 and 299.")
+ .build();
+
+ protected static final Relationship FAILURE = new Relationship.Builder()
+ .name("failure")
+ .description("Request FlowFiles transferred when receiving socket communication errors.")
+ .build();
+
+ protected static final Relationship SUCCESS = new Relationship.Builder()
+ .name("success")
+ .description("Response FlowFiles transferred when receiving HTTP responses with a status code between 200 and 299.")
+ .build();
+
+ protected static final Set<Relationship> RELATIONSHIPS = Collections.unmodifiableSet(new HashSet<>(Arrays.asList(ORIGINAL, SUCCESS, FAILURE)));
+ protected static final List<PropertyDescriptor> PROPERTIES = Collections.unmodifiableList(Arrays.asList(
+ REPORT_URL,
+ WORKDAY_USERNAME,
+ WORKDAY_PASSWORD,
+ WEB_CLIENT_SERVICE,
+ FIELDS_TO_HASH,
+ HASHING_ALGORITHM,
+ RECORD_READER_FACTORY,
+ RECORD_WRITER_FACTORY
+ ));
+
+ private final AtomicReference<WebClientService> webClientReference = new AtomicReference<>();
+ private final AtomicReference<RecordReaderFactory> recordReaderFactoryReference = new AtomicReference<>();
+ private final AtomicReference<RecordSetWriterFactory> recordSetWriterFactoryReference = new AtomicReference<>();
+
+ @Override
+ protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
+ return PROPERTIES;
+ }
+
+ @Override
+ public Set<Relationship> getRelationships() {
+ return RELATIONSHIPS;
+ }
+
+ @OnScheduled
+ public void setUpClient(final ProcessContext context) {
+ WebClientServiceProvider standardWebClientServiceProvider = context.getProperty(WEB_CLIENT_SERVICE).asControllerService(WebClientServiceProvider.class);
+ RecordReaderFactory recordReaderFactory = context.getProperty(RECORD_READER_FACTORY).asControllerService(RecordReaderFactory.class);
+ RecordSetWriterFactory recordSetWriterFactory = context.getProperty(RECORD_WRITER_FACTORY).asControllerService(RecordSetWriterFactory.class);
+ WebClientService webClientService = standardWebClientServiceProvider.getWebClientService();
+ webClientReference.set(webClientService);
+ recordReaderFactoryReference.set(recordReaderFactory);
+ recordSetWriterFactoryReference.set(recordSetWriterFactory);
+ }
+
+ @Override
+ public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException {
+ FlowFile flowFile = session.get();
+
+ if (skipExecution(context, flowFile)) {
+ return;
+ }
+
+ FlowFile responseFlowFile = null;
+
+ try {
+ WebClientService webClientService = webClientReference.get();
+ URI uri = new URI(context.getProperty(REPORT_URL).evaluateAttributeExpressions(flowFile).getValue().trim());
+ long startNanos = System.nanoTime();
+ String authorization = createAuthorizationHeader(context, flowFile);
+
+ try(HttpResponseEntity httpResponseEntity = webClientService.get()
+ .uri(uri)
+ .header(HEADER_AUTHORIZATION, authorization)
+ .retrieve()) {
+ responseFlowFile = createResponseFlowFile(flowFile, session, context, httpResponseEntity);
+ long elapsedTime = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos);
+ Map<String, String> commonAttributes = createCommonAttributes(uri, httpResponseEntity, elapsedTime);
+
+ if (flowFile != null) {
+ flowFile = session.putAllAttributes(flowFile, setMimeType(commonAttributes, httpResponseEntity));
+ }
+ if (responseFlowFile != null) {
+ responseFlowFile = session.putAllAttributes(responseFlowFile, commonAttributes);
+ if (flowFile == null) {
+ session.getProvenanceReporter().receive(responseFlowFile, uri.toString(), elapsedTime);
+ } else {
+ session.getProvenanceReporter().fetch(responseFlowFile, uri.toString(), elapsedTime);
+ }
+ }
+
+ route(flowFile, responseFlowFile, session, context, httpResponseEntity.statusCode());
+ }
+ } catch (Exception e) {
+ if (flowFile == null) {
+ getLogger().error("Request Processing failed", e);
+ context.yield();
+ } else {
+ getLogger().error("Request Processing failed: {}", flowFile, e);
+ session.penalize(flowFile);
+ flowFile = session.putAttribute(flowFile, GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS, e.getClass().getSimpleName());
+ flowFile = session.putAttribute(flowFile, GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE, e.getMessage());
+ session.transfer(flowFile, FAILURE);
+ }
+
+ if (responseFlowFile != null) {
+ session.remove(responseFlowFile);
+ }
+ }
+ }
+
+ @Override
+ protected Collection<ValidationResult> customValidate(ValidationContext validationContext) {
+ List<ValidationResult> results = new ArrayList<>(super.customValidate(validationContext));
+ if (validationContext.getProperty(FIELDS_TO_HASH).isSet() && !validationContext.getProperty(RECORD_READER_FACTORY).isSet()) {
+ results.add(new ValidationResult.Builder()
+ .valid(false)
+ .explanation("Record-Reader and Record-Writer must be set if you would like to hash specific fields")
+ .subject("Workday report configuration")
+ .build());
+ }
+ return results;
+ }
+
+ /*
+ * If we have no FlowFile, and all incoming connections are self-loops then we can continue on.
+ * However, if we have no FlowFile and we have connections coming from other Processors, then
+ * we know that we should run only if we have a FlowFile.
+ */
+ private boolean skipExecution(ProcessContext context, FlowFile flowfile) {
+ return context.hasIncomingConnection() && flowfile == null && context.hasNonLoopConnection();
+ }
+
+ private FlowFile createResponseFlowFile(FlowFile flowfile, ProcessSession session, ProcessContext context, HttpResponseEntity httpResponseEntity)
+ throws IOException, SchemaNotFoundException, MalformedRecordException {
+ FlowFile responseFlowFile = null;
+ String hashingAlgorithm = context.getProperty(HASHING_ALGORITHM).getValue();
+ Set<String> columnsToHash = Optional.ofNullable(
+ context.getProperty(FIELDS_TO_HASH)
+ .evaluateAttributeExpressions(flowfile)
+ .getValue())
+ .map(String::trim)
+ .map(columns -> columns.split(COLUMNS_TO_HASH_SEPARATOR))
+ .map(Arrays::stream)
+ .map(columns -> columns.collect(Collectors.toSet()))
+ .orElse(Collections.emptySet());
+ try {
+ if (isSuccess(httpResponseEntity.statusCode())) {
+ responseFlowFile = flowfile == null ? session.create() : session.create(flowfile);
+ InputStream responseBodyStream = httpResponseEntity.body();
+ if (recordReaderFactoryReference.get() != null) {
+ TransformResult transformResult = transformRecords(session, flowfile, responseFlowFile, hashingAlgorithm, columnsToHash, responseBodyStream);
+ Map<String, String> attributes = new HashMap<>();
+ attributes.put(RECORD_COUNT, String.valueOf(transformResult.getNumberOfRecords()));
+ attributes.put(CoreAttributes.MIME_TYPE.key(), transformResult.getMimeType());
+ responseFlowFile = session.putAllAttributes(responseFlowFile, attributes);
+ } else {
+ responseFlowFile = session.importFrom(responseBodyStream, responseFlowFile);
+ Optional<String> mimeType = httpResponseEntity.headers().getFirstHeader(HEADER_CONTENT_TYPE);
+ if (mimeType.isPresent()) {
+ responseFlowFile = session.putAttribute(responseFlowFile, CoreAttributes.MIME_TYPE.key(), mimeType.get());
+ }
+ }
+ }
+ } catch (Exception e) {
+ session.remove(responseFlowFile);
+ throw e;
+ }
+ return responseFlowFile;
+ }
+
+ private String createAuthorizationHeader(ProcessContext context, FlowFile flowfile) {
+ String userName = context.getProperty(WORKDAY_USERNAME).evaluateAttributeExpressions(flowfile).getValue();
+ String password = context.getProperty(WORKDAY_PASSWORD).evaluateAttributeExpressions(flowfile).getValue();
+ String base64Credential = Base64.getEncoder().encodeToString((userName + USERNAME_PASSWORD_SEPARATOR + password).getBytes(StandardCharsets.UTF_8));
+ return BASIC_PREFIX + base64Credential;
+ }
+
+ private TransformResult transformRecords(ProcessSession session, FlowFile flowfile, FlowFile responseFlowFile, String hashingAlgorithm, Set<String> columnsToHash,
+ InputStream responseBodyStream) throws IOException, SchemaNotFoundException, MalformedRecordException {
+ int numberOfRecords = 0;
+ String mimeType = null;
+ try (RecordReader reader = recordReaderFactoryReference.get().createRecordReader(flowfile,
+ new BufferedInputStream(responseBodyStream), getLogger())) {
+ RecordSchema schema = recordSetWriterFactoryReference.get()
+ .getSchema(flowfile == null ? Collections.emptyMap() : flowfile.getAttributes(), reader.getSchema());
+ try (OutputStream responseStream = session.write(responseFlowFile);
+ RecordSetWriter recordSetWriter = recordSetWriterFactoryReference.get().createWriter(getLogger(), schema, responseStream, responseFlowFile)) {
+ mimeType = recordSetWriter.getMimeType();
+ recordSetWriter.beginRecordSet();
+ Record currentRecord;
+ // as the report can be changed independently from the flow, it's safer to ignore field types and unknown fields in the Record Reading process
+ while ((currentRecord = reader.nextRecord(false, true)) != null) {
+ for (String recordPath : columnsToHash) {
+ RecordPathResult evaluate = RecordPath.compile("hash(" + recordPath + ", '" + hashingAlgorithm + "')").evaluate(currentRecord);
+ evaluate.getSelectedFields().forEach(fieldVal -> fieldVal.updateValue(fieldVal.getValue(), RecordFieldType.STRING.getDataType()));
+ }
Review Comment:
Thanks!
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@nifi.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [nifi] exceptionfactory closed pull request #6376: NIFI-10455 Get workday report processor
Posted by GitBox <gi...@apache.org>.
exceptionfactory closed pull request #6376: NIFI-10455 Get workday report processor
URL: https://github.com/apache/nifi/pull/6376
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@nifi.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [nifi] exceptionfactory commented on pull request #6376: NIFI-10455 Get workday report processor
Posted by GitBox <gi...@apache.org>.
exceptionfactory commented on PR #6376:
URL: https://github.com/apache/nifi/pull/6376#issuecomment-1246758634
> @exceptionfactory could you help @ferencerdei resolve the dependency issues to help this getting merged quickly?
@arpadboda and @ferencerdei I will take a closer look at the dependencies and follow up with more details soon.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@nifi.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [nifi] exceptionfactory commented on a diff in pull request #6376: NIFI-10455 Get workday report processor
Posted by GitBox <gi...@apache.org>.
exceptionfactory commented on code in PR #6376:
URL: https://github.com/apache/nifi/pull/6376#discussion_r968428920
##########
nifi-nar-bundles/nifi-workday-bundle/nifi-workday-processors/src/main/java/org/apache/nifi/processors/workday/GetWorkdayReport.java:
##########
@@ -0,0 +1,442 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.processors.workday;
+
+import static org.apache.nifi.expression.ExpressionLanguageScope.FLOWFILE_ATTRIBUTES;
+import static org.apache.nifi.expression.ExpressionLanguageScope.NONE;
+import static org.apache.nifi.processor.util.StandardValidators.NON_BLANK_VALIDATOR;
+import static org.apache.nifi.processor.util.StandardValidators.URL_VALIDATOR;
+
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Base64;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import org.apache.nifi.annotation.behavior.EventDriven;
+import org.apache.nifi.annotation.behavior.InputRequirement;
+import org.apache.nifi.annotation.behavior.InputRequirement.Requirement;
+import org.apache.nifi.annotation.behavior.SideEffectFree;
+import org.apache.nifi.annotation.behavior.SupportsBatching;
+import org.apache.nifi.annotation.behavior.WritesAttribute;
+import org.apache.nifi.annotation.behavior.WritesAttributes;
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.annotation.lifecycle.OnScheduled;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.ValidationContext;
+import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.flowfile.attributes.CoreAttributes;
+import org.apache.nifi.logging.ComponentLog;
+import org.apache.nifi.processor.AbstractProcessor;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.processor.util.StandardValidators;
+import org.apache.nifi.record.path.RecordPath;
+import org.apache.nifi.record.path.RecordPathResult;
+import org.apache.nifi.schema.access.SchemaNotFoundException;
+import org.apache.nifi.security.util.crypto.HashAlgorithm;
+import org.apache.nifi.serialization.MalformedRecordException;
+import org.apache.nifi.serialization.RecordReader;
+import org.apache.nifi.serialization.RecordReaderFactory;
+import org.apache.nifi.serialization.RecordSetWriter;
+import org.apache.nifi.serialization.RecordSetWriterFactory;
+import org.apache.nifi.serialization.record.Record;
+import org.apache.nifi.serialization.record.RecordFieldType;
+import org.apache.nifi.serialization.record.RecordSchema;
+import org.apache.nifi.web.client.api.HttpResponseEntity;
+import org.apache.nifi.web.client.api.WebClientService;
+import org.apache.nifi.web.client.provider.api.WebClientServiceProvider;
+
+@Tags({"Workday", "report"})
+@InputRequirement(Requirement.INPUT_ALLOWED)
+@CapabilityDescription("A processor which can interact with a configurable Workday Report. The processor can forward the content without modification, or you can transform it by"
+ + " providing the specific Record Reader and Record Writer services based on your needs. You can also hash, or remove fields using the input parameters and schemes in the Record Writer. "
+ + "Supported Workday report formats are: csv, simplexml, json")
+@EventDriven
+@SideEffectFree
+@SupportsBatching
+@WritesAttributes({
+ @WritesAttribute(attribute = GetWorkdayReport.GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS, description = "The Java exception class raised when the processor fails"),
+ @WritesAttribute(attribute = GetWorkdayReport.GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE, description = "The Java exception message raised when the processor fails"),
+ @WritesAttribute(attribute = "mime.type", description = "Sets the mime.type attribute to the MIME Type specified by the Source / Record Writer"),
+ @WritesAttribute(attribute= GetWorkdayReport.RECORD_COUNT, description = "The number of records in an outgoing FlowFile. This is only populated on the 'success' relationship "
+ + "when Record Reader and Writer is set.")})
+public class GetWorkdayReport extends AbstractProcessor {
+
+ protected static final String STATUS_CODE = "getworkdayreport.status.code";
+ protected static final String REQUEST_URL = "getworkdayreport.request.url";
+ protected static final String REQUEST_DURATION = "getworkdayreport.request.duration";
+ protected static final String TRANSACTION_ID = "getworkdayreport.tx.id";
+ protected static final String GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS = "getworkdayreport.java.exception.class";
+ protected static final String GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE = "getworkdayreport.java.exception.message";
+ protected static final String RECORD_COUNT = "record.count";
+ protected static final String BASIC_PREFIX = "Basic ";
+ protected static final String COLUMNS_TO_HASH_SEPARATOR = ",";
+ protected static final String HEADER_AUTHORIZATION = "Authorization";
+ protected static final String HEADER_CONTENT_TYPE = "Content-Type";
+ protected static final String USERNAME_PASSWORD_SEPARATOR = ":";
+
+ protected static final PropertyDescriptor REPORT_URL = new PropertyDescriptor.Builder()
+ .name("Workday Report URL")
+ .displayName("Workday Report URL")
+ .description("HTTP remote URL of Workday report including a scheme of http or https, as well as a hostname or IP address with optional port and path elements.")
+ .required(true)
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .addValidator(URL_VALIDATOR)
+ .build();
+
+ protected static final PropertyDescriptor WORKDAY_USERNAME = new PropertyDescriptor.Builder()
+ .name("Workday Username")
+ .displayName("Workday Username")
+ .description("The username provided for authentication of Workday requests. Encoded using Base64 for HTTP Basic Authentication as described in RFC 7617.")
+ .required(true)
+ .addValidator(StandardValidators.createRegexMatchingValidator(Pattern.compile("^[\\x20-\\x39\\x3b-\\x7e\\x80-\\xff]+$")))
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .build();
+
+ protected static final PropertyDescriptor WORKDAY_PASSWORD = new PropertyDescriptor.Builder()
+ .name("Workday Password")
+ .displayName("Workday Password")
+ .description("The password provided for authentication of Workday requests. Encoded using Base64 for HTTP Basic Authentication as described in RFC 7617.")
+ .required(true)
+ .sensitive(true)
+ .addValidator(StandardValidators.createRegexMatchingValidator(Pattern.compile("^[\\x20-\\x7e\\x80-\\xff]+$")))
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .build();
+
+ protected static final PropertyDescriptor WEB_CLIENT_SERVICE = new PropertyDescriptor.Builder()
+ .name("Web Client Service Provider")
+ .description("Web client which is used to communicate with the Workday API.")
+ .required(true)
+ .identifiesControllerService(WebClientServiceProvider.class)
+ .build();
+
+ protected static final PropertyDescriptor FIELDS_TO_HASH = new PropertyDescriptor.Builder()
+ .name("Hashed Fields")
+ .displayName("Hashed Fields")
+ .description("Comma separated record paths to be replaced with their corresponding hash.")
+ .required(false)
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .addValidator(NON_BLANK_VALIDATOR)
+ .build();
+
+ protected static final PropertyDescriptor HASHING_ALGORITHM = new PropertyDescriptor.Builder()
+ .name("Hashing Algorithm")
+ .displayName("Hashing Algorithm")
+ .description("Determines what hashing algorithm should be used to perform the hashing function.")
+ .required(true)
+ .allowableValues(HashAlgorithm.SHA256.getName(), HashAlgorithm.SHA512.getName())
+ .defaultValue(HashAlgorithm.SHA256.getName())
+ .addValidator(NON_BLANK_VALIDATOR)
+ .expressionLanguageSupported(NONE)
+ .dependsOn(FIELDS_TO_HASH)
+ .build();
+
+ protected static final PropertyDescriptor RECORD_READER_FACTORY = new PropertyDescriptor.Builder()
+ .name("record-reader")
+ .displayName("Record Reader")
+ .description("Specifies the Controller Service to use for parsing incoming data and determining the data's schema.")
+ .identifiesControllerService(RecordReaderFactory.class)
+ .required(false)
+ .build();
+
+ protected static final PropertyDescriptor RECORD_WRITER_FACTORY = new PropertyDescriptor.Builder()
+ .name("record-writer")
+ .displayName("Record Writer")
+ .description("The Record Writer to use for serializing Records to an output FlowFile.")
+ .identifiesControllerService(RecordSetWriterFactory.class)
+ .dependsOn(RECORD_READER_FACTORY)
+ .required(true)
+ .build();
+
+ protected static final Relationship ORIGINAL = new Relationship.Builder()
+ .name("Original")
+ .description("Request FlowFiles transferred when receiving HTTP responses with a status code between 200 and 299.")
+ .build();
+
+ protected static final Relationship FAILURE = new Relationship.Builder()
+ .name("Failure")
+ .description("Request FlowFiles transferred when receiving socket communication errors.")
+ .build();
+
+ protected static final Relationship RESPONSE = new Relationship.Builder()
+ .name("Response")
+ .description("Response FlowFiles transferred when receiving HTTP responses with a status code between 200 and 299.")
+ .build();
+
+ protected static final Set<Relationship> RELATIONSHIPS = Collections.unmodifiableSet(new HashSet<>(Arrays.asList(ORIGINAL, RESPONSE, FAILURE)));
+ protected static final List<PropertyDescriptor> PROPERTIES = Collections.unmodifiableList(Arrays.asList(REPORT_URL, WORKDAY_USERNAME, WORKDAY_PASSWORD, WEB_CLIENT_SERVICE,
+ FIELDS_TO_HASH, HASHING_ALGORITHM, RECORD_READER_FACTORY, RECORD_WRITER_FACTORY));
+
+ private final AtomicReference<WebClientService> webClientReference = new AtomicReference<>();
+ private final AtomicReference<RecordReaderFactory> recordReaderFactoryReference = new AtomicReference<>();
+ private final AtomicReference<RecordSetWriterFactory> recordSetWriterFactoryReference = new AtomicReference<>();
+
+ @Override
+ protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
+ return PROPERTIES;
+ }
+
+ @Override
+ public Set<Relationship> getRelationships() {
+ return RELATIONSHIPS;
+ }
+
+ @OnScheduled
+ public void setUpClient(final ProcessContext context) {
+ WebClientServiceProvider standardWebClientServiceProvider = context.getProperty(WEB_CLIENT_SERVICE).asControllerService(WebClientServiceProvider.class);
+ RecordReaderFactory recordReaderFactory = context.getProperty(RECORD_READER_FACTORY).asControllerService(RecordReaderFactory.class);
+ RecordSetWriterFactory recordSetWriterFactory = context.getProperty(RECORD_WRITER_FACTORY).asControllerService(RecordSetWriterFactory.class);
+ WebClientService webClientService = standardWebClientServiceProvider.getWebClientService();
+ webClientReference.set(webClientService);
+ recordReaderFactoryReference.set(recordReaderFactory);
+ recordSetWriterFactoryReference.set(recordSetWriterFactory);
+ }
+
+ @Override
+ public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException {
+ FlowFile flowFile = session.get();
+
+ if (skipExecution(context, flowFile)) {
+ return;
+ }
+
+ ComponentLog logger = getLogger();
Review Comment:
The general pattern for writing logs is to call `getLogger()` directly, as opposed to assigning a method-local variable.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@nifi.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [nifi] ferencerdei commented on pull request #6376: NIFI-10455 Get workday report processor
Posted by GitBox <gi...@apache.org>.
ferencerdei commented on PR #6376:
URL: https://github.com/apache/nifi/pull/6376#issuecomment-1243402701
Thanks for the review, I've applied the requested changes.
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@nifi.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
[GitHub] [nifi] ferencerdei commented on a diff in pull request #6376: NIFI-10455 Get workday report processor
Posted by GitBox <gi...@apache.org>.
ferencerdei commented on code in PR #6376:
URL: https://github.com/apache/nifi/pull/6376#discussion_r968132476
##########
nifi-nar-bundles/nifi-workday-bundle/nifi-workday-processors/src/main/java/org/apache/nifi/processors/workday/GetWorkdayReport.java:
##########
@@ -0,0 +1,432 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.processors.workday;
+
+import static org.apache.nifi.expression.ExpressionLanguageScope.FLOWFILE_ATTRIBUTES;
+import static org.apache.nifi.expression.ExpressionLanguageScope.NONE;
+import static org.apache.nifi.processor.util.StandardValidators.NON_BLANK_VALIDATOR;
+import static org.apache.nifi.processor.util.StandardValidators.URL_VALIDATOR;
+
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Base64;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import org.apache.nifi.annotation.behavior.EventDriven;
+import org.apache.nifi.annotation.behavior.InputRequirement;
+import org.apache.nifi.annotation.behavior.InputRequirement.Requirement;
+import org.apache.nifi.annotation.behavior.SideEffectFree;
+import org.apache.nifi.annotation.behavior.SupportsBatching;
+import org.apache.nifi.annotation.behavior.WritesAttribute;
+import org.apache.nifi.annotation.behavior.WritesAttributes;
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.annotation.lifecycle.OnScheduled;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.ValidationContext;
+import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.flowfile.attributes.CoreAttributes;
+import org.apache.nifi.logging.ComponentLog;
+import org.apache.nifi.processor.AbstractProcessor;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.processor.util.StandardValidators;
+import org.apache.nifi.record.path.RecordPath;
+import org.apache.nifi.record.path.RecordPathResult;
+import org.apache.nifi.schema.access.SchemaNotFoundException;
+import org.apache.nifi.security.util.crypto.HashAlgorithm;
+import org.apache.nifi.security.util.crypto.HashService;
+import org.apache.nifi.serialization.MalformedRecordException;
+import org.apache.nifi.serialization.RecordReader;
+import org.apache.nifi.serialization.RecordReaderFactory;
+import org.apache.nifi.serialization.RecordSetWriter;
+import org.apache.nifi.serialization.RecordSetWriterFactory;
+import org.apache.nifi.serialization.record.Record;
+import org.apache.nifi.serialization.record.RecordFieldType;
+import org.apache.nifi.serialization.record.RecordSchema;
+import org.apache.nifi.web.client.api.HttpResponseEntity;
+import org.apache.nifi.web.client.api.WebClientService;
+import org.apache.nifi.web.client.provider.api.WebClientServiceProvider;
+
+@Tags({"Workday", "report", "get"})
+@InputRequirement(Requirement.INPUT_ALLOWED)
+@CapabilityDescription("A processor which can interact with a configurable Workday Report. The processor can forward the content without modification, or you can transform it by"
+ + " providing the specific Record Reader and Record Writer services based on your needs. You can also hash, or remove fields using the input parameters and schemes in the Record writer. "
+ + "Supported Workday report formats are: csv, simplexml, json")
+@EventDriven
+@SideEffectFree
+@SupportsBatching
+@WritesAttributes({
+ @WritesAttribute(attribute = GetWorkdayReport.GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS, description = "The Java exception class raised when the processor fails"),
+ @WritesAttribute(attribute = GetWorkdayReport.GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE, description = "The Java exception message raised when the processor fails"),
+ @WritesAttribute(attribute = "mime.type", description = "Sets the mime.type attribute to the MIME Type specified by the Source / Record Writer"),
+ @WritesAttribute(attribute= GetWorkdayReport.RECORD_COUNT, description = "The number of records in an outgoing FlowFile. This is only populated on the 'success' relationship "
+ + "when Record Reader and Writer is set.")})
+public class GetWorkdayReport extends AbstractProcessor {
+
+ protected static final String STATUS_CODE = "getworkdayreport.status.code";
+ protected static final String REQUEST_URL = "getworkdayreport.request.url";
+ protected static final String REQUEST_DURATION = "getworkdayreport.request.duration";
+ protected static final String TRANSACTION_ID = "getworkdayreport.tx.id";
+ protected static final String GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS = "getworkdayreport.java.exception.class";
+ protected static final String GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE = "getworkdayreport.java.exception.message";
+ protected static final String RECORD_COUNT = "record.count";
+ protected static final String BASIC_PREFIX = "Basic ";
+ protected static final String COLUMNS_TO_HASH_SEPARATOR = ",";
+ protected static final String HEADER_AUTHORIZATION = "Authorization";
+ protected static final String HEADER_CONTENT_TYPE = "Content-Type";
+ protected static final String USERNAME_PASSWORD_SEPARATOR = ":";
+
+ protected static final PropertyDescriptor REPORT_URL = new PropertyDescriptor.Builder()
+ .name("Workday report URL")
+ .displayName("Workday report URL")
+ .description("HTTP remote URL of Workday report including a scheme of http or https, as well as a hostname or IP address with optional port and path elements.")
+ .required(true)
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .addValidator(URL_VALIDATOR)
+ .build();
+
+ protected static final PropertyDescriptor WORKDAY_USERNAME = new PropertyDescriptor.Builder()
+ .name("Workday Username")
+ .displayName("Workday Username")
+ .description("The username provided for authentication of Workday requests. Encoded using Base64 for HTTP Basic Authentication as described in RFC 7617.")
+ .required(true)
+ .addValidator(StandardValidators.createRegexMatchingValidator(Pattern.compile("^[\\x20-\\x39\\x3b-\\x7e\\x80-\\xff]+$")))
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .build();
+
+ protected static final PropertyDescriptor WORKDAY_PASSWORD = new PropertyDescriptor.Builder()
+ .name("Workday Password")
+ .displayName("Workday Password")
+ .description("The password provided for authentication of Workday requests. Encoded using Base64 for HTTP Basic Authentication as described in RFC 7617.")
+ .required(true)
+ .sensitive(true)
+ .addValidator(StandardValidators.createRegexMatchingValidator(Pattern.compile("^[\\x20-\\x7e\\x80-\\xff]+$")))
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .build();
+
+ protected static final PropertyDescriptor WEB_CLIENT_SERVICE = new PropertyDescriptor.Builder()
+ .name("Standard Web Client Service")
+ .description("Web client which is used to communicate with the Workday API.")
+ .required(true)
+ .identifiesControllerService(WebClientServiceProvider.class)
+ .build();
+
+ protected static final PropertyDescriptor FIELDS_TO_HASH = new PropertyDescriptor.Builder()
+ .name("Fields to hash")
+ .displayName("Fields to hash")
+ .description("Comma separated record paths to replace with hash.")
+ .required(false)
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .addValidator(NON_BLANK_VALIDATOR)
+ .build();
+
+ protected static final PropertyDescriptor HASHING_ALGORITHM = new PropertyDescriptor.Builder()
+ .name("Hashing algorithm")
+ .displayName("Hashing algorithm")
+ .description("Determines what hashing algorithm should be used to perform the hashing function.")
+ .required(true)
+ .allowableValues(HashService.buildHashAlgorithmAllowableValues())
+ .defaultValue(HashAlgorithm.SHA256.getName())
+ .addValidator(NON_BLANK_VALIDATOR)
+ .expressionLanguageSupported(NONE)
+ .dependsOn(FIELDS_TO_HASH)
+ .build();
+
+ protected static final PropertyDescriptor RECORD_READER_FACTORY = new PropertyDescriptor.Builder()
+ .name("record-reader")
+ .displayName("Record Reader")
+ .description("Specifies the Controller Service to use for parsing incoming data and determining the data's schema.")
+ .identifiesControllerService(RecordReaderFactory.class)
+ .required(false)
+ .build();
+
+ protected static final PropertyDescriptor RECORD_WRITER_FACTORY = new PropertyDescriptor.Builder()
+ .name("record-writer")
+ .displayName("Record Writer")
+ .description("The Record Writer to use for serializing Records to an output FlowFile.")
+ .identifiesControllerService(RecordSetWriterFactory.class)
+ .dependsOn(RECORD_READER_FACTORY)
+ .required(true)
+ .build();
+
+ protected static final Relationship ORIGINAL = new Relationship.Builder()
+ .name("Original")
+ .description("Request FlowFiles transferred when receiving HTTP responses with a status code between 200 and 299.")
+ .build();
+
+ protected static final Relationship FAILURE = new Relationship.Builder()
+ .name("Failure")
+ .description("Request FlowFiles transferred when receiving socket communication errors.")
+ .build();
+
+ protected static final Relationship RESPONSE = new Relationship.Builder()
+ .name("Response")
+ .description("Response FlowFiles transferred when receiving HTTP responses with a status code between 200 and 299.")
+ .build();
+
+ protected static final Set<Relationship> RELATIONSHIPS = Collections.unmodifiableSet(new HashSet<>(Arrays.asList(ORIGINAL, RESPONSE, FAILURE)));
+ protected static final List<PropertyDescriptor> PROPERTIES = Collections.unmodifiableList(Arrays.asList(REPORT_URL, WORKDAY_USERNAME, WORKDAY_PASSWORD, WEB_CLIENT_SERVICE,
+ FIELDS_TO_HASH, HASHING_ALGORITHM, RECORD_READER_FACTORY, RECORD_WRITER_FACTORY));
+
+ private final AtomicReference<WebClientService> webClientAtomicReference = new AtomicReference<>();
+ private final AtomicReference<RecordReaderFactory> recordReaderFactoryAtomicReference = new AtomicReference<>();
+ private final AtomicReference<RecordSetWriterFactory> recordSetWriterFactoryAtomicReference = new AtomicReference<>();
Review Comment:
done
##########
nifi-nar-bundles/nifi-workday-bundle/nifi-workday-processors/src/main/java/org/apache/nifi/processors/workday/GetWorkdayReport.java:
##########
@@ -0,0 +1,432 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.nifi.processors.workday;
+
+import static org.apache.nifi.expression.ExpressionLanguageScope.FLOWFILE_ATTRIBUTES;
+import static org.apache.nifi.expression.ExpressionLanguageScope.NONE;
+import static org.apache.nifi.processor.util.StandardValidators.NON_BLANK_VALIDATOR;
+import static org.apache.nifi.processor.util.StandardValidators.URL_VALIDATOR;
+
+import java.io.BufferedInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.net.URI;
+import java.nio.charset.StandardCharsets;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Base64;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Optional;
+import java.util.Set;
+import java.util.UUID;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import org.apache.nifi.annotation.behavior.EventDriven;
+import org.apache.nifi.annotation.behavior.InputRequirement;
+import org.apache.nifi.annotation.behavior.InputRequirement.Requirement;
+import org.apache.nifi.annotation.behavior.SideEffectFree;
+import org.apache.nifi.annotation.behavior.SupportsBatching;
+import org.apache.nifi.annotation.behavior.WritesAttribute;
+import org.apache.nifi.annotation.behavior.WritesAttributes;
+import org.apache.nifi.annotation.documentation.CapabilityDescription;
+import org.apache.nifi.annotation.documentation.Tags;
+import org.apache.nifi.annotation.lifecycle.OnScheduled;
+import org.apache.nifi.components.PropertyDescriptor;
+import org.apache.nifi.components.ValidationContext;
+import org.apache.nifi.components.ValidationResult;
+import org.apache.nifi.flowfile.FlowFile;
+import org.apache.nifi.flowfile.attributes.CoreAttributes;
+import org.apache.nifi.logging.ComponentLog;
+import org.apache.nifi.processor.AbstractProcessor;
+import org.apache.nifi.processor.ProcessContext;
+import org.apache.nifi.processor.ProcessSession;
+import org.apache.nifi.processor.Relationship;
+import org.apache.nifi.processor.exception.ProcessException;
+import org.apache.nifi.processor.util.StandardValidators;
+import org.apache.nifi.record.path.RecordPath;
+import org.apache.nifi.record.path.RecordPathResult;
+import org.apache.nifi.schema.access.SchemaNotFoundException;
+import org.apache.nifi.security.util.crypto.HashAlgorithm;
+import org.apache.nifi.security.util.crypto.HashService;
+import org.apache.nifi.serialization.MalformedRecordException;
+import org.apache.nifi.serialization.RecordReader;
+import org.apache.nifi.serialization.RecordReaderFactory;
+import org.apache.nifi.serialization.RecordSetWriter;
+import org.apache.nifi.serialization.RecordSetWriterFactory;
+import org.apache.nifi.serialization.record.Record;
+import org.apache.nifi.serialization.record.RecordFieldType;
+import org.apache.nifi.serialization.record.RecordSchema;
+import org.apache.nifi.web.client.api.HttpResponseEntity;
+import org.apache.nifi.web.client.api.WebClientService;
+import org.apache.nifi.web.client.provider.api.WebClientServiceProvider;
+
+@Tags({"Workday", "report", "get"})
+@InputRequirement(Requirement.INPUT_ALLOWED)
+@CapabilityDescription("A processor which can interact with a configurable Workday Report. The processor can forward the content without modification, or you can transform it by"
+ + " providing the specific Record Reader and Record Writer services based on your needs. You can also hash, or remove fields using the input parameters and schemes in the Record writer. "
+ + "Supported Workday report formats are: csv, simplexml, json")
+@EventDriven
+@SideEffectFree
+@SupportsBatching
+@WritesAttributes({
+ @WritesAttribute(attribute = GetWorkdayReport.GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS, description = "The Java exception class raised when the processor fails"),
+ @WritesAttribute(attribute = GetWorkdayReport.GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE, description = "The Java exception message raised when the processor fails"),
+ @WritesAttribute(attribute = "mime.type", description = "Sets the mime.type attribute to the MIME Type specified by the Source / Record Writer"),
+ @WritesAttribute(attribute= GetWorkdayReport.RECORD_COUNT, description = "The number of records in an outgoing FlowFile. This is only populated on the 'success' relationship "
+ + "when Record Reader and Writer is set.")})
+public class GetWorkdayReport extends AbstractProcessor {
+
+ protected static final String STATUS_CODE = "getworkdayreport.status.code";
+ protected static final String REQUEST_URL = "getworkdayreport.request.url";
+ protected static final String REQUEST_DURATION = "getworkdayreport.request.duration";
+ protected static final String TRANSACTION_ID = "getworkdayreport.tx.id";
+ protected static final String GET_WORKDAY_REPORT_JAVA_EXCEPTION_CLASS = "getworkdayreport.java.exception.class";
+ protected static final String GET_WORKDAY_REPORT_JAVA_EXCEPTION_MESSAGE = "getworkdayreport.java.exception.message";
+ protected static final String RECORD_COUNT = "record.count";
+ protected static final String BASIC_PREFIX = "Basic ";
+ protected static final String COLUMNS_TO_HASH_SEPARATOR = ",";
+ protected static final String HEADER_AUTHORIZATION = "Authorization";
+ protected static final String HEADER_CONTENT_TYPE = "Content-Type";
+ protected static final String USERNAME_PASSWORD_SEPARATOR = ":";
+
+ protected static final PropertyDescriptor REPORT_URL = new PropertyDescriptor.Builder()
+ .name("Workday report URL")
+ .displayName("Workday report URL")
+ .description("HTTP remote URL of Workday report including a scheme of http or https, as well as a hostname or IP address with optional port and path elements.")
+ .required(true)
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .addValidator(URL_VALIDATOR)
+ .build();
+
+ protected static final PropertyDescriptor WORKDAY_USERNAME = new PropertyDescriptor.Builder()
+ .name("Workday Username")
+ .displayName("Workday Username")
+ .description("The username provided for authentication of Workday requests. Encoded using Base64 for HTTP Basic Authentication as described in RFC 7617.")
+ .required(true)
+ .addValidator(StandardValidators.createRegexMatchingValidator(Pattern.compile("^[\\x20-\\x39\\x3b-\\x7e\\x80-\\xff]+$")))
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .build();
+
+ protected static final PropertyDescriptor WORKDAY_PASSWORD = new PropertyDescriptor.Builder()
+ .name("Workday Password")
+ .displayName("Workday Password")
+ .description("The password provided for authentication of Workday requests. Encoded using Base64 for HTTP Basic Authentication as described in RFC 7617.")
+ .required(true)
+ .sensitive(true)
+ .addValidator(StandardValidators.createRegexMatchingValidator(Pattern.compile("^[\\x20-\\x7e\\x80-\\xff]+$")))
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .build();
+
+ protected static final PropertyDescriptor WEB_CLIENT_SERVICE = new PropertyDescriptor.Builder()
+ .name("Standard Web Client Service")
+ .description("Web client which is used to communicate with the Workday API.")
+ .required(true)
+ .identifiesControllerService(WebClientServiceProvider.class)
+ .build();
+
+ protected static final PropertyDescriptor FIELDS_TO_HASH = new PropertyDescriptor.Builder()
+ .name("Fields to hash")
+ .displayName("Fields to hash")
+ .description("Comma separated record paths to replace with hash.")
+ .required(false)
+ .expressionLanguageSupported(FLOWFILE_ATTRIBUTES)
+ .addValidator(NON_BLANK_VALIDATOR)
+ .build();
+
+ protected static final PropertyDescriptor HASHING_ALGORITHM = new PropertyDescriptor.Builder()
+ .name("Hashing algorithm")
+ .displayName("Hashing algorithm")
+ .description("Determines what hashing algorithm should be used to perform the hashing function.")
+ .required(true)
+ .allowableValues(HashService.buildHashAlgorithmAllowableValues())
+ .defaultValue(HashAlgorithm.SHA256.getName())
+ .addValidator(NON_BLANK_VALIDATOR)
+ .expressionLanguageSupported(NONE)
+ .dependsOn(FIELDS_TO_HASH)
+ .build();
+
+ protected static final PropertyDescriptor RECORD_READER_FACTORY = new PropertyDescriptor.Builder()
+ .name("record-reader")
+ .displayName("Record Reader")
+ .description("Specifies the Controller Service to use for parsing incoming data and determining the data's schema.")
+ .identifiesControllerService(RecordReaderFactory.class)
+ .required(false)
+ .build();
+
+ protected static final PropertyDescriptor RECORD_WRITER_FACTORY = new PropertyDescriptor.Builder()
+ .name("record-writer")
+ .displayName("Record Writer")
+ .description("The Record Writer to use for serializing Records to an output FlowFile.")
+ .identifiesControllerService(RecordSetWriterFactory.class)
+ .dependsOn(RECORD_READER_FACTORY)
+ .required(true)
+ .build();
+
+ protected static final Relationship ORIGINAL = new Relationship.Builder()
+ .name("Original")
+ .description("Request FlowFiles transferred when receiving HTTP responses with a status code between 200 and 299.")
+ .build();
+
+ protected static final Relationship FAILURE = new Relationship.Builder()
+ .name("Failure")
+ .description("Request FlowFiles transferred when receiving socket communication errors.")
+ .build();
+
+ protected static final Relationship RESPONSE = new Relationship.Builder()
+ .name("Response")
+ .description("Response FlowFiles transferred when receiving HTTP responses with a status code between 200 and 299.")
+ .build();
+
+ protected static final Set<Relationship> RELATIONSHIPS = Collections.unmodifiableSet(new HashSet<>(Arrays.asList(ORIGINAL, RESPONSE, FAILURE)));
+ protected static final List<PropertyDescriptor> PROPERTIES = Collections.unmodifiableList(Arrays.asList(REPORT_URL, WORKDAY_USERNAME, WORKDAY_PASSWORD, WEB_CLIENT_SERVICE,
+ FIELDS_TO_HASH, HASHING_ALGORITHM, RECORD_READER_FACTORY, RECORD_WRITER_FACTORY));
+
+ private final AtomicReference<WebClientService> webClientAtomicReference = new AtomicReference<>();
+ private final AtomicReference<RecordReaderFactory> recordReaderFactoryAtomicReference = new AtomicReference<>();
+ private final AtomicReference<RecordSetWriterFactory> recordSetWriterFactoryAtomicReference = new AtomicReference<>();
+
+ @Override
+ protected List<PropertyDescriptor> getSupportedPropertyDescriptors() {
+ return PROPERTIES;
+ }
+
+ @Override
+ public Set<Relationship> getRelationships() {
+ return RELATIONSHIPS;
+ }
+
+ @OnScheduled
+ public void setUpClient(final ProcessContext context) {
+ WebClientServiceProvider standardWebClientServiceProvider = context.getProperty(WEB_CLIENT_SERVICE).asControllerService(WebClientServiceProvider.class);
+ RecordReaderFactory recordReaderFactory = context.getProperty(RECORD_READER_FACTORY).asControllerService(RecordReaderFactory.class);
+ RecordSetWriterFactory recordSetWriterFactory = context.getProperty(RECORD_WRITER_FACTORY).asControllerService(RecordSetWriterFactory.class);
+ WebClientService webClientService = standardWebClientServiceProvider.getWebClientService();
+ webClientAtomicReference.set(webClientService);
+ recordReaderFactoryAtomicReference.set(recordReaderFactory);
+ recordSetWriterFactoryAtomicReference.set(recordSetWriterFactory);
+ }
+
+ @Override
+ public void onTrigger(ProcessContext context, ProcessSession session) throws ProcessException {
+ FlowFile flowfile = session.get();
+
+ if (skipExecution(context, flowfile)) {
+ return;
+ }
+
+ ComponentLog logger = getLogger();
+ FlowFile responseFlowFile = null;
+
+ try {
+ WebClientService webClientService = webClientAtomicReference.get();
+ URI uri = new URI(context.getProperty(REPORT_URL).evaluateAttributeExpressions(flowfile).getValue().trim());
+ long startNanos = System.nanoTime();
+ String authorization = createAuthorizationHeader(context, flowfile);
+
+ try(HttpResponseEntity httpResponseEntity = webClientService.get().uri(uri).header(HEADER_AUTHORIZATION, authorization).retrieve()) {
Review Comment:
done
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@nifi.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org