You are viewing a plain text version of this content. The canonical link for it is here.
Posted to dev@nifi.apache.org by jdye64 <gi...@git.apache.org> on 2015/11/13 21:04:14 UTC

[GitHub] nifi pull request: NIFI-1156: HTML Parsing Processors Bundle

GitHub user jdye64 opened a pull request:

    https://github.com/apache/nifi/pull/124

    NIFI-1156: HTML Parsing Processors Bundle

    NIFI-1156: HTML Parsing Processors Bundle. GetHTMLElement, ModifyHTMLElement, and PutHTMLElement

You can merge this pull request into a Git repository by running:

    $ git pull https://github.com/jdye64/nifi NIFI-1156

Alternatively you can review and apply these changes as the patch at:

    https://github.com/apache/nifi/pull/124.patch

To close this pull request, make a commit to your master/trunk branch
with (at least) the following in the commit message:

    This closes #124
    
----
commit c82fc18f8e306c5a31345856e529cfd9fe4c81ef
Author: Jeremy Dyer <jd...@gmail.com>
Date:   2015-11-13T20:01:10Z

    HTML Parsing Processors Bundle
    
    NIFI-1156 HTML Parsing Processors Bundle

commit f8d26d205901d8cd782c02a5bc57e9ec66ee2f33
Author: Jeremy Dyer <jd...@gmail.com>
Date:   2015-11-13T20:02:23Z

    Merge branch 'master' into NIFI-1156

----


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] nifi pull request: NIFI-1156: HTML Parsing Processors Bundle

Posted by markap14 <gi...@git.apache.org>.
Github user markap14 commented on a diff in the pull request:

    https://github.com/apache/nifi/pull/124#discussion_r49480004
  
    --- Diff: nifi-nar-bundles/nifi-html-bundle/nifi-html-processors/src/main/java/org/apache/nifi/GetHTMLElement.java ---
    @@ -0,0 +1,243 @@
    +/*
    + * Licensed to the Apache Software Foundation (ASF) under one or more
    + * contributor license agreements.  See the NOTICE file distributed with
    + * this work for additional information regarding copyright ownership.
    + * The ASF licenses this file to You under the Apache License, Version 2.0
    + * (the "License"); you may not use this file except in compliance with
    + * the License.  You may obtain a copy of the License at
    + *
    + *     http://www.apache.org/licenses/LICENSE-2.0
    + *
    + * Unless required by applicable law or agreed to in writing, software
    + * distributed under the License is distributed on an "AS IS" BASIS,
    + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    + * See the License for the specific language governing permissions and
    + * limitations under the License.
    + */
    +package org.apache.nifi;
    +
    +import org.apache.commons.lang3.StringUtils;
    +import org.apache.nifi.components.PropertyDescriptor;
    +import org.apache.nifi.flowfile.FlowFile;
    +import org.apache.nifi.processor.ProcessContext;
    +import org.apache.nifi.processor.ProcessSession;
    +import org.apache.nifi.processor.Relationship;
    +import org.apache.nifi.processor.ProcessorInitializationContext;
    +import org.apache.nifi.annotation.behavior.WritesAttribute;
    +import org.apache.nifi.annotation.behavior.WritesAttributes;
    +import org.apache.nifi.annotation.documentation.CapabilityDescription;
    +import org.apache.nifi.annotation.documentation.SeeAlso;
    +import org.apache.nifi.annotation.documentation.Tags;
    +import org.apache.nifi.processor.exception.ProcessException;
    +import org.apache.nifi.processor.io.StreamCallback;
    +import org.apache.nifi.processor.util.StandardValidators;
    +import org.jsoup.nodes.Document;
    +import org.jsoup.nodes.Element;
    +import org.jsoup.select.Elements;
    +
    +import java.io.IOException;
    +import java.io.InputStream;
    +import java.io.OutputStream;
    +import java.util.ArrayList;
    +import java.util.List;
    +import java.util.Set;
    +import java.util.HashSet;
    +import java.util.Collections;
    +
    +@Tags({"get", "html", "dom", "css", "element"})
    +@CapabilityDescription("Parses HTML input using CSS selector syntax and creates a new flowfile containing the extracted" +
    +        " element content for each matching CSS selector.")
    +@SeeAlso({ModifyHTMLElement.class, PutHTMLElement.class})
    +@WritesAttributes({@WritesAttribute(attribute="HTMLElement", description="Flowfile attribute where the element result" +
    +        " parsed from the HTML using the CSS selector syntax are placed if the destination is a flowfile attribute.")})
    +public class GetHTMLElement
    +        extends AbstractHTMLProcessor {
    +
    +    public static final String HTML_ELEMENT_ATTRIBUTE_NAME = "HTMLElement";
    +    public static final String DESTINATION_ATTRIBUTE = "flowfile-attribute";
    +    public static final String DESTINATION_CONTENT = "flowfile-content";
    +
    +    public static final PropertyDescriptor PREPEND_ELEMENT_VALUE = new PropertyDescriptor
    +            .Builder().name("Prepend Element value")
    +            .description("Prepends the specified value to the resulting Element")
    +            .required(false)
    +            .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
    +            .expressionLanguageSupported(true)
    +            .build();
    +
    +    public static final PropertyDescriptor APPEND_ELEMENT_VALUE = new PropertyDescriptor
    +            .Builder().name("Append Element value")
    +            .description("Appends the specified value to the resulting Element")
    +            .required(false)
    +            .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
    +            .expressionLanguageSupported(true)
    +            .build();
    +
    +    public static final PropertyDescriptor ATTRIBUTE_KEY = new PropertyDescriptor
    +            .Builder().name("Attribute Name")
    +            .description(("When getting the value of an element attribute this value is used as the key to determine" +
    +                    " which attribute on the selected element should be retrieved."))
    +            .required(false)
    +            .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
    +            .expressionLanguageSupported(true)
    +            .build();
    +
    +
    +    public static final PropertyDescriptor OUTPUT_TYPE = new PropertyDescriptor.Builder()
    +            .name("Output Type")
    +            .description("Controls the type of value that is retrieved from the element. " +
    +                    ELEMENT_HTML + "," + ELEMENT_TEXT + ", " + ELEMENT_ATTRIBUTE + " or " + ELEMENT_DATA)
    +            .required(true)
    +            .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
    +            .allowableValues(ELEMENT_HTML, ELEMENT_TEXT, ELEMENT_ATTRIBUTE, ELEMENT_DATA)
    +            .defaultValue(ELEMENT_HTML)
    +            .build();
    +
    +    public static final PropertyDescriptor DESTINATION = new PropertyDescriptor.Builder()
    +            .name("Destination")
    +            .description("Control if element extracted is written as a flowfile attribute or " +
    +                    "as flowfile content.")
    +            .required(true)
    +            .allowableValues(DESTINATION_ATTRIBUTE, DESTINATION_CONTENT)
    +            .defaultValue(DESTINATION_ATTRIBUTE)
    +            .build();
    +
    +    private List<PropertyDescriptor> descriptors;
    +
    +    private Set<Relationship> relationships;
    +
    +    @Override
    +    protected void init(final ProcessorInitializationContext context) {
    +        final List<PropertyDescriptor> descriptors = new ArrayList<>();
    +        descriptors.add(URL);
    +        descriptors.add(CSS_SELECTOR);
    +        descriptors.add(HTML_CHARSET);
    +        descriptors.add(OUTPUT_TYPE);
    +        descriptors.add(DESTINATION);
    +        descriptors.add(PREPEND_ELEMENT_VALUE);
    +        descriptors.add(APPEND_ELEMENT_VALUE);
    +        descriptors.add(ATTRIBUTE_KEY);
    +        this.descriptors = Collections.unmodifiableList(descriptors);
    +
    +        final Set<Relationship> relationships = new HashSet<>();
    +        relationships.add(REL_ORIGINAL);
    +        relationships.add(REL_SUCCESS);
    +        relationships.add(REL_FAILURE);
    +        relationships.add(REL_NOT_FOUND);
    +        this.relationships = Collections.unmodifiableSet(relationships);
    +    }
    +
    +    @Override
    +    public Set<Relationship> getRelationships() {
    +        return this.relationships;
    +    }
    +
    +    @Override
    +    public final List<PropertyDescriptor> getSupportedPropertyDescriptors() {
    +        return descriptors;
    +    }
    +
    +    @Override
    +    public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
    +        final FlowFile flowFile = session.get();
    +        if ( flowFile == null ) {
    +            return;
    +        }
    +
    +        try {
    +
    +            final Document doc = parseHTMLDocumentFromFlowfile(flowFile, context, session);
    +            final Elements eles = doc.select(context.getProperty(CSS_SELECTOR)
    +                    .evaluateAttributeExpressions().getValue());
    +            final String prependValue = context.getProperty(PREPEND_ELEMENT_VALUE)
    +                    .evaluateAttributeExpressions(flowFile).getValue();
    +            final String appendValue = context.getProperty(APPEND_ELEMENT_VALUE)
    +                    .evaluateAttributeExpressions(flowFile).getValue();
    +
    +            if (eles == null || eles.size() == 0) {
    +                //No element found
    +                session.transfer(flowFile, REL_NOT_FOUND);
    +            } else {
    +                for (final Element ele : eles) {
    +                    final FlowFile ff = session.create();
    +
    +                    switch (context.getProperty(DESTINATION).getValue()) {
    +                        case DESTINATION_ATTRIBUTE:
    +                            final FlowFile atFlowfile = session.putAttribute(ff, HTML_ELEMENT_ATTRIBUTE_NAME,
    +                                    extractElementValue(
    +                                            prependValue,
    +                                            context.getProperty(OUTPUT_TYPE).getValue(),
    +                                            appendValue,
    +                                            ele,
    +                                            context.getProperty(ATTRIBUTE_KEY).evaluateAttributeExpressions()
    +                                                    .getValue()));
    +                            session.getProvenanceReporter().create(atFlowfile);
    +                            session.transfer(atFlowfile, REL_SUCCESS);
    +                            break;
    +                        case DESTINATION_CONTENT:
    +                            final FlowFile conFlowfile = session.write(ff, new StreamCallback() {
    +                                @Override
    +                                public void process(InputStream inputStream, OutputStream outputStream) throws IOException {
    +                                    try {
    +                                        outputStream.write(extractElementValue(
    +                                                prependValue,
    +                                                context.getProperty(OUTPUT_TYPE).getValue(),
    +                                                appendValue,
    +                                                ele,
    +                                                context.getProperty(ATTRIBUTE_KEY).evaluateAttributeExpressions()
    +                                                        .getValue()).getBytes());
    +                                    } catch (Exception ex) {
    +                                        session.transfer(ff, REL_FAILURE);
    --- End diff --
    
    Should probably log what happened here.


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] nifi pull request: NIFI-1156: HTML Parsing Processors Bundle

Posted by markap14 <gi...@git.apache.org>.
Github user markap14 commented on a diff in the pull request:

    https://github.com/apache/nifi/pull/124#discussion_r49480382
  
    --- Diff: nifi-nar-bundles/nifi-html-bundle/nifi-html-processors/src/main/java/org/apache/nifi/GetHTMLElement.java ---
    @@ -0,0 +1,243 @@
    +/*
    + * Licensed to the Apache Software Foundation (ASF) under one or more
    + * contributor license agreements.  See the NOTICE file distributed with
    + * this work for additional information regarding copyright ownership.
    + * The ASF licenses this file to You under the Apache License, Version 2.0
    + * (the "License"); you may not use this file except in compliance with
    + * the License.  You may obtain a copy of the License at
    + *
    + *     http://www.apache.org/licenses/LICENSE-2.0
    + *
    + * Unless required by applicable law or agreed to in writing, software
    + * distributed under the License is distributed on an "AS IS" BASIS,
    + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    + * See the License for the specific language governing permissions and
    + * limitations under the License.
    + */
    +package org.apache.nifi;
    +
    +import org.apache.commons.lang3.StringUtils;
    +import org.apache.nifi.components.PropertyDescriptor;
    +import org.apache.nifi.flowfile.FlowFile;
    +import org.apache.nifi.processor.ProcessContext;
    +import org.apache.nifi.processor.ProcessSession;
    +import org.apache.nifi.processor.Relationship;
    +import org.apache.nifi.processor.ProcessorInitializationContext;
    +import org.apache.nifi.annotation.behavior.WritesAttribute;
    +import org.apache.nifi.annotation.behavior.WritesAttributes;
    +import org.apache.nifi.annotation.documentation.CapabilityDescription;
    +import org.apache.nifi.annotation.documentation.SeeAlso;
    +import org.apache.nifi.annotation.documentation.Tags;
    +import org.apache.nifi.processor.exception.ProcessException;
    +import org.apache.nifi.processor.io.StreamCallback;
    +import org.apache.nifi.processor.util.StandardValidators;
    +import org.jsoup.nodes.Document;
    +import org.jsoup.nodes.Element;
    +import org.jsoup.select.Elements;
    +
    +import java.io.IOException;
    +import java.io.InputStream;
    +import java.io.OutputStream;
    +import java.util.ArrayList;
    +import java.util.List;
    +import java.util.Set;
    +import java.util.HashSet;
    +import java.util.Collections;
    +
    +@Tags({"get", "html", "dom", "css", "element"})
    +@CapabilityDescription("Parses HTML input using CSS selector syntax and creates a new flowfile containing the extracted" +
    +        " element content for each matching CSS selector.")
    +@SeeAlso({ModifyHTMLElement.class, PutHTMLElement.class})
    +@WritesAttributes({@WritesAttribute(attribute="HTMLElement", description="Flowfile attribute where the element result" +
    +        " parsed from the HTML using the CSS selector syntax are placed if the destination is a flowfile attribute.")})
    +public class GetHTMLElement
    +        extends AbstractHTMLProcessor {
    +
    +    public static final String HTML_ELEMENT_ATTRIBUTE_NAME = "HTMLElement";
    +    public static final String DESTINATION_ATTRIBUTE = "flowfile-attribute";
    +    public static final String DESTINATION_CONTENT = "flowfile-content";
    +
    +    public static final PropertyDescriptor PREPEND_ELEMENT_VALUE = new PropertyDescriptor
    +            .Builder().name("Prepend Element value")
    --- End diff --
    
    Recommend we keep with Title Case: Prepend Element Value


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] nifi pull request: NIFI-1156: HTML Parsing Processors Bundle

Posted by markap14 <gi...@git.apache.org>.
Github user markap14 commented on a diff in the pull request:

    https://github.com/apache/nifi/pull/124#discussion_r49482104
  
    --- Diff: nifi-nar-bundles/nifi-html-bundle/nifi-html-processors/src/main/java/org/apache/nifi/GetHTMLElement.java ---
    @@ -0,0 +1,243 @@
    +/*
    + * Licensed to the Apache Software Foundation (ASF) under one or more
    + * contributor license agreements.  See the NOTICE file distributed with
    + * this work for additional information regarding copyright ownership.
    + * The ASF licenses this file to You under the Apache License, Version 2.0
    + * (the "License"); you may not use this file except in compliance with
    + * the License.  You may obtain a copy of the License at
    + *
    + *     http://www.apache.org/licenses/LICENSE-2.0
    + *
    + * Unless required by applicable law or agreed to in writing, software
    + * distributed under the License is distributed on an "AS IS" BASIS,
    + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    + * See the License for the specific language governing permissions and
    + * limitations under the License.
    + */
    +package org.apache.nifi;
    +
    +import org.apache.commons.lang3.StringUtils;
    +import org.apache.nifi.components.PropertyDescriptor;
    +import org.apache.nifi.flowfile.FlowFile;
    +import org.apache.nifi.processor.ProcessContext;
    +import org.apache.nifi.processor.ProcessSession;
    +import org.apache.nifi.processor.Relationship;
    +import org.apache.nifi.processor.ProcessorInitializationContext;
    +import org.apache.nifi.annotation.behavior.WritesAttribute;
    +import org.apache.nifi.annotation.behavior.WritesAttributes;
    +import org.apache.nifi.annotation.documentation.CapabilityDescription;
    +import org.apache.nifi.annotation.documentation.SeeAlso;
    +import org.apache.nifi.annotation.documentation.Tags;
    +import org.apache.nifi.processor.exception.ProcessException;
    +import org.apache.nifi.processor.io.StreamCallback;
    +import org.apache.nifi.processor.util.StandardValidators;
    +import org.jsoup.nodes.Document;
    +import org.jsoup.nodes.Element;
    +import org.jsoup.select.Elements;
    +
    +import java.io.IOException;
    +import java.io.InputStream;
    +import java.io.OutputStream;
    +import java.util.ArrayList;
    +import java.util.List;
    +import java.util.Set;
    +import java.util.HashSet;
    +import java.util.Collections;
    +
    +@Tags({"get", "html", "dom", "css", "element"})
    +@CapabilityDescription("Parses HTML input using CSS selector syntax and creates a new flowfile containing the extracted" +
    +        " element content for each matching CSS selector.")
    +@SeeAlso({ModifyHTMLElement.class, PutHTMLElement.class})
    +@WritesAttributes({@WritesAttribute(attribute="HTMLElement", description="Flowfile attribute where the element result" +
    --- End diff --
    
    Should have an @InputRequirement(Requirement.INPUT_FORBIDDEN)


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] nifi pull request: NIFI-1156: HTML Parsing Processors Bundle

Posted by markap14 <gi...@git.apache.org>.
Github user markap14 commented on a diff in the pull request:

    https://github.com/apache/nifi/pull/124#discussion_r49481210
  
    --- Diff: nifi-nar-bundles/nifi-html-bundle/nifi-html-processors/src/test/java/org/apache/nifi/AbstractHTMLTest.java ---
    @@ -0,0 +1,74 @@
    +/*
    + * Licensed to the Apache Software Foundation (ASF) under one or more
    + * contributor license agreements.  See the NOTICE file distributed with
    + * this work for additional information regarding copyright ownership.
    + * The ASF licenses this file to You under the Apache License, Version 2.0
    + * (the "License"); you may not use this file except in compliance with
    + * the License.  You may obtain a copy of the License at
    + *
    + *     http://www.apache.org/licenses/LICENSE-2.0
    + *
    + * Unless required by applicable law or agreed to in writing, software
    + * distributed under the License is distributed on an "AS IS" BASIS,
    + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    + * See the License for the specific language governing permissions and
    + * limitations under the License.
    + */
    +package org.apache.nifi;
    +
    +import org.apache.nifi.flowfile.FlowFile;
    +import org.apache.nifi.processor.ProcessSession;
    +import org.apache.nifi.processor.io.StreamCallback;
    +
    +import java.io.IOException;
    +import java.io.InputStream;
    +import java.io.OutputStream;
    +
    +public class AbstractHTMLTest {
    +
    +    protected final String ATL_WEATHER_TEXT = "Atlanta Weather";
    +    protected final String GDR_WEATHER_TEXT = "<i>Grand Rapids Weather</i>";
    +    protected final String ATL_WEATHER_LINK = "http://w1.weather.gov/obhistory/KPDK.html";
    +    protected final String GR_WEATHER_LINK = "http://w1.weather.gov/obhistory/KGRR.html";
    +    protected final String AUTHOR_NAME = "Jeremy Dyer";
    +    protected final String ATL_ID = "ATL";
    +    protected final String GDR_ID = "GDR";
    +
    +    protected final String HTML = "<!doctype html>\n" +
    --- End diff --
    
    We should probably move this into a file under src/test/resources. Keeps the unit tests cleaner. Then we can easily enqueue it via runner.enqueue(new File("src/test/resources/XYZ.HTML").toPath());


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] nifi pull request: NIFI-1156: HTML Parsing Processors Bundle

Posted by markap14 <gi...@git.apache.org>.
Github user markap14 commented on a diff in the pull request:

    https://github.com/apache/nifi/pull/124#discussion_r49481748
  
    --- Diff: nifi-nar-bundles/nifi-html-bundle/nifi-html-processors/src/test/java/org/apache/nifi/TestGetHTMLElement.java ---
    @@ -0,0 +1,319 @@
    +/*
    + * Licensed to the Apache Software Foundation (ASF) under one or more
    + * contributor license agreements.  See the NOTICE file distributed with
    + * this work for additional information regarding copyright ownership.
    + * The ASF licenses this file to You under the Apache License, Version 2.0
    + * (the "License"); you may not use this file except in compliance with
    + * the License.  You may obtain a copy of the License at
    + *
    + *     http://www.apache.org/licenses/LICENSE-2.0
    + *
    + * Unless required by applicable law or agreed to in writing, software
    + * distributed under the License is distributed on an "AS IS" BASIS,
    + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    + * See the License for the specific language governing permissions and
    + * limitations under the License.
    + */
    +package org.apache.nifi;
    +
    +import org.apache.commons.lang3.StringUtils;
    +import org.apache.nifi.flowfile.FlowFile;
    +import org.apache.nifi.processor.ProcessSession;
    +import org.apache.nifi.util.MockFlowFile;
    +import org.apache.nifi.util.TestRunner;
    +import org.apache.nifi.util.TestRunners;
    +import org.junit.Before;
    +import org.junit.Test;
    +
    +import java.lang.Exception;
    +import java.util.List;
    +
    +import static org.junit.Assert.assertTrue;
    +
    +public class TestGetHTMLElement extends AbstractHTMLTest {
    +
    +    private TestRunner testRunner;
    +
    +    @Before
    +    public void init() {
    +        testRunner = TestRunners.newTestRunner(GetHTMLElement.class);
    +        testRunner.setProperty(GetHTMLElement.URL, "http://localhost");
    +        testRunner.setProperty(GetHTMLElement.OUTPUT_TYPE, GetHTMLElement.ELEMENT_HTML);
    +        testRunner.setProperty(GetHTMLElement.DESTINATION, GetHTMLElement.DESTINATION_CONTENT);
    +        testRunner.setProperty(GetHTMLElement.HTML_CHARSET, "UTF-8");
    +    }
    +
    +    @Test
    +    public void testNoElementFound() throws Exception {
    +        testRunner.setProperty(GetHTMLElement.CSS_SELECTOR, "b");   //Bold element is not present in sample HTML
    +//        testRunner.setProperty(GetHTMLElement.APPEND_ELEMENT_VALUE, "");
    +
    +        ProcessSession session = testRunner.getProcessSessionFactory().createSession();
    +        FlowFile ff = writeContentToNewFlowFile(HTML.getBytes(), session);
    +
    +        testRunner.enqueue(ff);
    +        testRunner.run();
    +
    +        testRunner.assertTransferCount(GetHTMLElement.REL_SUCCESS, 0);
    +        testRunner.assertTransferCount(GetHTMLElement.REL_FAILURE, 0);
    +        testRunner.assertTransferCount(GetHTMLElement.REL_NOT_FOUND, 1);
    +    }
    +
    +    @Test
    +    public void testInvalidSelector() throws Exception {
    +        testRunner.setProperty(GetHTMLElement.CSS_SELECTOR, "InvalidCSSSelectorSyntax");
    +
    +        ProcessSession session = testRunner.getProcessSessionFactory().createSession();
    +        FlowFile ff = writeContentToNewFlowFile(HTML.getBytes(), session);
    +
    +        testRunner.enqueue(ff);
    +        testRunner.run();
    +
    +        testRunner.assertTransferCount(GetHTMLElement.REL_SUCCESS, 0);
    +        testRunner.assertTransferCount(GetHTMLElement.REL_FAILURE, 0);
    +        testRunner.assertTransferCount(GetHTMLElement.REL_NOT_FOUND, 1);
    +    }
    +
    +    @Test
    +    public void testSingleElementFound() throws Exception {
    +        testRunner.setProperty(GetHTMLElement.CSS_SELECTOR, "head");
    +
    +        ProcessSession session = testRunner.getProcessSessionFactory().createSession();
    +        FlowFile ff = writeContentToNewFlowFile(HTML.getBytes(), session);
    +
    +        testRunner.enqueue(ff);
    +        testRunner.run();
    +
    +        testRunner.assertTransferCount(GetHTMLElement.REL_SUCCESS, 1);
    +        testRunner.assertTransferCount(GetHTMLElement.REL_FAILURE, 0);
    +        testRunner.assertTransferCount(GetHTMLElement.REL_ORIGINAL, 1);
    +        testRunner.assertTransferCount(GetHTMLElement.REL_NOT_FOUND, 0);
    +    }
    +
    +    @Test
    +    public void testMultipleElementFound() throws Exception {
    +        testRunner.setProperty(GetHTMLElement.CSS_SELECTOR, "a");
    +
    +        ProcessSession session = testRunner.getProcessSessionFactory().createSession();
    +        FlowFile ff = writeContentToNewFlowFile(HTML.getBytes(), session);
    +
    +        testRunner.enqueue(ff);
    +        testRunner.run();
    +
    +        testRunner.assertTransferCount(GetHTMLElement.REL_SUCCESS, 3);
    +        testRunner.assertTransferCount(GetHTMLElement.REL_FAILURE, 0);
    +        testRunner.assertTransferCount(GetHTMLElement.REL_ORIGINAL, 1);
    +        testRunner.assertTransferCount(GetHTMLElement.REL_NOT_FOUND, 0);
    +    }
    +
    +    @Test
    +    public void testElementFoundWriteToAttribute() throws Exception {
    +        testRunner.setProperty(GetHTMLElement.CSS_SELECTOR, "#" + ATL_ID);
    +        testRunner.setProperty(GetHTMLElement.DESTINATION, GetHTMLElement.DESTINATION_ATTRIBUTE);
    +        testRunner.setProperty(GetHTMLElement.OUTPUT_TYPE, GetHTMLElement.ELEMENT_ATTRIBUTE);
    +        testRunner.setProperty(GetHTMLElement.ATTRIBUTE_KEY, "href");
    +
    +        ProcessSession session = testRunner.getProcessSessionFactory().createSession();
    +        FlowFile ff = writeContentToNewFlowFile(HTML.getBytes(), session);
    +
    +        testRunner.enqueue(ff);
    +        testRunner.run();
    +
    +        testRunner.assertTransferCount(GetHTMLElement.REL_SUCCESS, 1);
    +        testRunner.assertTransferCount(GetHTMLElement.REL_FAILURE, 0);
    +        testRunner.assertTransferCount(GetHTMLElement.REL_ORIGINAL, 1);
    +        testRunner.assertTransferCount(GetHTMLElement.REL_NOT_FOUND, 0);
    +
    +        List<MockFlowFile> ffs = testRunner.getFlowFilesForRelationship(GetHTMLElement.REL_SUCCESS);
    +        assertTrue(ffs.size() == 1);
    +        MockFlowFile fff = ffs.get(0);
    +        String atValue = fff.getAttribute(GetHTMLElement.HTML_ELEMENT_ATTRIBUTE_NAME);
    +        assertTrue(StringUtils.equals(ATL_WEATHER_LINK, atValue));
    +    }
    +
    +    @Test
    +    public void testElementFoundWriteToContent() throws Exception {
    +        testRunner.setProperty(GetHTMLElement.CSS_SELECTOR, "#" + ATL_ID);
    +        testRunner.setProperty(GetHTMLElement.DESTINATION, GetHTMLElement.DESTINATION_CONTENT);
    +        testRunner.setProperty(GetHTMLElement.OUTPUT_TYPE, GetHTMLElement.ELEMENT_ATTRIBUTE);
    +        testRunner.setProperty(GetHTMLElement.ATTRIBUTE_KEY, "href");
    +
    +        ProcessSession session = testRunner.getProcessSessionFactory().createSession();
    +        FlowFile ff = writeContentToNewFlowFile(HTML.getBytes(), session);
    +
    +        testRunner.enqueue(ff);
    +        testRunner.run();
    +
    +        testRunner.assertTransferCount(GetHTMLElement.REL_SUCCESS, 1);
    +        testRunner.assertTransferCount(GetHTMLElement.REL_FAILURE, 0);
    +        testRunner.assertTransferCount(GetHTMLElement.REL_ORIGINAL, 1);
    +        testRunner.assertTransferCount(GetHTMLElement.REL_NOT_FOUND, 0);
    +
    +        List<MockFlowFile> ffs = testRunner.getFlowFilesForRelationship(GetHTMLElement.REL_SUCCESS);
    +        assertTrue(ffs.size() == 1);
    +        String data = new String(testRunner.getContentAsByteArray(ffs.get(0)));
    --- End diff --
    
    Recommend we go with:
    ffs.get(0).assertContentEquals(ATL_WEATHER_LINK);
    
    This gives us a nice output if the unit test were to fail, and would indicate what we expected the value to be vs. what it actually was.


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] nifi pull request: NIFI-1156: HTML Parsing Processors Bundle

Posted by olegz <gi...@git.apache.org>.
Github user olegz commented on a diff in the pull request:

    https://github.com/apache/nifi/pull/124#discussion_r45008293
  
    --- Diff: nifi-nar-bundles/nifi-html-bundle/nifi-html-processors/pom.xml ---
    @@ -0,0 +1,59 @@
    +<?xml version="1.0" encoding="UTF-8"?>
    +<!--
    +  Licensed to the Apache Software Foundation (ASF) under one or more
    +  contributor license agreements. See the NOTICE file distributed with
    +  this work for additional information regarding copyright ownership.
    +  The ASF licenses this file to You under the Apache License, Version 2.0
    +  (the "License"); you may not use this file except in compliance with
    +  the License. You may obtain a copy of the License at
    +  http://www.apache.org/licenses/LICENSE-2.0
    +  Unless required by applicable law or agreed to in writing, software
    +  distributed under the License is distributed on an "AS IS" BASIS,
    +  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    +  See the License for the specific language governing permissions and
    +  limitations under the License.
    +-->
    +<project xmlns="http://maven.apache.org/POM/4.0.0" xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance" xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd">
    +    <modelVersion>4.0.0</modelVersion>
    +
    +    <parent>
    +        <groupId>org.apache.nifi</groupId>
    +        <artifactId>nifi-html-bundle</artifactId>
    +        <version>0.4.0-SNAPSHOT</version>
    +    </parent>
    +
    +    <artifactId>nifi-html-processors</artifactId>
    +    <description>Support for parsing HTML documents</description>
    +
    +    <dependencies>
    +        <dependency>
    +            <groupId>org.jsoup</groupId>
    +            <artifactId>jsoup</artifactId>
    +            <version>1.8.3</version>
    +        </dependency>
    +        <dependency>
    +            <groupId>org.apache.nifi</groupId>
    +            <artifactId>nifi-api</artifactId>
    +        </dependency>
    +        <dependency>
    +            <groupId>org.apache.nifi</groupId>
    +            <artifactId>nifi-processor-utils</artifactId>
    +        </dependency>
    +        <dependency>
    +            <groupId>org.apache.nifi</groupId>
    +            <artifactId>nifi-mock</artifactId>
    +            <scope>test</scope>
    +        </dependency>
    +        <dependency>
    +            <groupId>org.slf4j</groupId>
    +            <artifactId>slf4j-simple</artifactId>
    +            <scope>test</scope>
    +        </dependency>
    +        <dependency>
    +            <groupId>junit</groupId>
    +            <artifactId>junit</artifactId>
    +            <version>4.11</version>
    --- End diff --
    
    Does the main POM declares a version of JUnit 4.12? Can we use that one?


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] nifi pull request: NIFI-1156: HTML Parsing Processors Bundle

Posted by markap14 <gi...@git.apache.org>.
Github user markap14 commented on a diff in the pull request:

    https://github.com/apache/nifi/pull/124#discussion_r49480077
  
    --- Diff: nifi-nar-bundles/nifi-html-bundle/nifi-html-processors/src/main/java/org/apache/nifi/GetHTMLElement.java ---
    @@ -0,0 +1,243 @@
    +/*
    + * Licensed to the Apache Software Foundation (ASF) under one or more
    + * contributor license agreements.  See the NOTICE file distributed with
    + * this work for additional information regarding copyright ownership.
    + * The ASF licenses this file to You under the Apache License, Version 2.0
    + * (the "License"); you may not use this file except in compliance with
    + * the License.  You may obtain a copy of the License at
    + *
    + *     http://www.apache.org/licenses/LICENSE-2.0
    + *
    + * Unless required by applicable law or agreed to in writing, software
    + * distributed under the License is distributed on an "AS IS" BASIS,
    + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    + * See the License for the specific language governing permissions and
    + * limitations under the License.
    + */
    +package org.apache.nifi;
    +
    +import org.apache.commons.lang3.StringUtils;
    +import org.apache.nifi.components.PropertyDescriptor;
    +import org.apache.nifi.flowfile.FlowFile;
    +import org.apache.nifi.processor.ProcessContext;
    +import org.apache.nifi.processor.ProcessSession;
    +import org.apache.nifi.processor.Relationship;
    +import org.apache.nifi.processor.ProcessorInitializationContext;
    +import org.apache.nifi.annotation.behavior.WritesAttribute;
    +import org.apache.nifi.annotation.behavior.WritesAttributes;
    +import org.apache.nifi.annotation.documentation.CapabilityDescription;
    +import org.apache.nifi.annotation.documentation.SeeAlso;
    +import org.apache.nifi.annotation.documentation.Tags;
    +import org.apache.nifi.processor.exception.ProcessException;
    +import org.apache.nifi.processor.io.StreamCallback;
    +import org.apache.nifi.processor.util.StandardValidators;
    +import org.jsoup.nodes.Document;
    +import org.jsoup.nodes.Element;
    +import org.jsoup.select.Elements;
    +
    +import java.io.IOException;
    +import java.io.InputStream;
    +import java.io.OutputStream;
    +import java.util.ArrayList;
    +import java.util.List;
    +import java.util.Set;
    +import java.util.HashSet;
    +import java.util.Collections;
    +
    +@Tags({"get", "html", "dom", "css", "element"})
    +@CapabilityDescription("Parses HTML input using CSS selector syntax and creates a new flowfile containing the extracted" +
    +        " element content for each matching CSS selector.")
    +@SeeAlso({ModifyHTMLElement.class, PutHTMLElement.class})
    +@WritesAttributes({@WritesAttribute(attribute="HTMLElement", description="Flowfile attribute where the element result" +
    +        " parsed from the HTML using the CSS selector syntax are placed if the destination is a flowfile attribute.")})
    +public class GetHTMLElement
    +        extends AbstractHTMLProcessor {
    +
    +    public static final String HTML_ELEMENT_ATTRIBUTE_NAME = "HTMLElement";
    +    public static final String DESTINATION_ATTRIBUTE = "flowfile-attribute";
    +    public static final String DESTINATION_CONTENT = "flowfile-content";
    +
    +    public static final PropertyDescriptor PREPEND_ELEMENT_VALUE = new PropertyDescriptor
    +            .Builder().name("Prepend Element value")
    +            .description("Prepends the specified value to the resulting Element")
    +            .required(false)
    +            .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
    +            .expressionLanguageSupported(true)
    +            .build();
    +
    +    public static final PropertyDescriptor APPEND_ELEMENT_VALUE = new PropertyDescriptor
    +            .Builder().name("Append Element value")
    +            .description("Appends the specified value to the resulting Element")
    +            .required(false)
    +            .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
    +            .expressionLanguageSupported(true)
    +            .build();
    +
    +    public static final PropertyDescriptor ATTRIBUTE_KEY = new PropertyDescriptor
    +            .Builder().name("Attribute Name")
    +            .description(("When getting the value of an element attribute this value is used as the key to determine" +
    +                    " which attribute on the selected element should be retrieved."))
    +            .required(false)
    +            .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
    +            .expressionLanguageSupported(true)
    +            .build();
    +
    +
    +    public static final PropertyDescriptor OUTPUT_TYPE = new PropertyDescriptor.Builder()
    +            .name("Output Type")
    +            .description("Controls the type of value that is retrieved from the element. " +
    +                    ELEMENT_HTML + "," + ELEMENT_TEXT + ", " + ELEMENT_ATTRIBUTE + " or " + ELEMENT_DATA)
    +            .required(true)
    +            .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
    +            .allowableValues(ELEMENT_HTML, ELEMENT_TEXT, ELEMENT_ATTRIBUTE, ELEMENT_DATA)
    +            .defaultValue(ELEMENT_HTML)
    +            .build();
    +
    +    public static final PropertyDescriptor DESTINATION = new PropertyDescriptor.Builder()
    +            .name("Destination")
    +            .description("Control if element extracted is written as a flowfile attribute or " +
    +                    "as flowfile content.")
    +            .required(true)
    +            .allowableValues(DESTINATION_ATTRIBUTE, DESTINATION_CONTENT)
    +            .defaultValue(DESTINATION_ATTRIBUTE)
    +            .build();
    +
    +    private List<PropertyDescriptor> descriptors;
    +
    +    private Set<Relationship> relationships;
    +
    +    @Override
    +    protected void init(final ProcessorInitializationContext context) {
    +        final List<PropertyDescriptor> descriptors = new ArrayList<>();
    +        descriptors.add(URL);
    +        descriptors.add(CSS_SELECTOR);
    +        descriptors.add(HTML_CHARSET);
    +        descriptors.add(OUTPUT_TYPE);
    +        descriptors.add(DESTINATION);
    +        descriptors.add(PREPEND_ELEMENT_VALUE);
    +        descriptors.add(APPEND_ELEMENT_VALUE);
    +        descriptors.add(ATTRIBUTE_KEY);
    +        this.descriptors = Collections.unmodifiableList(descriptors);
    +
    +        final Set<Relationship> relationships = new HashSet<>();
    +        relationships.add(REL_ORIGINAL);
    +        relationships.add(REL_SUCCESS);
    +        relationships.add(REL_FAILURE);
    +        relationships.add(REL_NOT_FOUND);
    +        this.relationships = Collections.unmodifiableSet(relationships);
    +    }
    +
    +    @Override
    +    public Set<Relationship> getRelationships() {
    +        return this.relationships;
    +    }
    +
    +    @Override
    +    public final List<PropertyDescriptor> getSupportedPropertyDescriptors() {
    +        return descriptors;
    +    }
    +
    +    @Override
    +    public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
    +        final FlowFile flowFile = session.get();
    +        if ( flowFile == null ) {
    +            return;
    +        }
    +
    +        try {
    +
    +            final Document doc = parseHTMLDocumentFromFlowfile(flowFile, context, session);
    +            final Elements eles = doc.select(context.getProperty(CSS_SELECTOR)
    +                    .evaluateAttributeExpressions().getValue());
    +            final String prependValue = context.getProperty(PREPEND_ELEMENT_VALUE)
    +                    .evaluateAttributeExpressions(flowFile).getValue();
    +            final String appendValue = context.getProperty(APPEND_ELEMENT_VALUE)
    +                    .evaluateAttributeExpressions(flowFile).getValue();
    +
    +            if (eles == null || eles.size() == 0) {
    +                //No element found
    +                session.transfer(flowFile, REL_NOT_FOUND);
    +            } else {
    +                for (final Element ele : eles) {
    +                    final FlowFile ff = session.create();
    +
    +                    switch (context.getProperty(DESTINATION).getValue()) {
    +                        case DESTINATION_ATTRIBUTE:
    +                            final FlowFile atFlowfile = session.putAttribute(ff, HTML_ELEMENT_ATTRIBUTE_NAME,
    +                                    extractElementValue(
    +                                            prependValue,
    +                                            context.getProperty(OUTPUT_TYPE).getValue(),
    +                                            appendValue,
    +                                            ele,
    +                                            context.getProperty(ATTRIBUTE_KEY).evaluateAttributeExpressions()
    +                                                    .getValue()));
    +                            session.getProvenanceReporter().create(atFlowfile);
    +                            session.transfer(atFlowfile, REL_SUCCESS);
    +                            break;
    +                        case DESTINATION_CONTENT:
    +                            final FlowFile conFlowfile = session.write(ff, new StreamCallback() {
    +                                @Override
    +                                public void process(InputStream inputStream, OutputStream outputStream) throws IOException {
    +                                    try {
    +                                        outputStream.write(extractElementValue(
    +                                                prependValue,
    +                                                context.getProperty(OUTPUT_TYPE).getValue(),
    +                                                appendValue,
    +                                                ele,
    +                                                context.getProperty(ATTRIBUTE_KEY).evaluateAttributeExpressions()
    +                                                        .getValue()).getBytes());
    +                                    } catch (Exception ex) {
    +                                        session.transfer(ff, REL_FAILURE);
    +                                    }
    +                                }
    +                            });
    +
    +                            session.getProvenanceReporter().create(conFlowfile);
    --- End diff --
    
    Same point as above, should be a FORK, not CREATE, but it'll be taken care of by the framework.


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] nifi pull request: NIFI-1156: HTML Parsing Processors Bundle

Posted by markap14 <gi...@git.apache.org>.
Github user markap14 commented on a diff in the pull request:

    https://github.com/apache/nifi/pull/124#discussion_r49479161
  
    --- Diff: nifi-nar-bundles/nifi-html-bundle/nifi-html-processors/src/main/java/org/apache/nifi/AbstractHTMLProcessor.java ---
    @@ -0,0 +1,120 @@
    +/*
    + * Licensed to the Apache Software Foundation (ASF) under one or more
    + * contributor license agreements.  See the NOTICE file distributed with
    + * this work for additional information regarding copyright ownership.
    + * The ASF licenses this file to You under the Apache License, Version 2.0
    + * (the "License"); you may not use this file except in compliance with
    + * the License.  You may obtain a copy of the License at
    + *
    + *     http://www.apache.org/licenses/LICENSE-2.0
    + *
    + * Unless required by applicable law or agreed to in writing, software
    + * distributed under the License is distributed on an "AS IS" BASIS,
    + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    + * See the License for the specific language governing permissions and
    + * limitations under the License.
    + */
    +package org.apache.nifi;
    +
    +import org.apache.nifi.components.PropertyDescriptor;
    +import org.apache.nifi.flowfile.FlowFile;
    +import org.apache.nifi.processor.AbstractProcessor;
    +import org.apache.nifi.processor.ProcessContext;
    +import org.apache.nifi.processor.ProcessSession;
    +import org.apache.nifi.processor.Relationship;
    +import org.apache.nifi.processor.io.InputStreamCallback;
    +import org.apache.nifi.processor.util.StandardValidators;
    +import org.jsoup.Jsoup;
    +import org.jsoup.nodes.Document;
    +
    +import java.io.IOException;
    +import java.io.InputStream;
    +import java.util.concurrent.atomic.AtomicReference;
    +
    +public abstract class AbstractHTMLProcessor extends AbstractProcessor {
    +
    +    protected static final String ELEMENT_HTML = "HTML";
    +    protected static final String ELEMENT_TEXT = "Text";
    +    protected static final String ELEMENT_DATA = "Data";
    +    protected static final String ELEMENT_ATTRIBUTE = "Attribute";
    +
    +    public static final PropertyDescriptor URL = new PropertyDescriptor
    +            .Builder().name("URL")
    +            .description("Base URL for the HTML page being parsed.")
    +            .required(true)
    +            .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
    +            .build();
    +
    +    public static final PropertyDescriptor CSS_SELECTOR = new PropertyDescriptor
    +            .Builder().name("CSS Selector")
    +            .description("CSS selector syntax string used to extract the desired HTML element(s).")
    +            .required(true)
    +            .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
    +            .expressionLanguageSupported(true)
    +            .build();
    +
    +    public static final PropertyDescriptor HTML_CHARSET = new PropertyDescriptor
    +            .Builder().name("HTML character encoding")
    +            .description("Character encoding of the input HTML")
    +            .defaultValue("UTF-8")
    +            .required(true)
    +            .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
    +            .build();
    +
    +    public static final Relationship REL_ORIGINAL = new Relationship.Builder()
    +            .name("original")
    +            .description("The original HTML input")
    +            .build();
    +
    +    public static final Relationship REL_SUCCESS = new Relationship.Builder()
    +            .name("success")
    +            .description("Successfully parsed HTML element")
    +            .build();
    +
    +    public static final Relationship REL_FAILURE = new Relationship.Builder()
    +            .name("failure")
    +            .description("Failed to parse HTML content")
    --- End diff --
    
    I am not sure that I understand the difference between 'failure' and 'invalid html' relationships. I would think that if we "fail to parse HTML content", then it is because "The input HTML syntax is invalid" but those are two separate relationships. Can you expand on the descriptions a bit?


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] nifi pull request: NIFI-1156: HTML Parsing Processors Bundle

Posted by markap14 <gi...@git.apache.org>.
Github user markap14 commented on a diff in the pull request:

    https://github.com/apache/nifi/pull/124#discussion_r49481366
  
    --- Diff: nifi-nar-bundles/nifi-html-bundle/nifi-html-processors/src/test/java/org/apache/nifi/TestGetHTMLElement.java ---
    @@ -0,0 +1,319 @@
    +/*
    + * Licensed to the Apache Software Foundation (ASF) under one or more
    + * contributor license agreements.  See the NOTICE file distributed with
    + * this work for additional information regarding copyright ownership.
    + * The ASF licenses this file to You under the Apache License, Version 2.0
    + * (the "License"); you may not use this file except in compliance with
    + * the License.  You may obtain a copy of the License at
    + *
    + *     http://www.apache.org/licenses/LICENSE-2.0
    + *
    + * Unless required by applicable law or agreed to in writing, software
    + * distributed under the License is distributed on an "AS IS" BASIS,
    + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    + * See the License for the specific language governing permissions and
    + * limitations under the License.
    + */
    +package org.apache.nifi;
    +
    +import org.apache.commons.lang3.StringUtils;
    +import org.apache.nifi.flowfile.FlowFile;
    +import org.apache.nifi.processor.ProcessSession;
    +import org.apache.nifi.util.MockFlowFile;
    +import org.apache.nifi.util.TestRunner;
    +import org.apache.nifi.util.TestRunners;
    +import org.junit.Before;
    +import org.junit.Test;
    +
    +import java.lang.Exception;
    +import java.util.List;
    +
    +import static org.junit.Assert.assertTrue;
    +
    +public class TestGetHTMLElement extends AbstractHTMLTest {
    +
    +    private TestRunner testRunner;
    +
    +    @Before
    +    public void init() {
    +        testRunner = TestRunners.newTestRunner(GetHTMLElement.class);
    +        testRunner.setProperty(GetHTMLElement.URL, "http://localhost");
    +        testRunner.setProperty(GetHTMLElement.OUTPUT_TYPE, GetHTMLElement.ELEMENT_HTML);
    +        testRunner.setProperty(GetHTMLElement.DESTINATION, GetHTMLElement.DESTINATION_CONTENT);
    +        testRunner.setProperty(GetHTMLElement.HTML_CHARSET, "UTF-8");
    +    }
    +
    +    @Test
    +    public void testNoElementFound() throws Exception {
    +        testRunner.setProperty(GetHTMLElement.CSS_SELECTOR, "b");   //Bold element is not present in sample HTML
    +//        testRunner.setProperty(GetHTMLElement.APPEND_ELEMENT_VALUE, "");
    --- End diff --
    
    Should remove code instead of commenting out


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] nifi pull request: NIFI-1156: HTML Parsing Processors Bundle

Posted by markap14 <gi...@git.apache.org>.
Github user markap14 commented on a diff in the pull request:

    https://github.com/apache/nifi/pull/124#discussion_r49480317
  
    --- Diff: nifi-nar-bundles/nifi-html-bundle/nifi-html-processors/src/main/java/org/apache/nifi/GetHTMLElement.java ---
    @@ -0,0 +1,243 @@
    +/*
    + * Licensed to the Apache Software Foundation (ASF) under one or more
    + * contributor license agreements.  See the NOTICE file distributed with
    + * this work for additional information regarding copyright ownership.
    + * The ASF licenses this file to You under the Apache License, Version 2.0
    + * (the "License"); you may not use this file except in compliance with
    + * the License.  You may obtain a copy of the License at
    + *
    + *     http://www.apache.org/licenses/LICENSE-2.0
    + *
    + * Unless required by applicable law or agreed to in writing, software
    + * distributed under the License is distributed on an "AS IS" BASIS,
    + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    + * See the License for the specific language governing permissions and
    + * limitations under the License.
    + */
    +package org.apache.nifi;
    +
    +import org.apache.commons.lang3.StringUtils;
    +import org.apache.nifi.components.PropertyDescriptor;
    +import org.apache.nifi.flowfile.FlowFile;
    +import org.apache.nifi.processor.ProcessContext;
    +import org.apache.nifi.processor.ProcessSession;
    +import org.apache.nifi.processor.Relationship;
    +import org.apache.nifi.processor.ProcessorInitializationContext;
    +import org.apache.nifi.annotation.behavior.WritesAttribute;
    +import org.apache.nifi.annotation.behavior.WritesAttributes;
    +import org.apache.nifi.annotation.documentation.CapabilityDescription;
    +import org.apache.nifi.annotation.documentation.SeeAlso;
    +import org.apache.nifi.annotation.documentation.Tags;
    +import org.apache.nifi.processor.exception.ProcessException;
    +import org.apache.nifi.processor.io.StreamCallback;
    +import org.apache.nifi.processor.util.StandardValidators;
    +import org.jsoup.nodes.Document;
    +import org.jsoup.nodes.Element;
    +import org.jsoup.select.Elements;
    +
    +import java.io.IOException;
    +import java.io.InputStream;
    +import java.io.OutputStream;
    +import java.util.ArrayList;
    +import java.util.List;
    +import java.util.Set;
    +import java.util.HashSet;
    +import java.util.Collections;
    +
    +@Tags({"get", "html", "dom", "css", "element"})
    +@CapabilityDescription("Parses HTML input using CSS selector syntax and creates a new flowfile containing the extracted" +
    +        " element content for each matching CSS selector.")
    +@SeeAlso({ModifyHTMLElement.class, PutHTMLElement.class})
    +@WritesAttributes({@WritesAttribute(attribute="HTMLElement", description="Flowfile attribute where the element result" +
    +        " parsed from the HTML using the CSS selector syntax are placed if the destination is a flowfile attribute.")})
    +public class GetHTMLElement
    +        extends AbstractHTMLProcessor {
    +
    +    public static final String HTML_ELEMENT_ATTRIBUTE_NAME = "HTMLElement";
    +    public static final String DESTINATION_ATTRIBUTE = "flowfile-attribute";
    +    public static final String DESTINATION_CONTENT = "flowfile-content";
    +
    +    public static final PropertyDescriptor PREPEND_ELEMENT_VALUE = new PropertyDescriptor
    +            .Builder().name("Prepend Element value")
    +            .description("Prepends the specified value to the resulting Element")
    +            .required(false)
    +            .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
    +            .expressionLanguageSupported(true)
    +            .build();
    +
    +    public static final PropertyDescriptor APPEND_ELEMENT_VALUE = new PropertyDescriptor
    +            .Builder().name("Append Element value")
    +            .description("Appends the specified value to the resulting Element")
    +            .required(false)
    +            .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
    +            .expressionLanguageSupported(true)
    +            .build();
    +
    +    public static final PropertyDescriptor ATTRIBUTE_KEY = new PropertyDescriptor
    +            .Builder().name("Attribute Name")
    +            .description(("When getting the value of an element attribute this value is used as the key to determine" +
    +                    " which attribute on the selected element should be retrieved."))
    +            .required(false)
    +            .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
    +            .expressionLanguageSupported(true)
    +            .build();
    +
    +
    +    public static final PropertyDescriptor OUTPUT_TYPE = new PropertyDescriptor.Builder()
    +            .name("Output Type")
    +            .description("Controls the type of value that is retrieved from the element. " +
    +                    ELEMENT_HTML + "," + ELEMENT_TEXT + ", " + ELEMENT_ATTRIBUTE + " or " + ELEMENT_DATA)
    --- End diff --
    
    We generally like to avoid listing the Allowable Values in the description. The UI will automatically provide this information based on the Allowable Values given; this also allows each of the Allowable Values to provide descriptions, if appropriate.


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] nifi pull request: NIFI-1156: HTML Parsing Processors Bundle

Posted by markap14 <gi...@git.apache.org>.
Github user markap14 commented on a diff in the pull request:

    https://github.com/apache/nifi/pull/124#discussion_r49479814
  
    --- Diff: nifi-nar-bundles/nifi-html-bundle/nifi-html-processors/src/main/java/org/apache/nifi/GetHTMLElement.java ---
    @@ -0,0 +1,243 @@
    +/*
    + * Licensed to the Apache Software Foundation (ASF) under one or more
    + * contributor license agreements.  See the NOTICE file distributed with
    + * this work for additional information regarding copyright ownership.
    + * The ASF licenses this file to You under the Apache License, Version 2.0
    + * (the "License"); you may not use this file except in compliance with
    + * the License.  You may obtain a copy of the License at
    + *
    + *     http://www.apache.org/licenses/LICENSE-2.0
    + *
    + * Unless required by applicable law or agreed to in writing, software
    + * distributed under the License is distributed on an "AS IS" BASIS,
    + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    + * See the License for the specific language governing permissions and
    + * limitations under the License.
    + */
    +package org.apache.nifi;
    +
    +import org.apache.commons.lang3.StringUtils;
    +import org.apache.nifi.components.PropertyDescriptor;
    +import org.apache.nifi.flowfile.FlowFile;
    +import org.apache.nifi.processor.ProcessContext;
    +import org.apache.nifi.processor.ProcessSession;
    +import org.apache.nifi.processor.Relationship;
    +import org.apache.nifi.processor.ProcessorInitializationContext;
    +import org.apache.nifi.annotation.behavior.WritesAttribute;
    +import org.apache.nifi.annotation.behavior.WritesAttributes;
    +import org.apache.nifi.annotation.documentation.CapabilityDescription;
    +import org.apache.nifi.annotation.documentation.SeeAlso;
    +import org.apache.nifi.annotation.documentation.Tags;
    +import org.apache.nifi.processor.exception.ProcessException;
    +import org.apache.nifi.processor.io.StreamCallback;
    +import org.apache.nifi.processor.util.StandardValidators;
    +import org.jsoup.nodes.Document;
    +import org.jsoup.nodes.Element;
    +import org.jsoup.select.Elements;
    +
    +import java.io.IOException;
    +import java.io.InputStream;
    +import java.io.OutputStream;
    +import java.util.ArrayList;
    +import java.util.List;
    +import java.util.Set;
    +import java.util.HashSet;
    +import java.util.Collections;
    +
    +@Tags({"get", "html", "dom", "css", "element"})
    +@CapabilityDescription("Parses HTML input using CSS selector syntax and creates a new flowfile containing the extracted" +
    +        " element content for each matching CSS selector.")
    +@SeeAlso({ModifyHTMLElement.class, PutHTMLElement.class})
    +@WritesAttributes({@WritesAttribute(attribute="HTMLElement", description="Flowfile attribute where the element result" +
    +        " parsed from the HTML using the CSS selector syntax are placed if the destination is a flowfile attribute.")})
    +public class GetHTMLElement
    +        extends AbstractHTMLProcessor {
    +
    +    public static final String HTML_ELEMENT_ATTRIBUTE_NAME = "HTMLElement";
    +    public static final String DESTINATION_ATTRIBUTE = "flowfile-attribute";
    +    public static final String DESTINATION_CONTENT = "flowfile-content";
    +
    +    public static final PropertyDescriptor PREPEND_ELEMENT_VALUE = new PropertyDescriptor
    +            .Builder().name("Prepend Element value")
    +            .description("Prepends the specified value to the resulting Element")
    +            .required(false)
    +            .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
    +            .expressionLanguageSupported(true)
    +            .build();
    +
    +    public static final PropertyDescriptor APPEND_ELEMENT_VALUE = new PropertyDescriptor
    +            .Builder().name("Append Element value")
    +            .description("Appends the specified value to the resulting Element")
    +            .required(false)
    +            .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
    +            .expressionLanguageSupported(true)
    +            .build();
    +
    +    public static final PropertyDescriptor ATTRIBUTE_KEY = new PropertyDescriptor
    +            .Builder().name("Attribute Name")
    +            .description(("When getting the value of an element attribute this value is used as the key to determine" +
    +                    " which attribute on the selected element should be retrieved."))
    +            .required(false)
    +            .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
    +            .expressionLanguageSupported(true)
    +            .build();
    +
    +
    +    public static final PropertyDescriptor OUTPUT_TYPE = new PropertyDescriptor.Builder()
    +            .name("Output Type")
    +            .description("Controls the type of value that is retrieved from the element. " +
    +                    ELEMENT_HTML + "," + ELEMENT_TEXT + ", " + ELEMENT_ATTRIBUTE + " or " + ELEMENT_DATA)
    +            .required(true)
    +            .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
    +            .allowableValues(ELEMENT_HTML, ELEMENT_TEXT, ELEMENT_ATTRIBUTE, ELEMENT_DATA)
    +            .defaultValue(ELEMENT_HTML)
    +            .build();
    +
    +    public static final PropertyDescriptor DESTINATION = new PropertyDescriptor.Builder()
    +            .name("Destination")
    +            .description("Control if element extracted is written as a flowfile attribute or " +
    +                    "as flowfile content.")
    +            .required(true)
    +            .allowableValues(DESTINATION_ATTRIBUTE, DESTINATION_CONTENT)
    +            .defaultValue(DESTINATION_ATTRIBUTE)
    +            .build();
    +
    +    private List<PropertyDescriptor> descriptors;
    +
    +    private Set<Relationship> relationships;
    +
    +    @Override
    +    protected void init(final ProcessorInitializationContext context) {
    +        final List<PropertyDescriptor> descriptors = new ArrayList<>();
    +        descriptors.add(URL);
    +        descriptors.add(CSS_SELECTOR);
    +        descriptors.add(HTML_CHARSET);
    +        descriptors.add(OUTPUT_TYPE);
    +        descriptors.add(DESTINATION);
    +        descriptors.add(PREPEND_ELEMENT_VALUE);
    +        descriptors.add(APPEND_ELEMENT_VALUE);
    +        descriptors.add(ATTRIBUTE_KEY);
    +        this.descriptors = Collections.unmodifiableList(descriptors);
    +
    +        final Set<Relationship> relationships = new HashSet<>();
    +        relationships.add(REL_ORIGINAL);
    +        relationships.add(REL_SUCCESS);
    +        relationships.add(REL_FAILURE);
    +        relationships.add(REL_NOT_FOUND);
    +        this.relationships = Collections.unmodifiableSet(relationships);
    +    }
    +
    +    @Override
    +    public Set<Relationship> getRelationships() {
    +        return this.relationships;
    +    }
    +
    +    @Override
    +    public final List<PropertyDescriptor> getSupportedPropertyDescriptors() {
    +        return descriptors;
    +    }
    +
    +    @Override
    +    public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
    +        final FlowFile flowFile = session.get();
    +        if ( flowFile == null ) {
    +            return;
    +        }
    +
    +        try {
    +
    +            final Document doc = parseHTMLDocumentFromFlowfile(flowFile, context, session);
    +            final Elements eles = doc.select(context.getProperty(CSS_SELECTOR)
    +                    .evaluateAttributeExpressions().getValue());
    +            final String prependValue = context.getProperty(PREPEND_ELEMENT_VALUE)
    +                    .evaluateAttributeExpressions(flowFile).getValue();
    +            final String appendValue = context.getProperty(APPEND_ELEMENT_VALUE)
    +                    .evaluateAttributeExpressions(flowFile).getValue();
    +
    +            if (eles == null || eles.size() == 0) {
    +                //No element found
    +                session.transfer(flowFile, REL_NOT_FOUND);
    +            } else {
    +                for (final Element ele : eles) {
    +                    final FlowFile ff = session.create();
    --- End diff --
    
    Should be calling session.create(flowFile) so that the Provenance Lineage is handled appropriately and original FlowFile's attributes are transferred along.


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] nifi pull request: NIFI-1156: HTML Parsing Processors Bundle

Posted by joewitt <gi...@git.apache.org>.
Github user joewitt commented on the pull request:

    https://github.com/apache/nifi/pull/124#issuecomment-157222242
  
    MIT is perfectly fine both for source and binary dependency.  Details can be found here with links to all sorts of goodness "https://nifi.apache.org/licensing-guide.html"


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] nifi pull request: NIFI-1156: HTML Parsing Processors Bundle

Posted by markap14 <gi...@git.apache.org>.
Github user markap14 commented on the pull request:

    https://github.com/apache/nifi/pull/124#issuecomment-173351611
  
    Jeremy,
    
    I've been reviewing this and I made a handful of minor changes:
    * Copied licensing information to the nar - it was already present in nifi-assembly. Just had to create a LICENSE file for the nar itself.
    * Updated version from 0.4.0-SNAPSHOT to 0.4.2-SNAPSHOT
    * Improved error handling (if we hit any RuntimeException, such as the Content Repository is out of disk space, it was routing to the "invalid html" relationship)
    * Updated some code to match the nifi style a bit more (adding 'final' keywords, using longer max line width (200 chars is the max we use with nifi))
    
    I did not change the logic of any of the Processors, though. All very solid stuff.
    Excited to see this getting into NiFi! Thanks for all of the work that you've put into this!


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] nifi pull request: NIFI-1156: HTML Parsing Processors Bundle

Posted by markap14 <gi...@git.apache.org>.
Github user markap14 commented on a diff in the pull request:

    https://github.com/apache/nifi/pull/124#discussion_r49480952
  
    --- Diff: nifi-nar-bundles/nifi-html-bundle/nifi-html-processors/src/main/java/org/apache/nifi/PutHTMLElement.java ---
    @@ -0,0 +1,150 @@
    +/*
    + * Licensed to the Apache Software Foundation (ASF) under one or more
    + * contributor license agreements.  See the NOTICE file distributed with
    + * this work for additional information regarding copyright ownership.
    + * The ASF licenses this file to You under the Apache License, Version 2.0
    + * (the "License"); you may not use this file except in compliance with
    + * the License.  You may obtain a copy of the License at
    + *
    + *     http://www.apache.org/licenses/LICENSE-2.0
    + *
    + * Unless required by applicable law or agreed to in writing, software
    + * distributed under the License is distributed on an "AS IS" BASIS,
    + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    + * See the License for the specific language governing permissions and
    + * limitations under the License.
    + */
    +package org.apache.nifi;
    +
    +import org.apache.nifi.annotation.documentation.CapabilityDescription;
    +import org.apache.nifi.annotation.documentation.SeeAlso;
    +import org.apache.nifi.annotation.documentation.Tags;
    +import org.apache.nifi.components.PropertyDescriptor;
    +import org.apache.nifi.flowfile.FlowFile;
    +import org.apache.nifi.processor.ProcessContext;
    +import org.apache.nifi.processor.ProcessSession;
    +import org.apache.nifi.processor.Relationship;
    +import org.apache.nifi.processor.ProcessorInitializationContext;
    +import org.apache.nifi.processor.exception.ProcessException;
    +import org.apache.nifi.processor.io.StreamCallback;
    +import org.apache.nifi.processor.util.StandardValidators;
    +import org.jsoup.nodes.Document;
    +import org.jsoup.nodes.Element;
    +import org.jsoup.select.Elements;
    +
    +import java.io.IOException;
    +import java.io.InputStream;
    +import java.io.OutputStream;
    +import java.util.ArrayList;
    +import java.util.List;
    +import java.util.Set;
    +import java.util.HashSet;
    +import java.util.Collections;
    +
    +@Tags({"put", "html", "dom", "css", "element"})
    +@CapabilityDescription("Creates a new HTML element in the input HTML")
    +@SeeAlso({GetHTMLElement.class, ModifyHTMLElement.class})
    +public class PutHTMLElement extends AbstractHTMLProcessor {
    +
    +    public static final String APPEND_ELEMENT = "append-html";
    +    public static final String PREPEND_ELEMENT = "prepend-html";
    +
    +    public static final PropertyDescriptor PUT_LOCATION_TYPE = new PropertyDescriptor.Builder()
    +            .name("Element Insert Location Type")
    +            .description("Controls whether the new element is prepended or appended to the children of the " +
    +                    "Element located by the CSS selector. EX: prepended value '<b>Hi</b>' inside of " +
    +                    "Element (using CSS Selector 'p') '<p>There</p>' would result in " +
    +                    "'<p><b>Hi</b>There</p>'. Appending the value would result in '<p>There<b>Hi</b></p>'")
    +            .required(true)
    +            .allowableValues(APPEND_ELEMENT, PREPEND_ELEMENT)
    +            .defaultValue(APPEND_ELEMENT)
    +            .build();
    +
    +    public static final PropertyDescriptor PUT_VALUE = new PropertyDescriptor.Builder()
    +            .name("Put Value")
    +            .description("Value used when creating the new Element. Value should be a valid HTML element. " +
    +                    "The text should be supplied unencoded: characters like '<', '>', etc will be properly HTML " +
    +                    "encoded in the output.")
    +            .required(true)
    +            .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
    +            .expressionLanguageSupported(true)
    +            .build();
    +
    +    private List<PropertyDescriptor> descriptors;
    +
    +    private Set<Relationship> relationships;
    +
    +    @Override
    +    protected void init(final ProcessorInitializationContext context) {
    +        final List<PropertyDescriptor> descriptors = new ArrayList<PropertyDescriptor>();
    +        descriptors.add(URL);
    +        descriptors.add(CSS_SELECTOR);
    +        descriptors.add(HTML_CHARSET);
    +        descriptors.add(PUT_LOCATION_TYPE);
    +        descriptors.add(PUT_VALUE);
    +        this.descriptors = Collections.unmodifiableList(descriptors);
    +
    +        final Set<Relationship> relationships = new HashSet<Relationship>();
    +        relationships.add(REL_ORIGINAL);
    +        relationships.add(REL_SUCCESS);
    +        relationships.add(REL_FAILURE);
    +        relationships.add(REL_INVALID_HTML);
    +        this.relationships = Collections.unmodifiableSet(relationships);
    +    }
    +
    +    @Override
    +    public Set<Relationship> getRelationships() {
    +        return this.relationships;
    +    }
    +
    +    @Override
    +    public final List<PropertyDescriptor> getSupportedPropertyDescriptors() {
    +        return descriptors;
    +    }
    +
    +    @Override
    +    public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
    +        final FlowFile flowFile = session.get();
    +        if (flowFile == null) {
    +            return;
    +        }
    +
    +        try {
    +            final Document doc = parseHTMLDocumentFromFlowfile(flowFile, context, session);
    +            final Elements eles = doc.select(context.getProperty(CSS_SELECTOR).evaluateAttributeExpressions().getValue());
    +
    +            if (eles == null || eles.size() == 0) {
    +                //No element found
    +                session.transfer(flowFile, REL_NOT_FOUND);
    +            } else {
    +                for (Element ele : eles) {
    +                    switch (context.getProperty(PUT_LOCATION_TYPE).getValue()) {
    +                        case APPEND_ELEMENT:
    +                            ele.append(context.getProperty(PUT_VALUE).evaluateAttributeExpressions().getValue());
    --- End diff --
    
    This should call evaluateAttributeExpressions(flowFile) so that we can take into account FlowFile Attributes.


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] nifi pull request: NIFI-1156: HTML Parsing Processors Bundle

Posted by asfgit <gi...@git.apache.org>.
Github user asfgit closed the pull request at:

    https://github.com/apache/nifi/pull/124


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] nifi pull request: NIFI-1156: HTML Parsing Processors Bundle

Posted by markap14 <gi...@git.apache.org>.
Github user markap14 commented on a diff in the pull request:

    https://github.com/apache/nifi/pull/124#discussion_r49480407
  
    --- Diff: nifi-nar-bundles/nifi-html-bundle/nifi-html-processors/src/main/java/org/apache/nifi/GetHTMLElement.java ---
    @@ -0,0 +1,243 @@
    +/*
    + * Licensed to the Apache Software Foundation (ASF) under one or more
    + * contributor license agreements.  See the NOTICE file distributed with
    + * this work for additional information regarding copyright ownership.
    + * The ASF licenses this file to You under the Apache License, Version 2.0
    + * (the "License"); you may not use this file except in compliance with
    + * the License.  You may obtain a copy of the License at
    + *
    + *     http://www.apache.org/licenses/LICENSE-2.0
    + *
    + * Unless required by applicable law or agreed to in writing, software
    + * distributed under the License is distributed on an "AS IS" BASIS,
    + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    + * See the License for the specific language governing permissions and
    + * limitations under the License.
    + */
    +package org.apache.nifi;
    +
    +import org.apache.commons.lang3.StringUtils;
    +import org.apache.nifi.components.PropertyDescriptor;
    +import org.apache.nifi.flowfile.FlowFile;
    +import org.apache.nifi.processor.ProcessContext;
    +import org.apache.nifi.processor.ProcessSession;
    +import org.apache.nifi.processor.Relationship;
    +import org.apache.nifi.processor.ProcessorInitializationContext;
    +import org.apache.nifi.annotation.behavior.WritesAttribute;
    +import org.apache.nifi.annotation.behavior.WritesAttributes;
    +import org.apache.nifi.annotation.documentation.CapabilityDescription;
    +import org.apache.nifi.annotation.documentation.SeeAlso;
    +import org.apache.nifi.annotation.documentation.Tags;
    +import org.apache.nifi.processor.exception.ProcessException;
    +import org.apache.nifi.processor.io.StreamCallback;
    +import org.apache.nifi.processor.util.StandardValidators;
    +import org.jsoup.nodes.Document;
    +import org.jsoup.nodes.Element;
    +import org.jsoup.select.Elements;
    +
    +import java.io.IOException;
    +import java.io.InputStream;
    +import java.io.OutputStream;
    +import java.util.ArrayList;
    +import java.util.List;
    +import java.util.Set;
    +import java.util.HashSet;
    +import java.util.Collections;
    +
    +@Tags({"get", "html", "dom", "css", "element"})
    +@CapabilityDescription("Parses HTML input using CSS selector syntax and creates a new flowfile containing the extracted" +
    +        " element content for each matching CSS selector.")
    +@SeeAlso({ModifyHTMLElement.class, PutHTMLElement.class})
    +@WritesAttributes({@WritesAttribute(attribute="HTMLElement", description="Flowfile attribute where the element result" +
    +        " parsed from the HTML using the CSS selector syntax are placed if the destination is a flowfile attribute.")})
    +public class GetHTMLElement
    +        extends AbstractHTMLProcessor {
    +
    +    public static final String HTML_ELEMENT_ATTRIBUTE_NAME = "HTMLElement";
    +    public static final String DESTINATION_ATTRIBUTE = "flowfile-attribute";
    +    public static final String DESTINATION_CONTENT = "flowfile-content";
    +
    +    public static final PropertyDescriptor PREPEND_ELEMENT_VALUE = new PropertyDescriptor
    +            .Builder().name("Prepend Element value")
    +            .description("Prepends the specified value to the resulting Element")
    +            .required(false)
    +            .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
    +            .expressionLanguageSupported(true)
    +            .build();
    +
    +    public static final PropertyDescriptor APPEND_ELEMENT_VALUE = new PropertyDescriptor
    +            .Builder().name("Append Element value")
    --- End diff --
    
    Recommend we keep with Title Case: Append Element Value


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] nifi pull request: NIFI-1156: HTML Parsing Processors Bundle

Posted by markap14 <gi...@git.apache.org>.
Github user markap14 commented on a diff in the pull request:

    https://github.com/apache/nifi/pull/124#discussion_r49481551
  
    --- Diff: nifi-nar-bundles/nifi-html-bundle/nifi-html-processors/src/test/java/org/apache/nifi/TestGetHTMLElement.java ---
    @@ -0,0 +1,319 @@
    +/*
    + * Licensed to the Apache Software Foundation (ASF) under one or more
    + * contributor license agreements.  See the NOTICE file distributed with
    + * this work for additional information regarding copyright ownership.
    + * The ASF licenses this file to You under the Apache License, Version 2.0
    + * (the "License"); you may not use this file except in compliance with
    + * the License.  You may obtain a copy of the License at
    + *
    + *     http://www.apache.org/licenses/LICENSE-2.0
    + *
    + * Unless required by applicable law or agreed to in writing, software
    + * distributed under the License is distributed on an "AS IS" BASIS,
    + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    + * See the License for the specific language governing permissions and
    + * limitations under the License.
    + */
    +package org.apache.nifi;
    +
    +import org.apache.commons.lang3.StringUtils;
    +import org.apache.nifi.flowfile.FlowFile;
    +import org.apache.nifi.processor.ProcessSession;
    +import org.apache.nifi.util.MockFlowFile;
    +import org.apache.nifi.util.TestRunner;
    +import org.apache.nifi.util.TestRunners;
    +import org.junit.Before;
    +import org.junit.Test;
    +
    +import java.lang.Exception;
    +import java.util.List;
    +
    +import static org.junit.Assert.assertTrue;
    +
    +public class TestGetHTMLElement extends AbstractHTMLTest {
    +
    +    private TestRunner testRunner;
    +
    +    @Before
    +    public void init() {
    +        testRunner = TestRunners.newTestRunner(GetHTMLElement.class);
    +        testRunner.setProperty(GetHTMLElement.URL, "http://localhost");
    +        testRunner.setProperty(GetHTMLElement.OUTPUT_TYPE, GetHTMLElement.ELEMENT_HTML);
    +        testRunner.setProperty(GetHTMLElement.DESTINATION, GetHTMLElement.DESTINATION_CONTENT);
    +        testRunner.setProperty(GetHTMLElement.HTML_CHARSET, "UTF-8");
    +    }
    +
    +    @Test
    +    public void testNoElementFound() throws Exception {
    +        testRunner.setProperty(GetHTMLElement.CSS_SELECTOR, "b");   //Bold element is not present in sample HTML
    +//        testRunner.setProperty(GetHTMLElement.APPEND_ELEMENT_VALUE, "");
    +
    +        ProcessSession session = testRunner.getProcessSessionFactory().createSession();
    +        FlowFile ff = writeContentToNewFlowFile(HTML.getBytes(), session);
    +
    +        testRunner.enqueue(ff);
    +        testRunner.run();
    +
    +        testRunner.assertTransferCount(GetHTMLElement.REL_SUCCESS, 0);
    +        testRunner.assertTransferCount(GetHTMLElement.REL_FAILURE, 0);
    +        testRunner.assertTransferCount(GetHTMLElement.REL_NOT_FOUND, 1);
    +    }
    +
    +    @Test
    +    public void testInvalidSelector() throws Exception {
    +        testRunner.setProperty(GetHTMLElement.CSS_SELECTOR, "InvalidCSSSelectorSyntax");
    +
    +        ProcessSession session = testRunner.getProcessSessionFactory().createSession();
    +        FlowFile ff = writeContentToNewFlowFile(HTML.getBytes(), session);
    +
    +        testRunner.enqueue(ff);
    --- End diff --
    
    This all is a lot easier if we just do testRunner.enqueue(HTML.getBytes()) - no need to create a session and write to a FlowFile.


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] nifi pull request: NIFI-1156: HTML Parsing Processors Bundle

Posted by markap14 <gi...@git.apache.org>.
Github user markap14 commented on the pull request:

    https://github.com/apache/nifi/pull/124#issuecomment-170978164
  
    @jdye64 I left a lot of inline comments while doing a code review. Mostly just pointing out either naming conventions that we should strive to be consistent with, or pointing out some utils/convenience methods that will help to keep the code clean.
    
    Overall, I think it's an awesome contribution! Thanks for putting in all of the effort to get this going. Code looks great overall - excellent unit tests to verify behavior. Very cool stuff!


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] nifi pull request: NIFI-1156: HTML Parsing Processors Bundle

Posted by markap14 <gi...@git.apache.org>.
Github user markap14 commented on a diff in the pull request:

    https://github.com/apache/nifi/pull/124#discussion_r49478696
  
    --- Diff: nifi-nar-bundles/nifi-html-bundle/nifi-html-processors/src/main/java/org/apache/nifi/AbstractHTMLProcessor.java ---
    @@ -0,0 +1,120 @@
    +/*
    + * Licensed to the Apache Software Foundation (ASF) under one or more
    + * contributor license agreements.  See the NOTICE file distributed with
    + * this work for additional information regarding copyright ownership.
    + * The ASF licenses this file to You under the Apache License, Version 2.0
    + * (the "License"); you may not use this file except in compliance with
    + * the License.  You may obtain a copy of the License at
    + *
    + *     http://www.apache.org/licenses/LICENSE-2.0
    + *
    + * Unless required by applicable law or agreed to in writing, software
    + * distributed under the License is distributed on an "AS IS" BASIS,
    + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    + * See the License for the specific language governing permissions and
    + * limitations under the License.
    + */
    +package org.apache.nifi;
    +
    +import org.apache.nifi.components.PropertyDescriptor;
    +import org.apache.nifi.flowfile.FlowFile;
    +import org.apache.nifi.processor.AbstractProcessor;
    +import org.apache.nifi.processor.ProcessContext;
    +import org.apache.nifi.processor.ProcessSession;
    +import org.apache.nifi.processor.Relationship;
    +import org.apache.nifi.processor.io.InputStreamCallback;
    +import org.apache.nifi.processor.util.StandardValidators;
    +import org.jsoup.Jsoup;
    +import org.jsoup.nodes.Document;
    +
    +import java.io.IOException;
    +import java.io.InputStream;
    +import java.util.concurrent.atomic.AtomicReference;
    +
    +public abstract class AbstractHTMLProcessor extends AbstractProcessor {
    +
    +    protected static final String ELEMENT_HTML = "HTML";
    +    protected static final String ELEMENT_TEXT = "Text";
    +    protected static final String ELEMENT_DATA = "Data";
    +    protected static final String ELEMENT_ATTRIBUTE = "Attribute";
    +
    +    public static final PropertyDescriptor URL = new PropertyDescriptor
    +            .Builder().name("URL")
    +            .description("Base URL for the HTML page being parsed.")
    +            .required(true)
    +            .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
    +            .build();
    +
    +    public static final PropertyDescriptor CSS_SELECTOR = new PropertyDescriptor
    +            .Builder().name("CSS Selector")
    +            .description("CSS selector syntax string used to extract the desired HTML element(s).")
    +            .required(true)
    +            .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
    +            .expressionLanguageSupported(true)
    +            .build();
    +
    +    public static final PropertyDescriptor HTML_CHARSET = new PropertyDescriptor
    +            .Builder().name("HTML character encoding")
    --- End diff --
    
    Recommend using Title Case in order to stay consistent with other property names


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] nifi pull request: NIFI-1156: HTML Parsing Processors Bundle

Posted by markap14 <gi...@git.apache.org>.
Github user markap14 commented on a diff in the pull request:

    https://github.com/apache/nifi/pull/124#discussion_r49479943
  
    --- Diff: nifi-nar-bundles/nifi-html-bundle/nifi-html-processors/src/main/java/org/apache/nifi/GetHTMLElement.java ---
    @@ -0,0 +1,243 @@
    +/*
    + * Licensed to the Apache Software Foundation (ASF) under one or more
    + * contributor license agreements.  See the NOTICE file distributed with
    + * this work for additional information regarding copyright ownership.
    + * The ASF licenses this file to You under the Apache License, Version 2.0
    + * (the "License"); you may not use this file except in compliance with
    + * the License.  You may obtain a copy of the License at
    + *
    + *     http://www.apache.org/licenses/LICENSE-2.0
    + *
    + * Unless required by applicable law or agreed to in writing, software
    + * distributed under the License is distributed on an "AS IS" BASIS,
    + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    + * See the License for the specific language governing permissions and
    + * limitations under the License.
    + */
    +package org.apache.nifi;
    +
    +import org.apache.commons.lang3.StringUtils;
    +import org.apache.nifi.components.PropertyDescriptor;
    +import org.apache.nifi.flowfile.FlowFile;
    +import org.apache.nifi.processor.ProcessContext;
    +import org.apache.nifi.processor.ProcessSession;
    +import org.apache.nifi.processor.Relationship;
    +import org.apache.nifi.processor.ProcessorInitializationContext;
    +import org.apache.nifi.annotation.behavior.WritesAttribute;
    +import org.apache.nifi.annotation.behavior.WritesAttributes;
    +import org.apache.nifi.annotation.documentation.CapabilityDescription;
    +import org.apache.nifi.annotation.documentation.SeeAlso;
    +import org.apache.nifi.annotation.documentation.Tags;
    +import org.apache.nifi.processor.exception.ProcessException;
    +import org.apache.nifi.processor.io.StreamCallback;
    +import org.apache.nifi.processor.util.StandardValidators;
    +import org.jsoup.nodes.Document;
    +import org.jsoup.nodes.Element;
    +import org.jsoup.select.Elements;
    +
    +import java.io.IOException;
    +import java.io.InputStream;
    +import java.io.OutputStream;
    +import java.util.ArrayList;
    +import java.util.List;
    +import java.util.Set;
    +import java.util.HashSet;
    +import java.util.Collections;
    +
    +@Tags({"get", "html", "dom", "css", "element"})
    +@CapabilityDescription("Parses HTML input using CSS selector syntax and creates a new flowfile containing the extracted" +
    +        " element content for each matching CSS selector.")
    +@SeeAlso({ModifyHTMLElement.class, PutHTMLElement.class})
    +@WritesAttributes({@WritesAttribute(attribute="HTMLElement", description="Flowfile attribute where the element result" +
    +        " parsed from the HTML using the CSS selector syntax are placed if the destination is a flowfile attribute.")})
    +public class GetHTMLElement
    +        extends AbstractHTMLProcessor {
    +
    +    public static final String HTML_ELEMENT_ATTRIBUTE_NAME = "HTMLElement";
    +    public static final String DESTINATION_ATTRIBUTE = "flowfile-attribute";
    +    public static final String DESTINATION_CONTENT = "flowfile-content";
    +
    +    public static final PropertyDescriptor PREPEND_ELEMENT_VALUE = new PropertyDescriptor
    +            .Builder().name("Prepend Element value")
    +            .description("Prepends the specified value to the resulting Element")
    +            .required(false)
    +            .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
    +            .expressionLanguageSupported(true)
    +            .build();
    +
    +    public static final PropertyDescriptor APPEND_ELEMENT_VALUE = new PropertyDescriptor
    +            .Builder().name("Append Element value")
    +            .description("Appends the specified value to the resulting Element")
    +            .required(false)
    +            .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
    +            .expressionLanguageSupported(true)
    +            .build();
    +
    +    public static final PropertyDescriptor ATTRIBUTE_KEY = new PropertyDescriptor
    +            .Builder().name("Attribute Name")
    +            .description(("When getting the value of an element attribute this value is used as the key to determine" +
    +                    " which attribute on the selected element should be retrieved."))
    +            .required(false)
    +            .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
    +            .expressionLanguageSupported(true)
    +            .build();
    +
    +
    +    public static final PropertyDescriptor OUTPUT_TYPE = new PropertyDescriptor.Builder()
    +            .name("Output Type")
    +            .description("Controls the type of value that is retrieved from the element. " +
    +                    ELEMENT_HTML + "," + ELEMENT_TEXT + ", " + ELEMENT_ATTRIBUTE + " or " + ELEMENT_DATA)
    +            .required(true)
    +            .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
    +            .allowableValues(ELEMENT_HTML, ELEMENT_TEXT, ELEMENT_ATTRIBUTE, ELEMENT_DATA)
    +            .defaultValue(ELEMENT_HTML)
    +            .build();
    +
    +    public static final PropertyDescriptor DESTINATION = new PropertyDescriptor.Builder()
    +            .name("Destination")
    +            .description("Control if element extracted is written as a flowfile attribute or " +
    +                    "as flowfile content.")
    +            .required(true)
    +            .allowableValues(DESTINATION_ATTRIBUTE, DESTINATION_CONTENT)
    +            .defaultValue(DESTINATION_ATTRIBUTE)
    +            .build();
    +
    +    private List<PropertyDescriptor> descriptors;
    +
    +    private Set<Relationship> relationships;
    +
    +    @Override
    +    protected void init(final ProcessorInitializationContext context) {
    +        final List<PropertyDescriptor> descriptors = new ArrayList<>();
    +        descriptors.add(URL);
    +        descriptors.add(CSS_SELECTOR);
    +        descriptors.add(HTML_CHARSET);
    +        descriptors.add(OUTPUT_TYPE);
    +        descriptors.add(DESTINATION);
    +        descriptors.add(PREPEND_ELEMENT_VALUE);
    +        descriptors.add(APPEND_ELEMENT_VALUE);
    +        descriptors.add(ATTRIBUTE_KEY);
    +        this.descriptors = Collections.unmodifiableList(descriptors);
    +
    +        final Set<Relationship> relationships = new HashSet<>();
    +        relationships.add(REL_ORIGINAL);
    +        relationships.add(REL_SUCCESS);
    +        relationships.add(REL_FAILURE);
    +        relationships.add(REL_NOT_FOUND);
    +        this.relationships = Collections.unmodifiableSet(relationships);
    +    }
    +
    +    @Override
    +    public Set<Relationship> getRelationships() {
    +        return this.relationships;
    +    }
    +
    +    @Override
    +    public final List<PropertyDescriptor> getSupportedPropertyDescriptors() {
    +        return descriptors;
    +    }
    +
    +    @Override
    +    public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
    +        final FlowFile flowFile = session.get();
    +        if ( flowFile == null ) {
    +            return;
    +        }
    +
    +        try {
    +
    +            final Document doc = parseHTMLDocumentFromFlowfile(flowFile, context, session);
    +            final Elements eles = doc.select(context.getProperty(CSS_SELECTOR)
    +                    .evaluateAttributeExpressions().getValue());
    +            final String prependValue = context.getProperty(PREPEND_ELEMENT_VALUE)
    +                    .evaluateAttributeExpressions(flowFile).getValue();
    +            final String appendValue = context.getProperty(APPEND_ELEMENT_VALUE)
    +                    .evaluateAttributeExpressions(flowFile).getValue();
    +
    +            if (eles == null || eles.size() == 0) {
    +                //No element found
    +                session.transfer(flowFile, REL_NOT_FOUND);
    +            } else {
    +                for (final Element ele : eles) {
    +                    final FlowFile ff = session.create();
    +
    +                    switch (context.getProperty(DESTINATION).getValue()) {
    +                        case DESTINATION_ATTRIBUTE:
    +                            final FlowFile atFlowfile = session.putAttribute(ff, HTML_ELEMENT_ATTRIBUTE_NAME,
    +                                    extractElementValue(
    +                                            prependValue,
    +                                            context.getProperty(OUTPUT_TYPE).getValue(),
    +                                            appendValue,
    +                                            ele,
    +                                            context.getProperty(ATTRIBUTE_KEY).evaluateAttributeExpressions()
    +                                                    .getValue()));
    +                            session.getProvenanceReporter().create(atFlowfile);
    --- End diff --
    
    This should really be a FORK, not a CREATE, since we are deriving content from an input FlowFile. However, there's no need to emit a Provenance Event at all, since the session.create(flowFile) mentioned above will automatically generate the event for you.


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] nifi pull request: NIFI-1156: HTML Parsing Processors Bundle

Posted by markap14 <gi...@git.apache.org>.
Github user markap14 commented on a diff in the pull request:

    https://github.com/apache/nifi/pull/124#discussion_r49480961
  
    --- Diff: nifi-nar-bundles/nifi-html-bundle/nifi-html-processors/src/main/java/org/apache/nifi/PutHTMLElement.java ---
    @@ -0,0 +1,150 @@
    +/*
    + * Licensed to the Apache Software Foundation (ASF) under one or more
    + * contributor license agreements.  See the NOTICE file distributed with
    + * this work for additional information regarding copyright ownership.
    + * The ASF licenses this file to You under the Apache License, Version 2.0
    + * (the "License"); you may not use this file except in compliance with
    + * the License.  You may obtain a copy of the License at
    + *
    + *     http://www.apache.org/licenses/LICENSE-2.0
    + *
    + * Unless required by applicable law or agreed to in writing, software
    + * distributed under the License is distributed on an "AS IS" BASIS,
    + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    + * See the License for the specific language governing permissions and
    + * limitations under the License.
    + */
    +package org.apache.nifi;
    +
    +import org.apache.nifi.annotation.documentation.CapabilityDescription;
    +import org.apache.nifi.annotation.documentation.SeeAlso;
    +import org.apache.nifi.annotation.documentation.Tags;
    +import org.apache.nifi.components.PropertyDescriptor;
    +import org.apache.nifi.flowfile.FlowFile;
    +import org.apache.nifi.processor.ProcessContext;
    +import org.apache.nifi.processor.ProcessSession;
    +import org.apache.nifi.processor.Relationship;
    +import org.apache.nifi.processor.ProcessorInitializationContext;
    +import org.apache.nifi.processor.exception.ProcessException;
    +import org.apache.nifi.processor.io.StreamCallback;
    +import org.apache.nifi.processor.util.StandardValidators;
    +import org.jsoup.nodes.Document;
    +import org.jsoup.nodes.Element;
    +import org.jsoup.select.Elements;
    +
    +import java.io.IOException;
    +import java.io.InputStream;
    +import java.io.OutputStream;
    +import java.util.ArrayList;
    +import java.util.List;
    +import java.util.Set;
    +import java.util.HashSet;
    +import java.util.Collections;
    +
    +@Tags({"put", "html", "dom", "css", "element"})
    +@CapabilityDescription("Creates a new HTML element in the input HTML")
    +@SeeAlso({GetHTMLElement.class, ModifyHTMLElement.class})
    +public class PutHTMLElement extends AbstractHTMLProcessor {
    +
    +    public static final String APPEND_ELEMENT = "append-html";
    +    public static final String PREPEND_ELEMENT = "prepend-html";
    +
    +    public static final PropertyDescriptor PUT_LOCATION_TYPE = new PropertyDescriptor.Builder()
    +            .name("Element Insert Location Type")
    +            .description("Controls whether the new element is prepended or appended to the children of the " +
    +                    "Element located by the CSS selector. EX: prepended value '<b>Hi</b>' inside of " +
    +                    "Element (using CSS Selector 'p') '<p>There</p>' would result in " +
    +                    "'<p><b>Hi</b>There</p>'. Appending the value would result in '<p>There<b>Hi</b></p>'")
    +            .required(true)
    +            .allowableValues(APPEND_ELEMENT, PREPEND_ELEMENT)
    +            .defaultValue(APPEND_ELEMENT)
    +            .build();
    +
    +    public static final PropertyDescriptor PUT_VALUE = new PropertyDescriptor.Builder()
    +            .name("Put Value")
    +            .description("Value used when creating the new Element. Value should be a valid HTML element. " +
    +                    "The text should be supplied unencoded: characters like '<', '>', etc will be properly HTML " +
    +                    "encoded in the output.")
    +            .required(true)
    +            .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
    +            .expressionLanguageSupported(true)
    +            .build();
    +
    +    private List<PropertyDescriptor> descriptors;
    +
    +    private Set<Relationship> relationships;
    +
    +    @Override
    +    protected void init(final ProcessorInitializationContext context) {
    +        final List<PropertyDescriptor> descriptors = new ArrayList<PropertyDescriptor>();
    +        descriptors.add(URL);
    +        descriptors.add(CSS_SELECTOR);
    +        descriptors.add(HTML_CHARSET);
    +        descriptors.add(PUT_LOCATION_TYPE);
    +        descriptors.add(PUT_VALUE);
    +        this.descriptors = Collections.unmodifiableList(descriptors);
    +
    +        final Set<Relationship> relationships = new HashSet<Relationship>();
    +        relationships.add(REL_ORIGINAL);
    +        relationships.add(REL_SUCCESS);
    +        relationships.add(REL_FAILURE);
    +        relationships.add(REL_INVALID_HTML);
    +        this.relationships = Collections.unmodifiableSet(relationships);
    +    }
    +
    +    @Override
    +    public Set<Relationship> getRelationships() {
    +        return this.relationships;
    +    }
    +
    +    @Override
    +    public final List<PropertyDescriptor> getSupportedPropertyDescriptors() {
    +        return descriptors;
    +    }
    +
    +    @Override
    +    public void onTrigger(final ProcessContext context, final ProcessSession session) throws ProcessException {
    +        final FlowFile flowFile = session.get();
    +        if (flowFile == null) {
    +            return;
    +        }
    +
    +        try {
    +            final Document doc = parseHTMLDocumentFromFlowfile(flowFile, context, session);
    +            final Elements eles = doc.select(context.getProperty(CSS_SELECTOR).evaluateAttributeExpressions().getValue());
    +
    +            if (eles == null || eles.size() == 0) {
    +                //No element found
    +                session.transfer(flowFile, REL_NOT_FOUND);
    +            } else {
    +                for (Element ele : eles) {
    +                    switch (context.getProperty(PUT_LOCATION_TYPE).getValue()) {
    +                        case APPEND_ELEMENT:
    +                            ele.append(context.getProperty(PUT_VALUE).evaluateAttributeExpressions().getValue());
    +                            break;
    +                        case PREPEND_ELEMENT:
    +                            ele.prepend(context.getProperty(PUT_VALUE).evaluateAttributeExpressions().getValue());
    --- End diff --
    
    This should call evaluateAttributeExpressions(flowFile) so that we can take into account FlowFile Attributes.


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] nifi pull request: NIFI-1156: HTML Parsing Processors Bundle

Posted by markap14 <gi...@git.apache.org>.
Github user markap14 commented on a diff in the pull request:

    https://github.com/apache/nifi/pull/124#discussion_r49480617
  
    --- Diff: nifi-nar-bundles/nifi-html-bundle/nifi-html-processors/src/main/java/org/apache/nifi/AbstractHTMLProcessor.java ---
    @@ -0,0 +1,120 @@
    +/*
    + * Licensed to the Apache Software Foundation (ASF) under one or more
    + * contributor license agreements.  See the NOTICE file distributed with
    + * this work for additional information regarding copyright ownership.
    + * The ASF licenses this file to You under the Apache License, Version 2.0
    + * (the "License"); you may not use this file except in compliance with
    + * the License.  You may obtain a copy of the License at
    + *
    + *     http://www.apache.org/licenses/LICENSE-2.0
    + *
    + * Unless required by applicable law or agreed to in writing, software
    + * distributed under the License is distributed on an "AS IS" BASIS,
    + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    + * See the License for the specific language governing permissions and
    + * limitations under the License.
    + */
    +package org.apache.nifi;
    +
    +import org.apache.nifi.components.PropertyDescriptor;
    +import org.apache.nifi.flowfile.FlowFile;
    +import org.apache.nifi.processor.AbstractProcessor;
    +import org.apache.nifi.processor.ProcessContext;
    +import org.apache.nifi.processor.ProcessSession;
    +import org.apache.nifi.processor.Relationship;
    +import org.apache.nifi.processor.io.InputStreamCallback;
    +import org.apache.nifi.processor.util.StandardValidators;
    +import org.jsoup.Jsoup;
    +import org.jsoup.nodes.Document;
    +
    +import java.io.IOException;
    +import java.io.InputStream;
    +import java.util.concurrent.atomic.AtomicReference;
    +
    +public abstract class AbstractHTMLProcessor extends AbstractProcessor {
    +
    +    protected static final String ELEMENT_HTML = "HTML";
    +    protected static final String ELEMENT_TEXT = "Text";
    +    protected static final String ELEMENT_DATA = "Data";
    +    protected static final String ELEMENT_ATTRIBUTE = "Attribute";
    +
    +    public static final PropertyDescriptor URL = new PropertyDescriptor
    +            .Builder().name("URL")
    +            .description("Base URL for the HTML page being parsed.")
    +            .required(true)
    +            .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
    +            .build();
    +
    +    public static final PropertyDescriptor CSS_SELECTOR = new PropertyDescriptor
    +            .Builder().name("CSS Selector")
    +            .description("CSS selector syntax string used to extract the desired HTML element(s).")
    +            .required(true)
    +            .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
    --- End diff --
    
    Should consider a custom validator here, that verifies that the CSS Selector can be parsed appropriately by the library. This way, if the user enters something invalid, it is shown as a validation problem instead of letting them run the processor and find out after it runs that it's not right.


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] nifi pull request: NIFI-1156: HTML Parsing Processors Bundle

Posted by markap14 <gi...@git.apache.org>.
Github user markap14 commented on a diff in the pull request:

    https://github.com/apache/nifi/pull/124#discussion_r49478637
  
    --- Diff: nifi-nar-bundles/nifi-html-bundle/nifi-html-processors/src/main/java/org/apache/nifi/AbstractHTMLProcessor.java ---
    @@ -0,0 +1,120 @@
    +/*
    + * Licensed to the Apache Software Foundation (ASF) under one or more
    + * contributor license agreements.  See the NOTICE file distributed with
    + * this work for additional information regarding copyright ownership.
    + * The ASF licenses this file to You under the Apache License, Version 2.0
    + * (the "License"); you may not use this file except in compliance with
    + * the License.  You may obtain a copy of the License at
    + *
    + *     http://www.apache.org/licenses/LICENSE-2.0
    + *
    + * Unless required by applicable law or agreed to in writing, software
    + * distributed under the License is distributed on an "AS IS" BASIS,
    + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    + * See the License for the specific language governing permissions and
    + * limitations under the License.
    + */
    +package org.apache.nifi;
    +
    +import org.apache.nifi.components.PropertyDescriptor;
    +import org.apache.nifi.flowfile.FlowFile;
    +import org.apache.nifi.processor.AbstractProcessor;
    +import org.apache.nifi.processor.ProcessContext;
    +import org.apache.nifi.processor.ProcessSession;
    +import org.apache.nifi.processor.Relationship;
    +import org.apache.nifi.processor.io.InputStreamCallback;
    +import org.apache.nifi.processor.util.StandardValidators;
    +import org.jsoup.Jsoup;
    +import org.jsoup.nodes.Document;
    +
    +import java.io.IOException;
    +import java.io.InputStream;
    +import java.util.concurrent.atomic.AtomicReference;
    +
    +public abstract class AbstractHTMLProcessor extends AbstractProcessor {
    +
    +    protected static final String ELEMENT_HTML = "HTML";
    +    protected static final String ELEMENT_TEXT = "Text";
    +    protected static final String ELEMENT_DATA = "Data";
    +    protected static final String ELEMENT_ATTRIBUTE = "Attribute";
    +
    +    public static final PropertyDescriptor URL = new PropertyDescriptor
    +            .Builder().name("URL")
    +            .description("Base URL for the HTML page being parsed.")
    +            .required(true)
    +            .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
    +            .build();
    +
    +    public static final PropertyDescriptor CSS_SELECTOR = new PropertyDescriptor
    +            .Builder().name("CSS Selector")
    +            .description("CSS selector syntax string used to extract the desired HTML element(s).")
    +            .required(true)
    +            .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
    +            .expressionLanguageSupported(true)
    +            .build();
    +
    +    public static final PropertyDescriptor HTML_CHARSET = new PropertyDescriptor
    +            .Builder().name("HTML character encoding")
    +            .description("Character encoding of the input HTML")
    +            .defaultValue("UTF-8")
    +            .required(true)
    +            .addValidator(StandardValidators.NON_EMPTY_VALIDATOR)
    --- End diff --
    
    Should use StandardValidators.CHARACTER_SET_VALIDATOR


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] nifi pull request: NIFI-1156: HTML Parsing Processors Bundle

Posted by olegz <gi...@git.apache.org>.
Github user olegz commented on the pull request:

    https://github.com/apache/nifi/pull/124#issuecomment-157221978
  
    General question primarily for NiFi leadership team. JSoup MIT License - http://jsoup.org/license
    Are there any conflicts?


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] nifi pull request: NIFI-1156: HTML Parsing Processors Bundle

Posted by joewitt <gi...@git.apache.org>.
Github user joewitt commented on the pull request:

    https://github.com/apache/nifi/pull/124#issuecomment-168541446
  
    added more comments on the NIFI-1156 ticket.


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] nifi pull request: NIFI-1156: HTML Parsing Processors Bundle

Posted by markap14 <gi...@git.apache.org>.
Github user markap14 commented on a diff in the pull request:

    https://github.com/apache/nifi/pull/124#discussion_r49482019
  
    --- Diff: nifi-nar-bundles/nifi-html-bundle/nifi-html-processors/src/main/java/org/apache/nifi/ModifyHTMLElement.java ---
    @@ -0,0 +1,164 @@
    +/*
    + * Licensed to the Apache Software Foundation (ASF) under one or more
    + * contributor license agreements.  See the NOTICE file distributed with
    + * this work for additional information regarding copyright ownership.
    + * The ASF licenses this file to You under the Apache License, Version 2.0
    + * (the "License"); you may not use this file except in compliance with
    + * the License.  You may obtain a copy of the License at
    + *
    + *     http://www.apache.org/licenses/LICENSE-2.0
    + *
    + * Unless required by applicable law or agreed to in writing, software
    + * distributed under the License is distributed on an "AS IS" BASIS,
    + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    + * See the License for the specific language governing permissions and
    + * limitations under the License.
    + */
    +package org.apache.nifi;
    +
    +import org.apache.nifi.annotation.behavior.WritesAttribute;
    +import org.apache.nifi.annotation.behavior.WritesAttributes;
    +import org.apache.nifi.annotation.documentation.CapabilityDescription;
    +import org.apache.nifi.annotation.documentation.SeeAlso;
    +import org.apache.nifi.annotation.documentation.Tags;
    +import org.apache.nifi.components.PropertyDescriptor;
    +import org.apache.nifi.flowfile.FlowFile;
    +import org.apache.nifi.processor.ProcessContext;
    +import org.apache.nifi.processor.ProcessSession;
    +import org.apache.nifi.processor.Relationship;
    +import org.apache.nifi.processor.ProcessorInitializationContext;
    +import org.apache.nifi.processor.exception.ProcessException;
    +import org.apache.nifi.processor.io.StreamCallback;
    +import org.apache.nifi.processor.util.StandardValidators;
    +import org.jsoup.nodes.Document;
    +import org.jsoup.nodes.Element;
    +import org.jsoup.select.Elements;
    +
    +import java.io.IOException;
    +import java.io.InputStream;
    +import java.io.OutputStream;
    +import java.util.List;
    +import java.util.ArrayList;
    +import java.util.Set;
    +import java.util.HashSet;
    +import java.util.Collections;
    +
    +@Tags({"modify", "html", "dom", "css", "element"})
    +@CapabilityDescription("Modifies the value of an existing HTML element in the original input HTML")
    +@SeeAlso({GetHTMLElement.class, PutHTMLElement.class})
    +@WritesAttributes({@WritesAttribute(attribute="NumElementsModified", description="Total number of HTML " +
    --- End diff --
    
    Should also contain the @SupportsBatching annotation and @InputRequirement annotations.


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---

[GitHub] nifi pull request: NIFI-1156: HTML Parsing Processors Bundle

Posted by markap14 <gi...@git.apache.org>.
Github user markap14 commented on a diff in the pull request:

    https://github.com/apache/nifi/pull/124#discussion_r49482412
  
    --- Diff: nifi-nar-bundles/nifi-html-bundle/nifi-html-processors/src/main/java/org/apache/nifi/PutHTMLElement.java ---
    @@ -0,0 +1,150 @@
    +/*
    + * Licensed to the Apache Software Foundation (ASF) under one or more
    + * contributor license agreements.  See the NOTICE file distributed with
    + * this work for additional information regarding copyright ownership.
    + * The ASF licenses this file to You under the Apache License, Version 2.0
    + * (the "License"); you may not use this file except in compliance with
    + * the License.  You may obtain a copy of the License at
    + *
    + *     http://www.apache.org/licenses/LICENSE-2.0
    + *
    + * Unless required by applicable law or agreed to in writing, software
    + * distributed under the License is distributed on an "AS IS" BASIS,
    + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
    + * See the License for the specific language governing permissions and
    + * limitations under the License.
    + */
    +package org.apache.nifi;
    +
    +import org.apache.nifi.annotation.documentation.CapabilityDescription;
    +import org.apache.nifi.annotation.documentation.SeeAlso;
    +import org.apache.nifi.annotation.documentation.Tags;
    +import org.apache.nifi.components.PropertyDescriptor;
    +import org.apache.nifi.flowfile.FlowFile;
    +import org.apache.nifi.processor.ProcessContext;
    +import org.apache.nifi.processor.ProcessSession;
    +import org.apache.nifi.processor.Relationship;
    +import org.apache.nifi.processor.ProcessorInitializationContext;
    +import org.apache.nifi.processor.exception.ProcessException;
    +import org.apache.nifi.processor.io.StreamCallback;
    +import org.apache.nifi.processor.util.StandardValidators;
    +import org.jsoup.nodes.Document;
    +import org.jsoup.nodes.Element;
    +import org.jsoup.select.Elements;
    +
    +import java.io.IOException;
    +import java.io.InputStream;
    +import java.io.OutputStream;
    +import java.util.ArrayList;
    +import java.util.List;
    +import java.util.Set;
    +import java.util.HashSet;
    +import java.util.Collections;
    +
    +@Tags({"put", "html", "dom", "css", "element"})
    +@CapabilityDescription("Creates a new HTML element in the input HTML")
    +@SeeAlso({GetHTMLElement.class, ModifyHTMLElement.class})
    --- End diff --
    
    Should also have a @SupportsBatching annotation and an @InputRequirement(Requirement.INPUT_REQUIRED)


---
If your project is set up for it, you can reply to this email and have your
reply appear on GitHub as well. If your project does not have this feature
enabled and wishes so, or if the feature is enabled but not working, please
contact infrastructure at infrastructure@apache.org or file a JIRA ticket
with INFRA.
---