You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@sling.apache.org by np...@apache.org on 2017/09/12 09:32:00 UTC
svn commit: r1808091 - in /sling/trunk/contrib/extensions/sling-pipes: ./
src/main/java/org/apache/sling/pipes/
src/main/java/org/apache/sling/pipes/internal/
src/test/java/org/apache/sling/pipes/internal/ src/test/resources/
Author: npeltier
Date: Tue Sep 12 09:32:00 2017
New Revision: 1808091
URL: http://svn.apache.org/viewvc?rev=1808091&view=rev
Log:
SLING-7099 introducing csv pipe
- adding also unit test (using repository based input stream for now),
- adding PipeBuilder api, and fixing javadoc issues
Added:
sling/trunk/contrib/extensions/sling-pipes/src/main/java/org/apache/sling/pipes/internal/CsvPipe.java
sling/trunk/contrib/extensions/sling-pipes/src/test/java/org/apache/sling/pipes/internal/CsvPipeTest.java
sling/trunk/contrib/extensions/sling-pipes/src/test/resources/standardTest.csv
Modified:
sling/trunk/contrib/extensions/sling-pipes/pom.xml
sling/trunk/contrib/extensions/sling-pipes/src/main/java/org/apache/sling/pipes/AbstractInputStreamPipe.java
sling/trunk/contrib/extensions/sling-pipes/src/main/java/org/apache/sling/pipes/PipeBuilder.java
sling/trunk/contrib/extensions/sling-pipes/src/main/java/org/apache/sling/pipes/internal/PipeBuilderImpl.java
sling/trunk/contrib/extensions/sling-pipes/src/main/java/org/apache/sling/pipes/internal/PlumberImpl.java
sling/trunk/contrib/extensions/sling-pipes/src/main/java/org/apache/sling/pipes/internal/PlumberServlet.java
Modified: sling/trunk/contrib/extensions/sling-pipes/pom.xml
URL: http://svn.apache.org/viewvc/sling/trunk/contrib/extensions/sling-pipes/pom.xml?rev=1808091&r1=1808090&r2=1808091&view=diff
==============================================================================
--- sling/trunk/contrib/extensions/sling-pipes/pom.xml (original)
+++ sling/trunk/contrib/extensions/sling-pipes/pom.xml Tue Sep 12 09:32:00 2017
@@ -96,6 +96,16 @@
</execution>
</executions>
</plugin>
+ <plugin>
+ <groupId>org.apache.rat</groupId>
+ <artifactId>apache-rat-plugin</artifactId>
+ <configuration>
+ <excludes>
+ <!-- test csv files can't have licenses embedded -->
+ <exclude>src/test/resources/**/*.csv</exclude>
+ </excludes>
+ </configuration>
+ </plugin>
</plugins>
</build>
Modified: sling/trunk/contrib/extensions/sling-pipes/src/main/java/org/apache/sling/pipes/AbstractInputStreamPipe.java
URL: http://svn.apache.org/viewvc/sling/trunk/contrib/extensions/sling-pipes/src/main/java/org/apache/sling/pipes/AbstractInputStreamPipe.java?rev=1808091&r1=1808090&r2=1808091&view=diff
==============================================================================
--- sling/trunk/contrib/extensions/sling-pipes/src/main/java/org/apache/sling/pipes/AbstractInputStreamPipe.java (original)
+++ sling/trunk/contrib/extensions/sling-pipes/src/main/java/org/apache/sling/pipes/AbstractInputStreamPipe.java Tue Sep 12 09:32:00 2017
@@ -30,8 +30,6 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.io.ByteArrayInputStream;
-import java.io.File;
-import java.io.FileInputStream;
import java.io.IOException;
import java.io.InputStream;
import java.nio.charset.StandardCharsets;
@@ -40,13 +38,14 @@ import java.util.regex.Pattern;
/**
* Input Stream based pipe, coming from web, from request, resource tree, web
+ * binding is updated by the returned iterator
*/
public abstract class AbstractInputStreamPipe extends BasePipe {
private static Logger LOGGER = LoggerFactory.getLogger(AbstractInputStreamPipe.class);
public final String REMOTE_START = "http";
- protected final Pattern VALID_PATH = Pattern.compile("/([\\w\\d]+/)+[\\w\\d]+");
+ protected final Pattern VALID_PATH = Pattern.compile("/([\\w\\d\\.-_]+/)+[\\w\\d\\.-_]+");
public static final Object BINDING_IS = "org.apache.sling.pipes.RequestInputStream";
@@ -92,15 +91,14 @@ public abstract class AbstractInputStrea
}
} else if (VALID_PATH.matcher(expr).find()
&& resolver.getResource(expr) != null
- && resolver.getResource(expr).adaptTo(File.class) != null) {
- return new FileInputStream(resolver.getResource(expr).adaptTo(File.class));
+ && resolver.getResource(expr).adaptTo(InputStream.class) != null) {
+ return resolver.getResource(expr).adaptTo(InputStream.class);
} else if (getBindings().getBindings().get(BINDING_IS) != null){
return (InputStream)getBindings().getBindings().get(BINDING_IS);
}
return new ByteArrayInputStream(expr.getBytes(StandardCharsets.UTF_8));
}
-
@Override
public Object getOutputBinding() {
return binding;
Modified: sling/trunk/contrib/extensions/sling-pipes/src/main/java/org/apache/sling/pipes/PipeBuilder.java
URL: http://svn.apache.org/viewvc/sling/trunk/contrib/extensions/sling-pipes/src/main/java/org/apache/sling/pipes/PipeBuilder.java?rev=1808091&r1=1808090&r2=1808091&view=diff
==============================================================================
--- sling/trunk/contrib/extensions/sling-pipes/src/main/java/org/apache/sling/pipes/PipeBuilder.java (original)
+++ sling/trunk/contrib/extensions/sling-pipes/src/main/java/org/apache/sling/pipes/PipeBuilder.java Tue Sep 12 09:32:00 2017
@@ -87,8 +87,15 @@ public interface PipeBuilder {
PipeBuilder rm();
/**
+ * attach a csv pipe to the current context
+ * @param expr csv expr or URL or path in the resource tree
+ * @return updated instance of PipeBuilder
+ */
+ PipeBuilder csv(String expr);
+
+ /**
* attach a json pipe to the current context
- * @param expr json expr or URL
+ * @param expr json expr or URL or path in the resource tree
* @return updated instance of PipeBuilder
*/
PipeBuilder json(String expr);
@@ -124,8 +131,9 @@ public interface PipeBuilder {
* attach a reference pipe to the current context
* @param expr reference
* @return updated instance of PipeBuilder
+ * @throws IllegalAccessException in case it's called with wrong # of arguments
*/
- PipeBuilder ref(String expr) throws IllegalAccessException;
+ PipeBuilder ref(String expr)throws IllegalAccessException;
/**
* parameterized current pipe in the context
Added: sling/trunk/contrib/extensions/sling-pipes/src/main/java/org/apache/sling/pipes/internal/CsvPipe.java
URL: http://svn.apache.org/viewvc/sling/trunk/contrib/extensions/sling-pipes/src/main/java/org/apache/sling/pipes/internal/CsvPipe.java?rev=1808091&view=auto
==============================================================================
--- sling/trunk/contrib/extensions/sling-pipes/src/main/java/org/apache/sling/pipes/internal/CsvPipe.java (added)
+++ sling/trunk/contrib/extensions/sling-pipes/src/main/java/org/apache/sling/pipes/internal/CsvPipe.java Tue Sep 12 09:32:00 2017
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sling.pipes.internal;
+
+import org.apache.commons.lang3.StringUtils;
+import org.apache.sling.api.resource.Resource;
+import org.apache.sling.pipes.AbstractInputStreamPipe;
+import org.apache.sling.pipes.Plumber;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import java.io.BufferedReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.nio.charset.StandardCharsets;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+
+/**
+ * Csv input stream pipe, similar at what
+ */
+public class CsvPipe extends AbstractInputStreamPipe {
+ private static Logger logger = LoggerFactory.getLogger(JsonPipe.class);
+ public static final String RESOURCE_TYPE = RT_PREFIX + "csv";
+
+
+ protected static final String PN_SEPARATOR = "separator";
+
+ protected static final String DEFAULT_SEPARATOR = ",";
+
+ BufferedReader reader;
+
+ String nextLine = null;
+
+ int index = 0;
+
+ public CsvPipe(Plumber plumber, Resource resource) throws Exception {
+ super(plumber, resource);
+ }
+
+ @Override
+ public Iterator<Resource> getOutput(InputStream inputStream) {
+ Iterator<Resource> output = EMPTY_ITERATOR;
+ String separator = properties.get(PN_SEPARATOR, DEFAULT_SEPARATOR);
+ try {
+ reader = new BufferedReader(new InputStreamReader(inputStream, StandardCharsets.UTF_8));
+ String headersLine = reader.readLine();
+ final String[] headers = headersLine.split(separator);
+ if (headers.length > 0){
+ nextLine = reader.readLine();
+ output = new Iterator<Resource>() {
+ @Override
+ public boolean hasNext() {
+ return StringUtils.isNotBlank(nextLine);
+ }
+ @Override
+ public Resource next() {
+ try {
+ String[] values = nextLine.split(separator);
+ if (values.length < headers.length){
+ throw new IllegalArgumentException("wrong format line " + index + " should have at least the same number of columns than the headers");
+ }
+ Map<String, String> map = new HashMap<>();
+ for (int i = 0; i < headers.length; i ++){
+ map.put(headers[i], values[i]);
+ }
+ binding = map;
+ nextLine = reader.readLine();
+ } catch (Exception e) {
+ logger.error("Unable to retrieve {}nth line of csv file", index, e);
+ nextLine = null;
+ }
+ return getInput();
+ }
+ };
+ }
+ } catch (IllegalArgumentException iae){
+ logger.error("unable to correctly process csv file", iae);
+ } catch (IOException e){
+ logger.error("unable to process csv file", e);
+ }
+ return output;
+ }
+}
\ No newline at end of file
Modified: sling/trunk/contrib/extensions/sling-pipes/src/main/java/org/apache/sling/pipes/internal/PipeBuilderImpl.java
URL: http://svn.apache.org/viewvc/sling/trunk/contrib/extensions/sling-pipes/src/main/java/org/apache/sling/pipes/internal/PipeBuilderImpl.java?rev=1808091&r1=1808090&r2=1808091&view=diff
==============================================================================
--- sling/trunk/contrib/extensions/sling-pipes/src/main/java/org/apache/sling/pipes/internal/PipeBuilderImpl.java (original)
+++ sling/trunk/contrib/extensions/sling-pipes/src/main/java/org/apache/sling/pipes/internal/PipeBuilderImpl.java Tue Sep 12 09:32:00 2017
@@ -130,6 +130,12 @@ public class PipeBuilderImpl implements
return pipe(TraversePipe.RESOURCE_TYPE);
}
+
+ @Override
+ public PipeBuilder csv(String expr) {
+ return pipeWithExpr(CsvPipe.RESOURCE_TYPE, expr);
+ }
+
@Override
public PipeBuilder json(String expr) {
return pipeWithExpr(JsonPipe.RESOURCE_TYPE, expr);
@@ -172,8 +178,8 @@ public class PipeBuilderImpl implements
/**
* Checks arguments and throws exception if there is an issue
- * @param params
- * @throws IllegalArgumentException
+ * @param params arguments to check
+ * @throws IllegalArgumentException exception thrown in case arguments are wrong
*/
protected void checkArguments(Object... params) throws IllegalArgumentException {
if (params.length % 2 > 0){
@@ -183,8 +189,8 @@ public class PipeBuilderImpl implements
/**
* write key/value pairs into a map
- * @param map
- * @param params
+ * @param map target map
+ * @param params key/value pairs to write into the map
*/
protected void writeToMap(Map map, Object... params){
for (int i = 0; i < params.length; i += 2){
@@ -245,6 +251,7 @@ public class PipeBuilderImpl implements
* @param type type of the node to be created
* @param data map of properties to add
* @throws PersistenceException in case configuration resource couldn't be persisted
+ * @return resource created
*/
protected Resource createResource(ResourceResolver resolver, String path, String type, Map data) throws PersistenceException {
return ResourceUtil.getOrCreateResource(resolver, path, data, type, false);
@@ -257,10 +264,11 @@ public class PipeBuilderImpl implements
/**
* Persist a step at a given path
- * @param path
- * @param step
+ * @param path path into which step should be persisted
+ * @param parentType type of the parent resource
+ * @param step step to persist
* @return created resource
- * @throws PersistenceException
+ * @throws PersistenceException in case persistence fails
*/
protected Resource persistStep(String path, String parentType, Step step) throws PersistenceException {
Resource resource = createResource(resolver, path, parentType, step.properties);
Modified: sling/trunk/contrib/extensions/sling-pipes/src/main/java/org/apache/sling/pipes/internal/PlumberImpl.java
URL: http://svn.apache.org/viewvc/sling/trunk/contrib/extensions/sling-pipes/src/main/java/org/apache/sling/pipes/internal/PlumberImpl.java?rev=1808091&r1=1808090&r2=1808091&view=diff
==============================================================================
--- sling/trunk/contrib/extensions/sling-pipes/src/main/java/org/apache/sling/pipes/internal/PlumberImpl.java (original)
+++ sling/trunk/contrib/extensions/sling-pipes/src/main/java/org/apache/sling/pipes/internal/PlumberImpl.java Tue Sep 12 09:32:00 2017
@@ -119,6 +119,7 @@ public class PlumberImpl implements Plum
registerPipe(FilterPipe.RESOURCE_TYPE, FilterPipe.class);
registerPipe(NotPipe.RESOURCE_TYPE, NotPipe.class);
registerPipe(TraversePipe.RESOURCE_TYPE, TraversePipe.class);
+ registerPipe(CsvPipe.RESOURCE_TYPE, CsvPipe.class);
}
@Reference(policy= ReferencePolicy.DYNAMIC, cardinality= ReferenceCardinality.OPTIONAL)
Modified: sling/trunk/contrib/extensions/sling-pipes/src/main/java/org/apache/sling/pipes/internal/PlumberServlet.java
URL: http://svn.apache.org/viewvc/sling/trunk/contrib/extensions/sling-pipes/src/main/java/org/apache/sling/pipes/internal/PlumberServlet.java?rev=1808091&r1=1808090&r2=1808091&view=diff
==============================================================================
--- sling/trunk/contrib/extensions/sling-pipes/src/main/java/org/apache/sling/pipes/internal/PlumberServlet.java (original)
+++ sling/trunk/contrib/extensions/sling-pipes/src/main/java/org/apache/sling/pipes/internal/PlumberServlet.java Tue Sep 12 09:32:00 2017
@@ -139,7 +139,7 @@ public class PlumberServlet extends Slin
}
}
- if (request.getParameterMap().containsValue(PARAM_FILE)){
+ if (request.getRequestParameterMap() != null && request.getRequestParameterMap().containsKey(PARAM_FILE)){
bindings.put(AbstractInputStreamPipe.BINDING_IS, request.getRequestParameter(PARAM_FILE).getInputStream());
}
Added: sling/trunk/contrib/extensions/sling-pipes/src/test/java/org/apache/sling/pipes/internal/CsvPipeTest.java
URL: http://svn.apache.org/viewvc/sling/trunk/contrib/extensions/sling-pipes/src/test/java/org/apache/sling/pipes/internal/CsvPipeTest.java?rev=1808091&view=auto
==============================================================================
--- sling/trunk/contrib/extensions/sling-pipes/src/test/java/org/apache/sling/pipes/internal/CsvPipeTest.java (added)
+++ sling/trunk/contrib/extensions/sling-pipes/src/test/java/org/apache/sling/pipes/internal/CsvPipeTest.java Tue Sep 12 09:32:00 2017
@@ -0,0 +1,51 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.sling.pipes.internal;
+
+import org.apache.commons.collections.IteratorUtils;
+import org.apache.sling.api.resource.Resource;
+import org.apache.sling.pipes.AbstractPipeTest;
+import org.apache.sling.pipes.Pipe;
+import org.junit.Test;
+
+import java.util.Iterator;
+import java.util.List;
+import java.util.stream.Collectors;
+
+import static org.junit.Assert.*;
+
+/**
+ * testing csv pipe
+ */
+public class CsvPipeTest extends AbstractPipeTest{
+
+ @Test
+ public void getOutput() throws Exception {
+ String csvPath = "/content/test/standardTest.csv";
+ context.load().binaryFile("/standardTest.csv", csvPath);
+ Pipe pipe = plumber.newPipe(context.resourceResolver())
+ .csv(csvPath).name("csv")
+ .mkdir(PATH_FRUITS + "/csv/${csv.fruit}-${csv.color}-${csv.id}").build();
+ Iterator<Resource> output = pipe.getOutput();
+ List<Resource> resources = IteratorUtils.toList(output);
+ List<String> paths = resources.stream().map( resource -> resource.getPath()).collect(Collectors.toList());
+ assertEquals("there should be 3 elements", 3, paths.size());
+ assertEquals("first should be /content/fruits/csv/apple-green-1", "/content/fruits/csv/apple-green-1", paths.get(0));
+ assertEquals("second should be /content/fruits/csv/banana-yellow-2", "/content/fruits/csv/banana-yellow-2", paths.get(1));
+ assertEquals("first should be /content/fruits/csv/plum-purple-3", "/content/fruits/csv/plum-purple-3", paths.get(2));
+ }
+}
\ No newline at end of file
Added: sling/trunk/contrib/extensions/sling-pipes/src/test/resources/standardTest.csv
URL: http://svn.apache.org/viewvc/sling/trunk/contrib/extensions/sling-pipes/src/test/resources/standardTest.csv?rev=1808091&view=auto
==============================================================================
--- sling/trunk/contrib/extensions/sling-pipes/src/test/resources/standardTest.csv (added)
+++ sling/trunk/contrib/extensions/sling-pipes/src/test/resources/standardTest.csv Tue Sep 12 09:32:00 2017
@@ -0,0 +1,4 @@
+fruit,color,id
+apple,green,1
+banana,yellow,2
+plum,purple,3
\ No newline at end of file