You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@freemarker.apache.org by sg...@apache.org on 2020/06/14 20:51:52 UTC
[freemarker-generator] 03/04: FREEMARKER-144 Proof Of Concept for
providing DataFrames
This is an automated email from the ASF dual-hosted git repository.
sgoeschl pushed a commit to branch FREEMARKER-144
in repository https://gitbox.apache.org/repos/asf/freemarker-generator.git
commit eafb4eb02a2df438466ca50b22bfa9817234b02f
Author: Siegfried Goeschl <si...@gmail.com>
AuthorDate: Sun Jun 14 22:11:19 2020 +0200
FREEMARKER-144 Proof Of Concept for providing DataFrames
---
.../freemarker/generator/base/table/Table.java | 259 ++++++++++++++-------
.../freemarker/generator/base/util/ArrayUtils.java | 72 ------
.../freemarker/generator/base/util/ListUtils.java | 76 ++++++
.../freemarker/generator/base/util/MapBuilder.java | 26 ++-
.../freemarker/generator/table/TableTest.java | 118 ++++++++--
.../templates/dataframe/example.ftl | 6 +-
.../generator/tools/dataframe/DataFrameTool.java | 30 ++-
.../CSVConverter.java} | 4 +-
.../tools/dataframe/converter/ConverterUtils.java | 106 +++++++++
.../tools/dataframe/converter/ListConverter.java | 21 ++
.../tools/dataframe/converter/MapConverter.java | 39 ++++
.../tools/dataframe/impl/MapConverter.java | 85 -------
.../generator/tools/excel/ExcelTool.java | 8 +-
.../tools/dataframe/DataFrameToolTest.java | 27 ++-
.../generator/tools/excel/ExcelToolTest.java | 8 +-
15 files changed, 598 insertions(+), 287 deletions(-)
diff --git a/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/table/Table.java b/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/table/Table.java
index 7530844..aaba268 100644
--- a/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/table/Table.java
+++ b/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/table/Table.java
@@ -1,167 +1,258 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package org.apache.freemarker.generator.base.table;
-import org.apache.freemarker.generator.base.util.ArrayUtils;
-import org.apache.freemarker.generator.base.util.Validate;
+import org.apache.freemarker.generator.base.util.ListUtils;
-import java.util.Arrays;
+import java.util.ArrayList;
import java.util.Collection;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import java.util.stream.Collectors;
-import java.util.stream.IntStream;
import static java.util.Objects.requireNonNull;
/**
- * Simple table model filled from a Map.
+ * Simple table model filled from maps or lists representing tabular data.
*/
public class Table {
- private final String[] columnNames;
- private final Class<?>[] columnTypes;
- private final Object[][] values;
+
+ /** List of column names */
+ private final List<String> columnNames;
+
+ /** Column types derived from tabular data */
+ private final List<Class<?>> columnTypes;
+
+ /** Table data as rows */
+ private final List<List<Object>> values;
+
+ /** Map column names to numeric column values */
private final Map<String, Integer> columnMap;
- private Table(String[] columnNames, Class<?>[] columnTypes, Object[][] columnValuesList) {
- this.columnNames = requireNonNull(columnNames);
- this.columnTypes = requireNonNull(columnTypes);
- this.values = ArrayUtils.transpose(requireNonNull(columnValuesList));
+ private Table() {
+ this.columnNames = new ArrayList<>();
+ this.columnTypes = new ArrayList<>();
+ this.values = new ArrayList<>();
+ this.columnMap = new HashMap<>();
+ }
+
+ private Table(Collection<String> columnNames, Collection<Class<?>> columnTypes, List<List<Object>> columnValuesList) {
+ this.columnNames = new ArrayList<>(requireNonNull(columnNames));
+ this.columnTypes = new ArrayList<>(requireNonNull(columnTypes));
+ this.values = ListUtils.transpose(requireNonNull(columnValuesList));
this.columnMap = new HashMap<>();
- for (int i = 0; i < this.columnNames.length; i++) {
- this.columnMap.put(this.columnNames[i], i);
+ for (int i = 0; i < this.columnNames.size(); i++) {
+ this.columnMap.put(this.columnNames.get(i), i);
}
}
- public String[] getColumnNames() {
+ public List<String> getColumnNames() {
return columnNames;
}
- public Class<?>[] getColumnTypes() {
+ public List<Class<?>> getColumnTypes() {
return columnTypes;
}
+ public List<List<Object>> getValues() {
+ return values;
+ }
+
public int getNrOfColumns() {
- return columnNames.length;
+ return columnNames.isEmpty() ? (values.isEmpty() ? 0 : values.get(0).size()) : columnNames.size();
+ }
+
+ public int size() {
+ return values.size();
+ }
+
+ public boolean isEmpty() {
+ return values.isEmpty();
+ }
+
+ public List<Object> getRow(int row) {
+ return values.get(row);
}
- public int getNrOfRows() {
- return values.length;
+ public Object get(int row, int column) {
+ return values.get(row).get(column);
}
- public Object[] getRowValues(int row) {
- return values[row];
+ public Object get(int row, String column) {
+ return values.get(row).get(columnMap.get(column));
}
- public Row getRow(int row) {
- return new Row(columnMap, getRowValues(row));
+ public boolean hasColumnHeaderRow() {
+ return !columnNames.isEmpty();
}
+ /**
+ * Create a table from a list of maps. Non-tabular data is supported,
+ * i.e. not all maps contains all possible keys.
+ *
+ * @param maps list of maps
+ * @return table
+ */
public static Table fromMaps(List<Map<String, Object>> maps) {
- Validate.notNull(maps, "list is null");
+ if (maps == null || maps.isEmpty()) {
+ return new Table();
+ }
final List<String> columnNames = columnNames(maps);
- final Object[][] columnValuesList = columnValuesList(maps, columnNames);
+ final List<List<Object>> columnValuesList = columnValuesList(maps, columnNames);
final List<Class<?>> columnTypes = columnTypes(columnValuesList);
return new Table(
- columnNames.toArray(new String[0]),
- columnTypes.toArray(new Class[0]),
+ columnNames,
+ columnTypes,
columnValuesList);
}
- public static Table fromLists(List<List<Object>> list) {
- Validate.notNull(list, "list is null");
+ /**
+ * Create a table from a list of lists representing tabular data.
+ *
+ * @param lists row values as lists
+ * @return table
+ */
+ public static Table fromLists(List<List<Object>> lists) {
+ requireNonNull(lists, "lists is null");
- final List<String> columnNames = Arrays.asList(ArrayUtils.copy(list.toArray(new Object[0])));
- final Object[][] columnValuesList = columnValuesList(list.subList(1, list.size()));
+ final List<List<Object>> columnValuesList = ListUtils.transpose(lists);
final List<Class<?>> columnTypes = columnTypes(columnValuesList);
return new Table(
- columnNames.toArray(new String[0]),
- columnTypes.toArray(new Class[0]),
+ new ArrayList<>(),
+ columnTypes,
columnValuesList);
}
- public static final class Row {
- private final Map<String, Integer> columnMap;
- private final Object[] values;
-
- Row(Map<String, Integer> columnMap, Object[] values) {
- this.columnMap = columnMap;
- this.values = values;
+ /**
+ * Create a table from a list of lists representing tabular data
+ * where the first row may consists of column headers.
+ *
+ * @param lists row values as lists
+ * @param withFirstRowAsColumnNames column names as first row?
+ * @return table
+ */
+ public static Table fromLists(List<List<Object>> lists, boolean withFirstRowAsColumnNames) {
+ requireNonNull(lists, "lists is null");
+
+ if (withFirstRowAsColumnNames) {
+ final List<String> columnNames = columnNames(lists.get(0));
+ final List<List<Object>> table = lists.subList(1, lists.size());
+ return fromLists(columnNames, table);
+ } else {
+ return fromLists(lists);
}
+ }
- public Object[] getValues() {
- return values;
- }
+ /**
+ * Create a table from column names and row values.
+ *
+ * @param columnNames list of column names
+ * @param lists row values as lists
+ * @return table
+ */
+ public static Table fromLists(Collection<String> columnNames, List<List<Object>> lists) {
+ requireNonNull(columnNames, "columnNames is null");
+ requireNonNull(lists, "lists is null");
- public Object getValue(int column) {
- return values[column];
- }
+ final List<List<Object>> columnValuesList = ListUtils.transpose(lists);
+ final List<Class<?>> columnTypes = columnTypes(columnValuesList);
- public Object getValue(String column) {
- return getValue(columnMap.get(column));
- }
+ return new Table(
+ new ArrayList<>(columnNames),
+ columnTypes,
+ columnValuesList);
}
- private static List<String> columnNames(List<Map<String, Object>> list) {
- return list.stream()
+ /**
+ * Determine column names based on all available keys of the maps.
+ *
+ * @param maps list of maps
+ * @return column names
+ */
+ private static List<String> columnNames(Collection<Map<String, Object>> maps) {
+ return maps.stream()
.map(Map::keySet)
.flatMap(Collection::stream)
.distinct()
.collect(Collectors.toList());
}
- private static Object[][] columnValuesList(List<Map<String, Object>> list, List<String> columnNames) {
- return columnNames.stream()
- .map(columnName -> columnValues(list, columnName))
- .collect(Collectors.toList())
- .toArray(new Object[0][0]);
+ /**
+ * Determine column names based on a single list.
+ *
+ * @param list list of column names
+ * @return column names
+ */
+ private static List<String> columnNames(List<Object> list) {
+ return list.stream()
+ .map(Object::toString)
+ .collect(Collectors.toList());
}
- private static Object[][] columnValuesList(List<List<Object>> lists) {
- if (lists.isEmpty()) {
- return new Object[0][0];
- }
-
- return IntStream.range(0, lists.get(0).size())
- .mapToObj(i -> columnValues(lists, i))
- .collect(Collectors.toList())
- .toArray(new Object[0][0]);
+ /**
+ * Determine all column values.
+ *
+ * @param maps list of maps
+ * @param columnNames column names
+ * @return list of column values
+ */
+ private static List<List<Object>> columnValuesList(List<Map<String, Object>> maps, List<String> columnNames) {
+ return columnNames.stream()
+ .map(columnName -> columnValues(maps, columnName))
+ .collect(Collectors.toList());
}
- private static Object[] columnValues(List<Map<String, Object>> maps, String columnName) {
+ /**
+ * Determine the values of a single column.
+ *
+ * @param maps list of maps
+ * @param columnName column name
+ * @return values of the given column
+ */
+ private static List<Object> columnValues(List<Map<String, Object>> maps, String columnName) {
return maps.stream()
.map(map -> map.getOrDefault(columnName, null))
- .toArray();
- }
-
- private static Object[] columnValues(List<List<Object>> lists, int column) {
- return lists.stream()
- .map(list -> list.get(column))
- .toArray();
+ .collect(Collectors.toList());
}
- private static List<Class<?>> columnTypes(Object[][] columnValuesList) {
- return Arrays.stream(columnValuesList)
+ /**
+ * Determine the column types based on the first non-null values.
+ *
+ * @param columnValuesList list of column values
+ * @return classes of the first non-null value
+ */
+ private static List<Class<?>> columnTypes(List<List<Object>> columnValuesList) {
+ return columnValuesList.stream()
.map(Table::columnType)
.collect(Collectors.toList());
}
/**
- * Determine the column type.
+ * Determine the column type based on the first non-null value.
*
* @param columnValues column values
* @return class of the first non-null value
*/
- private static Class<?> columnType(Object[] columnValues) {
- for (final Object columnValue : columnValues) {
- if (columnValue != null) {
- return columnValue.getClass();
- }
- }
-
- throw new IllegalArgumentException("No column value found!!!");
+ private static Class<?> columnType(List<Object> columnValues) {
+ return ListUtils.coalesce(columnValues).getClass();
}
}
diff --git a/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/util/ArrayUtils.java b/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/util/ArrayUtils.java
deleted file mode 100644
index 82be8fa..0000000
--- a/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/util/ArrayUtils.java
+++ /dev/null
@@ -1,72 +0,0 @@
-package org.apache.freemarker.generator.base.util;
-
-import java.lang.reflect.Array;
-import java.util.Arrays;
-
-import static java.util.Objects.requireNonNull;
-
-public class ArrayUtils {
-
- /**
- * Transposes the given array, swapping rows with columns. The given array might contain arrays as elements that are
- * not all of the same length. The returned array will have {@code null} values at those places.
- *
- * @param <T> the type of the array
- * @param array the array
- * @return the transposed array
- * @throws NullPointerException if the given array is {@code null}
- */
- @SuppressWarnings("unchecked")
- public static <T> T[][] transpose(final T[][] array) {
- requireNonNull(array);
- // get y count
- final int yCount = Arrays.stream(array).mapToInt(a -> a.length).max().orElse(0);
- final int xCount = array.length;
- final Class<?> componentType = array.getClass().getComponentType().getComponentType();
- final T[][] result = (T[][]) Array.newInstance(componentType, yCount, xCount);
- for (int x = 0; x < xCount; x++) {
- for (int y = 0; y < yCount; y++) {
- if (array[x] == null || y >= array[x].length) {
- break;
- }
- result[y][x] = array[x][y];
- }
- }
- return result;
- }
-
- /**
- * Copy an array to another array while casting to <code>R</code>.
- *
- * @param array array to copy
- * @param <T> the source type of the array
- * @param <R> the target type of the array
- * @return copied array
- */
- @SuppressWarnings("unchecked")
- public static <T, R> R[] copy(final T[] array) {
- final Class<?> componentType = array.getClass().getComponentType();
- final R[] result = (R[]) Array.newInstance(componentType, array.length);
- for (int i = 0; i < array.length; i++) {
- result[i] = (R) array[i];
- }
- return result;
- }
-
- /**
- * Returns the first non-null value of the array.
- *
- * @param array array
- * @param <T> the type of the array
- * @return copied array
- */
- @SuppressWarnings("unchecked")
- public static <T> T coalesce(T... array) {
- for (T i : array) {
- if (i != null) {
- return i;
- }
- }
- return null;
- }
-}
diff --git a/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/util/ListUtils.java b/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/util/ListUtils.java
new file mode 100644
index 0000000..406a280
--- /dev/null
+++ b/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/util/ListUtils.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.freemarker.generator.base.util;
+
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Objects;
+
+public class ListUtils {
+
+ /**
+ * Transposes the given tabular data, swapping rows with columns.
+ *
+ * @param <T> the type of the table
+ * @param table the table
+ * @return the transposed table
+ * @throws NullPointerException if the given table is {@code null}
+ */
+ public static <T> List<List<T>> transpose(final List<List<T>> table) {
+ if (table.isEmpty()) {
+ return new ArrayList<>();
+ }
+
+ final List<List<T>> result = new ArrayList<>();
+ for (int i = 0; i < table.get(0).size(); i++) {
+ final List<T> col = new ArrayList<>();
+ for (List<T> row : table) {
+ col.add(row.get(i));
+ }
+ result.add(col);
+ }
+ return result;
+ }
+
+ /**
+ * Returns the first non-null value of the list.
+ *
+ * @param list array
+ * @param <T> the type of the array
+ * @return copied array
+ */
+ public static <T> T coalesce(List<T> list) {
+ return list.stream().filter(Objects::nonNull).findFirst().orElseGet(() -> null);
+ }
+
+ /**
+ * Copy an array to another array while casting to <code>R</code>.
+ *
+ * @param array array to copy
+ * @param <T> the source type of the array
+ * @param <R> the target type of the array
+ * @return copied array
+ */
+ @SuppressWarnings("unchecked")
+ public static <T, R> List<R> copy(final List<T> array) {
+ final List<R> result = new ArrayList<>();
+ for (int i = 0; i < array.size(); i++) {
+ result.set(i, (R) array.get(i));
+ }
+ return result;
+ }
+}
diff --git a/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/util/MapBuilder.java b/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/util/MapBuilder.java
index ab60a5e..60daa80 100644
--- a/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/util/MapBuilder.java
+++ b/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/util/MapBuilder.java
@@ -1,12 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package org.apache.freemarker.generator.base.util;
import java.util.HashMap;
+import java.util.LinkedHashMap;
+import java.util.Map;
public class MapBuilder {
- public static HashMap<String, Object> toHashMap(Object... data) {
+ public static Map<String, Object> toLinkedMap(Object... data) {
- final HashMap<String, Object> result = new HashMap<>();
+ final HashMap<String, Object> map = new LinkedHashMap<>();
if (data.length % 2 != 0) {
throw new IllegalArgumentException("Odd number of arguments");
@@ -25,11 +43,11 @@ public class MapBuilder {
currKey = value.toString();
continue;
case 1:
- result.put(currKey, value);
+ map.put(currKey, value);
break;
}
}
- return result;
+ return map;
}
}
diff --git a/freemarker-generator-base/src/test/java/org/apache/freemarker/generator/table/TableTest.java b/freemarker-generator-base/src/test/java/org/apache/freemarker/generator/table/TableTest.java
index 82b560a..695e140 100644
--- a/freemarker-generator-base/src/test/java/org/apache/freemarker/generator/table/TableTest.java
+++ b/freemarker-generator-base/src/test/java/org/apache/freemarker/generator/table/TableTest.java
@@ -1,3 +1,19 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
package org.apache.freemarker.generator.table;
import org.apache.freemarker.generator.base.table.Table;
@@ -10,50 +26,120 @@ import java.util.List;
import java.util.Map;
import static junit.framework.TestCase.assertEquals;
+import static junit.framework.TestCase.assertNull;
public class TableTest {
- public final List<Map<String, Object>> books = Arrays.asList(
- MapBuilder.toHashMap(
+ public final List<Map<String, Object>> booksMaps = Arrays.asList(
+ MapBuilder.toLinkedMap(
"Book ID", "1",
"Book Name", "Computer Architecture",
"Category", "Computers",
"In Stock", true,
"Price", 125.60),
- MapBuilder.toHashMap(
+ MapBuilder.toLinkedMap(
"Book ID", "2",
"Book Name", "Asp.Net 4 Blue Book",
"Category", "Programming",
"In Stock", null,
"Price", 56),
- MapBuilder.toHashMap(
+ MapBuilder.toLinkedMap(
"Book ID", "3",
"Book Name", "Popular Science",
"Category", "Science",
"Price", 210.40)
);
+ public final List<String> booksHeader = Arrays.asList(
+ "Book ID",
+ "Book Name",
+ "Category",
+ "In Stock",
+ "Price");
+
+
+ public final List<List<Object>> booksList = Arrays.asList(
+ Arrays.asList("1", "Computer Architecture", "Computers", true, 125.60),
+ Arrays.asList("2", "Asp.Net 4 Blue Book", "Programming", null, 56),
+ Arrays.asList("3", "Popular Science", "Science", null, 210.40)
+ );
+
+ public final List<List<Object>> booksListWithHeaders = Arrays.asList(
+ Arrays.asList("Book ID", "Book Name", "Category", "In Stock", "Price"),
+ Arrays.asList("1", "Computer Architecture", "Computers", true, 125.60),
+ Arrays.asList("2", "Asp.Net 4 Blue Book", "Programming", null, 56),
+ Arrays.asList("3", "Popular Science", "Science", null, 210.40)
+ );
+
@Test
- public void shouldConvertFromMapList() {
- final Table table = Table.fromMaps(books);
+ public void shouldConvertFromMaps() {
+ final Table table = Table.fromMaps(booksMaps);
- assertEquals(5, table.getNrOfColumns());
- assertEquals(3, table.getNrOfRows());
- assertEquals("1", table.getRow(0).getValue("Book ID"));
- assertEquals("2", table.getRow(1).getValue("Book ID"));
- assertEquals("3", table.getRow(2).getValue("Book ID"));
+ validateBooks(table);
+ assertEquals(booksHeader, table.getColumnNames());
}
@Test
- public void shouldConvertFromEmptyMapList() {
+ public void shouldConvertFromEmptyMaps() {
final Table table = Table.fromMaps(new ArrayList<>());
assertEquals(0, table.getNrOfColumns());
- assertEquals(0, table.getNrOfRows());
+ assertEquals(0, table.size());
}
- @Test(expected = IllegalArgumentException.class)
- public void shouldConvertFromNullpList() {
- Table.fromMaps(null);
+ @Test
+ public void shouldConvertFromNullMap() {
+ final Table table = Table.fromMaps(null);
+
+ assertEquals(0, table.getNrOfColumns());
+ assertEquals(0, table.size());
+ }
+
+ @Test
+ public void shouldConvertFromListsWithExplicitHeaders() {
+ final Table table = Table.fromLists(booksHeader, booksList);
+
+ validateBooks(table);
+ assertEquals(booksHeader, table.getColumnNames());
+ }
+
+ @Test
+ public void shouldConvertFromListsWithImplicitHeaders() {
+ final Table table = Table.fromLists(booksListWithHeaders, true);
+
+ validateBooks(table);
+ assertEquals(booksHeader, table.getColumnNames());
+ }
+
+ @Test
+ public void shouldConvertFromListsWithEmptyHeaders() {
+ final Table table = Table.fromLists(booksList);
+
+ validateBooks(table);
+ }
+
+ public void validateBooks(Table table) {
+ assertEquals(5, table.getNrOfColumns());
+ assertEquals(3, table.size());
+ // Book Id
+ assertEquals("1", table.get(0, 0));
+ assertEquals("2", table.get(1, 0));
+ assertEquals("3", table.get(2, 0));
+ // Book Name
+ assertEquals("Computer Architecture", table.get(0, 1));
+ assertEquals("Asp.Net 4 Blue Book", table.get(1, 1));
+ assertEquals("Popular Science", table.get(2, 1));
+ // Category
+ assertEquals("Computers", table.get(0, 2));
+ assertEquals("Programming", table.get(1, 2));
+ assertEquals("Science", table.get(2, 2));
+ // In Stock
+ assertEquals(true, table.get(0, 3));
+ assertNull(table.get(1, 3));
+ assertNull(table.get(2, 3));
+ // Price
+ assertEquals(125.60, table.get(0, 4));
+ assertEquals(56, table.get(1, 4));
+ assertEquals(210.40, table.get(2, 4));
}
}
diff --git a/freemarker-generator-cli/templates/dataframe/example.ftl b/freemarker-generator-cli/templates/dataframe/example.ftl
index 8acda9c..0ac3193 100644
--- a/freemarker-generator-cli/templates/dataframe/example.ftl
+++ b/freemarker-generator-cli/templates/dataframe/example.ftl
@@ -16,7 +16,7 @@
under the License.
-->
<#assign csvParser = CSVTool.parse(DataSources.get(0))>
-<#assign users = DataFrameTool.toDataFrame(csvParser)>
+<#assign users = DataFrameTool.fromCSVParser(csvParser)>
Original Data
=============================================================================
@@ -26,8 +26,8 @@ Select By Name & Country
=============================================================================
<#assign country = "Germany">
${DataFrameTool.print(users
- .select("(name == 'Schmitt' || name == 'Meier') && country == '${country}'")
- .sort("name", DataFrameTool.sortOrder["ASCENDING"]))}
+.select("(name == 'Schmitt' || name == 'Meier') && country == '${country}'")
+.sort("name", DataFrameTool.sortOrder["ASCENDING"]))}
Head of Users
=============================================================================
diff --git a/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/dataframe/DataFrameTool.java b/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/dataframe/DataFrameTool.java
index 67375aa..695cd04 100644
--- a/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/dataframe/DataFrameTool.java
+++ b/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/dataframe/DataFrameTool.java
@@ -23,8 +23,9 @@ import de.unknownreality.dataframe.transform.ColumnDataFrameTransform;
import de.unknownreality.dataframe.transform.CountTransformer;
import org.apache.commons.csv.CSVParser;
import org.apache.freemarker.generator.base.util.Validate;
-import org.apache.freemarker.generator.tools.dataframe.impl.CommonsCSVConverter;
-import org.apache.freemarker.generator.tools.dataframe.impl.MapConverter;
+import org.apache.freemarker.generator.tools.dataframe.converter.CSVConverter;
+import org.apache.freemarker.generator.tools.dataframe.converter.ListConverter;
+import org.apache.freemarker.generator.tools.dataframe.converter.MapConverter;
import java.io.Writer;
import java.util.HashMap;
@@ -59,20 +60,29 @@ public class DataFrameTool {
* @param csvParser CSV Parser
* @return data frame
*/
- public DataFrame toDataFrame(CSVParser csvParser) {
- return CommonsCSVConverter.toDataFrame(csvParser);
+ public DataFrame fromCSVParser(CSVParser csvParser) {
+ return CSVConverter.toDataFrame(csvParser);
}
/**
- * Create a data frame from a list of maps. It is assumed
- * that the map represent tabular data without missing
- * values.
+ * Create a data frame from a list of maps.
*
- * @param list map to build the data frame
+ * @param maps maps to build the data frame
* @return data frame
*/
- public DataFrame toDataFrame(List<Map<String, Object>> list) {
- return MapConverter.toDataFrame(list);
+ public DataFrame fromMaps(List<Map<String, Object>> maps) {
+ return MapConverter.toDataFrame(maps);
+ }
+
+ /**
+ * Create a data frame from a list of rows.
+ *
+ * @param lists lists to build the data frame from
+ * @param withFirstRowAsColumnNames column names as first row?
+ * @return data frame
+ */
+ public DataFrame fromLists(List<List<Object>> lists, boolean withFirstRowAsColumnNames) {
+ return ListConverter.toDataFrame(lists, withFirstRowAsColumnNames);
}
/**
diff --git a/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/dataframe/impl/CommonsCSVConverter.java b/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/dataframe/converter/CSVConverter.java
similarity index 95%
rename from freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/dataframe/impl/CommonsCSVConverter.java
rename to freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/dataframe/converter/CSVConverter.java
index 7506de0..06863fb 100644
--- a/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/dataframe/impl/CommonsCSVConverter.java
+++ b/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/dataframe/converter/CSVConverter.java
@@ -1,4 +1,4 @@
-package org.apache.freemarker.generator.tools.dataframe.impl;
+package org.apache.freemarker.generator.tools.dataframe.converter;
import de.unknownreality.dataframe.DataFrame;
import de.unknownreality.dataframe.DataFrameBuilder;
@@ -8,7 +8,7 @@ import org.apache.commons.csv.CSVRecord;
import java.io.IOException;
import java.util.List;
-public class CommonsCSVConverter {
+public class CSVConverter {
/**
* Create a data frame from Apache Commons CSV Parser.
diff --git a/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/dataframe/converter/ConverterUtils.java b/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/dataframe/converter/ConverterUtils.java
new file mode 100644
index 0000000..9a1e4fb
--- /dev/null
+++ b/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/dataframe/converter/ConverterUtils.java
@@ -0,0 +1,106 @@
+package org.apache.freemarker.generator.tools.dataframe.converter;
+
+import de.unknownreality.dataframe.DataFrame;
+import de.unknownreality.dataframe.DataFrameBuilder;
+import org.apache.freemarker.generator.base.table.Table;
+
+import java.util.List;
+
+public class ConverterUtils {
+
+ static DataFrame toDataFrame(Table table) {
+ final DataFrame dataFrame = create(table);
+ return appendValues(dataFrame, table);
+ }
+
+ private static DataFrameBuilder addColumn(DataFrameBuilder builder, String name, Class<?> clazz) {
+ switch (clazz.getName()) {
+ case "java.lang.Boolean":
+ return builder.addBooleanColumn(name);
+ case "java.lang.Byte":
+ return builder.addByteColumn(name);
+ case "java.lang.Double":
+ return builder.addDoubleColumn(name);
+ case "java.lang.Float":
+ return builder.addFloatColumn(name);
+ case "java.lang.Integer":
+ return builder.addIntegerColumn(name);
+ case "java.lang.Long":
+ return builder.addLongColumn(name);
+ case "java.lang.Short":
+ return builder.addShortColumn(name);
+ case "java.lang.String":
+ return builder.addStringColumn(name);
+ case "java.time.LocalDate":
+ return builder.addStringColumn(name);
+ case "java.time.LocalTime":
+ return builder.addStringColumn(name);
+ case "java.util.Date":
+ return builder.addStringColumn(name);
+ default:
+ throw new RuntimeException("Unable to add colum for the following type: " + clazz.getName());
+ }
+ }
+
+ private static Comparable<?>[] toComparables(List<Object> values) {
+ final int size = values.size();
+ final Comparable<?>[] comparables = new Comparable<?>[size];
+ for (int i = 0; i < size; i++) {
+ comparables[i] = (Comparable<?>) values.get(i);
+ }
+ return comparables;
+ }
+
+ private static Comparable<?>[] toComparables(Object[] values) {
+ final Comparable<?>[] comparables = new Comparable<?>[values.length];
+ for (int i = 0; i < values.length; i++) {
+ comparables[i] = (Comparable<?>) values[i];
+ }
+ return comparables;
+ }
+
+ /**
+ * Create a <code>DataFrame</code> from a table.
+ *
+ * @param table table
+ * @return data frame
+ */
+ private static DataFrame create(Table table) {
+ final DataFrameBuilder builder = DataFrameBuilder.create();
+
+ if (table.hasColumnHeaderRow()) {
+ for (int i = 0; i < table.getColumnNames().size(); i++) {
+ addColumn(builder, table.getColumnNames().get(i), table.getColumnTypes().get(i));
+ }
+ } else {
+ if (!table.isEmpty()) {
+ final List<Object> firstRecord = table.getRow(0);
+ for (int i = 0; i < firstRecord.size(); i++) {
+ builder.addStringColumn(getAlphaColumnName(i + 1));
+ }
+ }
+ }
+
+ return builder.build();
+ }
+
+ private static DataFrame appendValues(DataFrame dataFrame, Table table) {
+ // TODO the conversion to Comparable[] is ugly
+ for (int i = 0; i < table.size(); i++) {
+ dataFrame.append(ConverterUtils.toComparables(table.getRow(i)));
+ }
+ return dataFrame;
+ }
+
+ private static String getAlphaColumnName(int num) {
+ String result = "";
+ while (num > 0) {
+ num--; // 1 => a, not 0 => a
+ final int remainder = num % 26;
+ final char digit = (char) (remainder + 65);
+ result = digit + result;
+ num = (num - remainder) / 26;
+ }
+ return result;
+ }
+}
diff --git a/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/dataframe/converter/ListConverter.java b/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/dataframe/converter/ListConverter.java
new file mode 100644
index 0000000..95bc6bf
--- /dev/null
+++ b/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/dataframe/converter/ListConverter.java
@@ -0,0 +1,21 @@
+package org.apache.freemarker.generator.tools.dataframe.converter;
+
+import de.unknownreality.dataframe.DataFrame;
+import org.apache.freemarker.generator.base.table.Table;
+
+import java.util.List;
+
+public class ListConverter {
+
+ /**
+ * Create a data frame from a list of lists. It is assumed
+ * that the lists represent tabular data.
+ *
+ * @param lists lists to build the data frame
+ * @return <code>DataFrame</code>
+ */
+ public static DataFrame toDataFrame(List<List<Object>> lists, boolean withFirstRowAsColumnNames) {
+ final Table table = Table.fromLists(lists, withFirstRowAsColumnNames);
+ return ConverterUtils.toDataFrame(table);
+ }
+}
diff --git a/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/dataframe/converter/MapConverter.java b/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/dataframe/converter/MapConverter.java
new file mode 100644
index 0000000..e96d938
--- /dev/null
+++ b/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/dataframe/converter/MapConverter.java
@@ -0,0 +1,39 @@
+package org.apache.freemarker.generator.tools.dataframe.converter;
+
+import de.unknownreality.dataframe.DataFrame;
+import de.unknownreality.dataframe.DataFrameBuilder;
+import org.apache.freemarker.generator.base.table.Table;
+
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+
+public class MapConverter {
+
+ /**
+ * Create a data frame from a list of maps. It is assumed
+ * that the map represent tabular data.
+ *
+ * @param map map to build the data frame
+ * @return <code>DataFrame</code>
+ */
+ public static DataFrame toDataFrame(Map<String, Object> map) {
+ return toDataFrame(Collections.singletonList(map));
+ }
+
+ /**
+ * Create a data frame from a list of maps. It is assumed
+ * that the map represent tabular data.
+ *
+ * @param maps list of map to build the data frame
+ * @return <code>DataFrame</code>
+ */
+ public static DataFrame toDataFrame(List<Map<String, Object>> maps) {
+ if (maps == null || maps.isEmpty()) {
+ return DataFrameBuilder.createDefault();
+ }
+
+ final Table table = Table.fromMaps(maps);
+ return ConverterUtils.toDataFrame(table);
+ }
+}
diff --git a/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/dataframe/impl/MapConverter.java b/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/dataframe/impl/MapConverter.java
deleted file mode 100644
index 8ced6f7..0000000
--- a/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/dataframe/impl/MapConverter.java
+++ /dev/null
@@ -1,85 +0,0 @@
-package org.apache.freemarker.generator.tools.dataframe.impl;
-
-import de.unknownreality.dataframe.DataFrame;
-import de.unknownreality.dataframe.DataFrameBuilder;
-import org.apache.freemarker.generator.base.table.Table;
-
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-
-public class MapConverter {
-
- /**
- * Create a data frame from a list of maps. It is assumed
- * that the map represent tabular data.
- *
- * @param map map to build the data frame
- * @return <code>DataFrame</code>
- */
- public static DataFrame toDataFrame(Map<String, Object> map) {
- return toDataFrame(Collections.singletonList(map));
- }
-
- /**
- * Create a data frame from a list of maps. It is assumed
- * that the map represent tabular data.
- *
- * @param maps list of map to build the data frame
- * @return <code>DataFrame</code>
- */
- public static DataFrame toDataFrame(List<Map<String, Object>> maps) {
- if (maps == null || maps.isEmpty()) {
- return DataFrameBuilder.createDefault();
- }
-
- final Table table = Table.fromMaps(maps);
-
- // build dataframe with headers
- final DataFrameBuilder builder = DataFrameBuilder.create();
- for (int i = 0; i < table.getColumnNames().length; i++) {
- addColumn(builder, table.getColumnNames()[i], table.getColumnTypes()[i]);
- }
- final DataFrame dataFrame = builder.build();
-
- // populate rows
- for (int i = 0; i < table.getNrOfRows(); i++) {
- final Object[] values = table.getRowValues(i);
- dataFrame.append(toComparables(values));
- }
-
- return dataFrame;
- }
-
- private static DataFrameBuilder addColumn(DataFrameBuilder builder, String name, Class<?> clazz) {
- switch (clazz.getName()) {
- case "java.lang.Boolean":
- return builder.addBooleanColumn(name);
- case "java.lang.Byte":
- return builder.addByteColumn(name);
- case "java.lang.Double":
- return builder.addDoubleColumn(name);
- case "java.lang.Float":
- return builder.addFloatColumn(name);
- case "java.lang.Integer":
- return builder.addIntegerColumn(name);
- case "java.lang.Long":
- return builder.addLongColumn(name);
- case "java.lang.Short":
- return builder.addShortColumn(name);
- case "java.lang.String":
- return builder.addStringColumn(name);
- default:
- throw new RuntimeException("Unable to add colum for the following type: " + clazz.getName());
- }
- }
-
- private static Comparable<?>[] toComparables(Object[] values) {
- final Comparable<?>[] comparables = new Comparable<?>[values.length];
- for (int i = 0; i < values.length; i++) {
- comparables[i] = (Comparable<?>) values[i];
- }
- return comparables;
- }
-
-}
diff --git a/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/excel/ExcelTool.java b/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/excel/ExcelTool.java
index f03df80..b5ead1d 100644
--- a/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/excel/ExcelTool.java
+++ b/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/excel/ExcelTool.java
@@ -78,10 +78,10 @@ public class ExcelTool {
* @param sheet Excel sheet
* @return Table containing formatted cell values as strings
*/
- public List<List<String>> toTable(Sheet sheet) {
+ public List<List<Object>> toTable(Sheet sheet) {
final DataFormatter dataFormatter = dataFormatter();
final Iterator<Row> iterator = sheet.iterator();
- final List<List<String>> result = new ArrayList<>();
+ final List<List<Object>> result = new ArrayList<>();
while (iterator.hasNext()) {
final Row row = iterator.next();
@@ -117,8 +117,8 @@ public class ExcelTool {
return "Process Excels files (XLS, XLSX) using Apache POI (see https://poi.apache.org)";
}
- private static List<String> toColumns(Row row, DataFormatter dataFormatter) {
- final List<String> columnValues = new ArrayList<>();
+ private static List<Object> toColumns(Row row, DataFormatter dataFormatter) {
+ final List<Object> columnValues = new ArrayList<>();
for (int columnIndex = 0; columnIndex < row.getLastCellNum(); columnIndex++) {
final Cell cell = row.getCell(columnIndex, CREATE_NULL_AS_BLANK);
final String formatedCellValue = dataFormatter.formatCellValue(cell).trim();
diff --git a/freemarker-generator-tools/src/test/java/org/apache/freemarker/generator/tools/dataframe/DataFrameToolTest.java b/freemarker-generator-tools/src/test/java/org/apache/freemarker/generator/tools/dataframe/DataFrameToolTest.java
index dd6bd70..8c93dd3 100644
--- a/freemarker-generator-tools/src/test/java/org/apache/freemarker/generator/tools/dataframe/DataFrameToolTest.java
+++ b/freemarker-generator-tools/src/test/java/org/apache/freemarker/generator/tools/dataframe/DataFrameToolTest.java
@@ -19,8 +19,11 @@ package org.apache.freemarker.generator.tools.dataframe;
import de.unknownreality.dataframe.DataFrame;
import org.apache.commons.csv.CSVFormat;
import org.apache.commons.csv.CSVParser;
+import org.apache.freemarker.generator.base.datasource.DataSourceFactory;
import org.apache.freemarker.generator.tools.commonscsv.CommonsCSVTool;
+import org.apache.freemarker.generator.tools.excel.ExcelTool;
import org.apache.freemarker.generator.tools.gson.GsonTool;
+import org.apache.poi.ss.usermodel.Workbook;
import org.junit.Test;
import java.util.List;
@@ -67,7 +70,7 @@ public class DataFrameToolTest {
@Test
public void shouldParseCsvFileWithoutHeader() {
final CSVParser csvParser = csvParser(CSV_WITHOUT_HEADER, DEFAULT.withDelimiter(';'));
- final DataFrame dataFrame = dataFrameTool().toDataFrame(csvParser);
+ final DataFrame dataFrame = dataFrameTool().fromCSVParser(csvParser);
assertEquals(3, dataFrame.getColumns().size());
assertEquals(4, dataFrame.getRows().size());
@@ -79,7 +82,7 @@ public class DataFrameToolTest {
@Test
public void shouldParseCsvFileWithHeader() {
final CSVParser csvParser = csvParser(CSV_WITH_HEADER, DEFAULT.withHeader().withDelimiter(';'));
- final DataFrame dataFrame = dataFrameTool().toDataFrame(csvParser);
+ final DataFrame dataFrame = dataFrameTool().fromCSVParser(csvParser);
assertEquals(3, dataFrame.getColumns().size());
assertEquals(4, dataFrame.getRows().size());
@@ -95,7 +98,7 @@ public class DataFrameToolTest {
public void shouldParseJsonTable() {
final String columnName = "Book ID";
final List<Map<String, Object>> json = (List<Map<String, Object>>) gsonTool().parse(JSON_ARRAY);
- final DataFrame dataFrame = dataFrameTool().toDataFrame(json);
+ final DataFrame dataFrame = dataFrameTool().fromMaps(json);
assertEquals(5, dataFrame.getColumns().size());
assertEquals(3, dataFrame.getRows().size());
@@ -104,6 +107,20 @@ public class DataFrameToolTest {
assertEquals("3", dataFrame.getColumn(columnName).get(2));
}
+ // === Excel ============================================================
+
+ @Test
+ public void shouldParseExcelSheet() {
+ final ExcelTool excelTool = excelTool();
+ final Workbook workbook = excelTool.parse(DataSourceFactory.create("./src/test/data/excel/test.xls"));
+ final List<List<Object>> sheet = excelTool.toTable(workbook.getSheetAt(0));
+
+ final DataFrame dataFrame = dataFrameTool().fromLists(sheet, true);
+
+ return;
+
+ }
+
private DataFrameTool dataFrameTool() {
return new DataFrameTool();
}
@@ -116,6 +133,10 @@ public class DataFrameToolTest {
return new GsonTool();
}
+ private ExcelTool excelTool() {
+ return new ExcelTool();
+ }
+
private CSVParser csvParser(String csv, CSVFormat csvFormat) {
return commonsCSVTool().parse(csv, csvFormat);
}
diff --git a/freemarker-generator-tools/src/test/java/org/apache/freemarker/generator/tools/excel/ExcelToolTest.java b/freemarker-generator-tools/src/test/java/org/apache/freemarker/generator/tools/excel/ExcelToolTest.java
index 9c8ef8b..089c2d0 100644
--- a/freemarker-generator-tools/src/test/java/org/apache/freemarker/generator/tools/excel/ExcelToolTest.java
+++ b/freemarker-generator-tools/src/test/java/org/apache/freemarker/generator/tools/excel/ExcelToolTest.java
@@ -39,7 +39,7 @@ public class ExcelToolTest {
final Workbook workbook = workbook(TEST_XLS);
final List<Sheet> sheets = excelTool().getSheets(workbook);
- final List<List<String>> records = excelTool().toTable(sheets.get(0));
+ final List<List<Object>> records = excelTool().toTable(sheets.get(0));
assertEquals(1, sheets.size());
assertEquals(3, records.size());
@@ -50,7 +50,7 @@ public class ExcelToolTest {
final Workbook workbook = workbook(TEST_XLSX);
final List<Sheet> sheets = excelTool().getSheets(workbook);
- final List<List<String>> records = excelTool().toTable(sheets.get(0));
+ final List<List<Object>> records = excelTool().toTable(sheets.get(0));
assertEquals(1, sheets.size());
assertEquals(3, records.size());
@@ -71,9 +71,9 @@ public class ExcelToolTest {
public void shouldConvertSheetToTable() {
final Workbook workbook = workbook(TEST_XLSX);
final List<Sheet> sheets = excelTool().getSheets(workbook);
- final List<List<String>> records = excelTool().toTable(sheets.get(0));
+ final List<List<Object>> records = excelTool().toTable(sheets.get(0));
- final List<String> record = records.get(1);
+ final List<Object> record = records.get(1);
assertEquals("Row 1", record.get(0));
assertEquals("01/31/17", record.get(1));