You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@freemarker.apache.org by sg...@apache.org on 2020/06/14 20:51:50 UTC
[freemarker-generator] 01/04: FREEMARKER-144 Proof Of Concept for
providing DataFrames
This is an automated email from the ASF dual-hosted git repository.
sgoeschl pushed a commit to branch FREEMARKER-144
in repository https://gitbox.apache.org/repos/asf/freemarker-generator.git
commit 8dca941727f848e0fef971b10084a2b98b80b59c
Author: Siegfried Goeschl <si...@gmail.com>
AuthorDate: Mon Jun 8 16:14:06 2020 +0200
FREEMARKER-144 Proof Of Concept for providing DataFrames
---
.../freemarker/generator/base/table/Table.java | 163 +++++++++++++++++++++
.../freemarker/generator/base/util/MapBuilder.java | 35 +++++
.../freemarker/generator/table/TableTest.java | 59 ++++++++
.../site/template/nginx/nginx.conf.ftl | 2 +-
.../markdown/cli/usage/transforming-directories.md | 106 ++++++++++++--
.../generator/tools/dataframe/DataFrameTool.java | 106 +++++++++++---
.../tools/dataframe/DataFrameToolTest.java | 34 ++++-
7 files changed, 457 insertions(+), 48 deletions(-)
diff --git a/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/table/Table.java b/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/table/Table.java
new file mode 100644
index 0000000..3667e88
--- /dev/null
+++ b/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/table/Table.java
@@ -0,0 +1,163 @@
+package org.apache.freemarker.generator.base.table;
+
+import org.apache.freemarker.generator.base.util.Validate;
+
+import java.lang.reflect.Array;
+import java.util.Arrays;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+import static java.util.Objects.requireNonNull;
+
+/**
+ * Simple table model filled from a Map.
+ */
+public class Table {
+ private final String[] columnNames;
+ private final Class<?>[] columnTypes;
+ private final Object[][] values;
+ private final Map<String, Integer> columnMap;
+
+ private Table(String[] columnNames, Class<?>[] columnTypes, Object[][] columnValuesList) {
+ this.columnNames = requireNonNull(columnNames);
+ this.columnTypes = requireNonNull(columnTypes);
+ this.values = transpose(requireNonNull(columnValuesList));
+
+ this.columnMap = new HashMap<>();
+ for (int i = 0; i < this.columnNames.length; i++) {
+ this.columnMap.put(this.columnNames[i], i);
+ }
+ }
+
+ public String[] getColumnNames() {
+ return columnNames;
+ }
+
+ public Class<?>[] getColumnTypes() {
+ return columnTypes;
+ }
+
+ public int getNrOfColumns() {
+ return columnNames.length;
+ }
+
+ public int getNrOfRows() {
+ return values.length;
+ }
+
+ public Object[] getRowValues(int row) {
+ return values[row];
+ }
+
+ public Row getRow(int row) {
+ return new Row(columnMap, getRowValues(row));
+ }
+
+ public static Table fromMaps(List<Map<String, Object>> list) {
+ Validate.notNull(list, "list is null");
+
+ final List<String> columnNames = columnNames(list);
+ final Object[][] tableValues = columnValues(list, columnNames);
+ final List<Class<?>> columnTypes = columnTypes(tableValues);
+
+ return new Table(
+ columnNames.toArray(new String[0]),
+ columnTypes.toArray(new Class[0]),
+ tableValues);
+ }
+
+ public static final class Row {
+ private final Map<String, Integer> columnMap;
+ private final Object[] values;
+
+ Row(Map<String, Integer> columnMap, Object[] values) {
+ this.columnMap = columnMap;
+ this.values = values;
+ }
+
+ public Object[] getValues() {
+ return values;
+ }
+
+ public Object getValue(int column) {
+ return values[column];
+ }
+
+ public Object getValue(String column) {
+ return getValue(columnMap.get(column));
+ }
+ }
+
+ private static List<String> columnNames(List<Map<String, Object>> list) {
+ return list.stream()
+ .map(Map::keySet)
+ .flatMap(Collection::stream)
+ .distinct()
+ .collect(Collectors.toList());
+ }
+
+ private static Object[][] columnValues(List<Map<String, Object>> list, List<String> columnNames) {
+ return columnNames.stream()
+ .map(columnName -> columnValues(list, columnName))
+ .collect(Collectors.toList())
+ .toArray(new Object[0][0]);
+ }
+
+ private static Object[] columnValues(List<Map<String, Object>> list, String columnName) {
+ return list.stream()
+ .map(map -> map.getOrDefault(columnName, null))
+ .toArray();
+ }
+
+ private static List<Class<?>> columnTypes(Object[][] columnValuesList) {
+ return Arrays.stream(columnValuesList)
+ .map(Table::columnType)
+ .collect(Collectors.toList());
+ }
+
+ /**
+ * Determine the column type.
+ *
+ * @param columnValues column values
+ * @return class of the first non-null value
+ */
+ private static Class<?> columnType(Object[] columnValues) {
+ for (final Object columnValue : columnValues) {
+ if (columnValue != null) {
+ return columnValue.getClass();
+ }
+ }
+
+ throw new IllegalArgumentException("No column value found!!!");
+ }
+
+ /**
+ * Transposes the given array, swapping rows with columns. The given array might contain arrays as elements that are
+ * not all of the same length. The returned array will have {@code null} values at those places.
+ *
+ * @param <T> the type of the array
+ * @param array the array
+ * @return the transposed array
+ * @throws NullPointerException if the given array is {@code null}
+ */
+ public static <T> T[][] transpose(final T[][] array) {
+ requireNonNull(array);
+ // get y count
+ final int yCount = Arrays.stream(array).mapToInt(a -> a.length).max().orElse(0);
+ final int xCount = array.length;
+ final Class<?> componentType = array.getClass().getComponentType().getComponentType();
+ @SuppressWarnings("unchecked") final T[][] newArray = (T[][]) Array.newInstance(componentType, yCount, xCount);
+ for (int x = 0; x < xCount; x++) {
+ for (int y = 0; y < yCount; y++) {
+ if (array[x] == null || y >= array[x].length) {
+ break;
+ }
+ newArray[y][x] = array[x][y];
+ }
+ }
+ return newArray;
+ }
+}
diff --git a/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/util/MapBuilder.java b/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/util/MapBuilder.java
new file mode 100644
index 0000000..ab60a5e
--- /dev/null
+++ b/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/util/MapBuilder.java
@@ -0,0 +1,35 @@
+package org.apache.freemarker.generator.base.util;
+
+import java.util.HashMap;
+
+public class MapBuilder {
+
+ public static HashMap<String, Object> toHashMap(Object... data) {
+
+ final HashMap<String, Object> result = new HashMap<>();
+
+ if (data.length % 2 != 0) {
+ throw new IllegalArgumentException("Odd number of arguments");
+ }
+
+ String currKey = null;
+ int step = -1;
+
+ for (Object value : data) {
+ step++;
+ switch (step % 2) {
+ case 0:
+ if (value == null) {
+ throw new IllegalArgumentException("Null key value");
+ }
+ currKey = value.toString();
+ continue;
+ case 1:
+ result.put(currKey, value);
+ break;
+ }
+ }
+
+ return result;
+ }
+}
diff --git a/freemarker-generator-base/src/test/java/org/apache/freemarker/generator/table/TableTest.java b/freemarker-generator-base/src/test/java/org/apache/freemarker/generator/table/TableTest.java
new file mode 100644
index 0000000..82b560a
--- /dev/null
+++ b/freemarker-generator-base/src/test/java/org/apache/freemarker/generator/table/TableTest.java
@@ -0,0 +1,59 @@
+package org.apache.freemarker.generator.table;
+
+import org.apache.freemarker.generator.base.table.Table;
+import org.apache.freemarker.generator.base.util.MapBuilder;
+import org.junit.Test;
+
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+
+import static junit.framework.TestCase.assertEquals;
+
+public class TableTest {
+
+ public final List<Map<String, Object>> books = Arrays.asList(
+ MapBuilder.toHashMap(
+ "Book ID", "1",
+ "Book Name", "Computer Architecture",
+ "Category", "Computers",
+ "In Stock", true,
+ "Price", 125.60),
+ MapBuilder.toHashMap(
+ "Book ID", "2",
+ "Book Name", "Asp.Net 4 Blue Book",
+ "Category", "Programming",
+ "In Stock", null,
+ "Price", 56),
+ MapBuilder.toHashMap(
+ "Book ID", "3",
+ "Book Name", "Popular Science",
+ "Category", "Science",
+ "Price", 210.40)
+ );
+
+ @Test
+ public void shouldConvertFromMapList() {
+ final Table table = Table.fromMaps(books);
+
+ assertEquals(5, table.getNrOfColumns());
+ assertEquals(3, table.getNrOfRows());
+ assertEquals("1", table.getRow(0).getValue("Book ID"));
+ assertEquals("2", table.getRow(1).getValue("Book ID"));
+ assertEquals("3", table.getRow(2).getValue("Book ID"));
+ }
+
+ @Test
+ public void shouldConvertFromEmptyMapList() {
+ final Table table = Table.fromMaps(new ArrayList<>());
+
+ assertEquals(0, table.getNrOfColumns());
+ assertEquals(0, table.getNrOfRows());
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void shouldConvertFromNullpList() {
+ Table.fromMaps(null);
+ }
+}
diff --git a/freemarker-generator-cli/site/template/nginx/nginx.conf.ftl b/freemarker-generator-cli/site/template/nginx/nginx.conf.ftl
index 72a1b3a..7cb9f3d 100644
--- a/freemarker-generator-cli/site/template/nginx/nginx.conf.ftl
+++ b/freemarker-generator-cli/site/template/nginx/nginx.conf.ftl
@@ -14,7 +14,7 @@
specific language governing permissions and limitations
under the License.
-->
-# == nginx-conf =============================================================
+# == nginx-conf ==============================================================
server {
listen ${NGINX_PORT!"80"};
server_name ${NGINX_HOSTNAME!"127.0.0.1"};
diff --git a/freemarker-generator-cli/src/site/markdown/cli/usage/transforming-directories.md b/freemarker-generator-cli/src/site/markdown/cli/usage/transforming-directories.md
index 62af750..7a65dc0 100644
--- a/freemarker-generator-cli/src/site/markdown/cli/usage/transforming-directories.md
+++ b/freemarker-generator-cli/src/site/markdown/cli/usage/transforming-directories.md
@@ -5,16 +5,45 @@ FreeMarker CLI supports the transformation of directories
* Transform an input directory recursively into an output directory
* If a template has a ".ftl" extension this extension will be removed after processing
* Only a single directory is support
-* Currently no inclusion / exclusion pattern for templates are supported
+* Currently no inclusion / exclusion patterns for templates are supported
+
+The following sample files are used
+
+* template/application.properties
+* template/nginx/nginx.conf.ftl
+
+```
+appassembler> tree site/template/
+site/template/
+|-- application.properties
+`-- nginx
+ `-- nginx.conf.ftl
+
+# == application.properties ==================================================
+server.name=${NGINX_HOSTNAME!"127.0.0.1"}
+server.logs=${NGINX_LOGS!"/var/log/nginx"}
+```
+
+```
+# == nginx-conf ==============================================================
+server {
+ listen ${NGINX_PORT!"80"};
+ server_name ${NGINX_HOSTNAME!"127.0.0.1"};
+
+ root ${NGINX_WEBROOT!"/usr/share/nginx/www"};
+ index index.htm;
+```
### Transform Template Directory To STDOUT
+If no output directory is provided all output is written to `stdout`
+
```
bin/freemarker-cli -t site/template/
# == application.properties ==================================================
server.name=localhost
server.logs=/var/log/nginx
-# == nginx-conf =============================================================
+# == nginx-conf ==============================================================
server {
listen 80;
server_name 127.0.0.1;
@@ -26,21 +55,30 @@ server {
### Transform Template Directory To Output Directory
+The transformed templates are written to an `out` directory
+
+* `nginx.conf.ftl` was changed to `nginx.conf" during the transformation
+
```
-bin/freemarker-cli -t site/template/ -o out; ls -l out
-total 8
--rw-r--r-- 1 sgoeschl staff 128 May 30 20:02 application.properties
-drwxr-xr-x 3 sgoeschl staff 96 May 30 20:02 nginx
+bin/freemarker-cli -t site/template/ -o out; tree out
+out
+|-- application.properties
+`-- nginx
+ `-- nginx.conf
+
+1 directory, 2 files
```
### Use Command Line Parameters
+A user-supplied parameter `NGINX_HOSTNAME` is used to render the templates
+
```
bin/freemarker-cli -t site/template/ -P NGINX_HOSTNAME=localhost
# == application.properties ==================================================
server.name=localhost
server.logs=/var/log/nginx
-# == nginx-conf =============================================================
+# == nginx-conf ==============================================================
server {
listen 80;
server_name localhost;
@@ -52,13 +90,18 @@ server {
### Use Environment Variables
+All environment variables can be copied to the top-level data model by providing `-m env:///`
+
+* `-m` or `-data-model` creates a data model
+* `env:///` is an URI referencing all environment variables
+
```
export NGINX_PORT=8080
bin/freemarker-cli -t site/template/ -m env:///
# == application.properties ==================================================
server.name=localhost
server.logs=/var/log/nginx
-# == nginx-conf =============================================================
+# == nginx-conf ==============================================================
server {
listen 8080;
server_name 127.0.0.1;
@@ -70,13 +113,15 @@ server {
### Use Environment File
+Instead of environment variables an environment file (aka properties file) can be used
+
```
echo "NGINX_PORT=8080" > nginx.env
bin/freemarker-cli -t site/template/ -m nginx.env
# == application.properties ==================================================
server.name=localhost
server.logs=/var/log/nginx
-# == nginx-conf =============================================================
+# == nginx-conf ==============================================================
server {
listen 8080;
server_name 127.0.0.1;
@@ -88,13 +133,15 @@ server {
### Use JSON File
+Another option is passing the information as JSON file
+
```
echo '{"NGINX_PORT":"8443","NGINX_HOSTNAME":"localhost"}' > nginx.json
bin/freemarker-cli -t site/template/ -m nginx.json
# == application.properties ==================================================
server.name=localhost
server.logs=/var/log/nginx
-# == nginx-conf =============================================================
+# == nginx-conf ==============================================================
server {
listen 8443;
server_name localhost;
@@ -106,13 +153,15 @@ server {
### Use YAML File
+Yet another option is using a YAML file
+
```
echo -e "- NGINX_PORT": "\"8443\"\n- NGINX_HOSTNAME": "localhost" > nginx.yaml
bin/freemarker-cli -t site/template/ -m nginx.yaml
# == application.properties ==================================================
server.name=localhost
server.logs=/var/log/nginx
-# == nginx-conf =============================================================
+# == nginx-conf ==============================================================
server {
listen 8443;
server_name localhost;
@@ -124,15 +173,18 @@ server {
### Use Environment Variable With JSON Payload
+In the cloud it is common to pass JSON configuration as environment variable
+
+* `env:///NGINX_CONF` selects the `NGINX_CONF` environment variable
+* `#mimetype=application/json` defines that JSON content is parsed
+
```
-export NGINX_CONF='{"NGINX_PORT":"8443","NGINX_HOSTNAME":"somehost"}'
-echo $NGINX_CONF
-{"NGINX_PORT":"8443","NGINX_HOSTNAME":"localhost"}
+export NGINX_CONF='{"NGINX_PORT":"8443","NGINX_HOSTNAME":"localhost"}'
bin/freemarker-cli -t site/template/ -m env:///NGINX_CONF#mimetype=application/json
# == application.properties ==================================================
server.name=localhost
server.logs=/var/log/nginx
-# == nginx-conf =============================================================
+# == nginx-conf ==============================================================
server {
listen 8443;
server_name localhost;
@@ -140,4 +192,26 @@ server {
root /usr/share/nginx/www;
index index.htm;
}
-```
\ No newline at end of file
+```
+
+### Overriding Values From The Command Line
+
+For testing purpose it is useful to override certain settings
+
+```
+export NGINX_CONF='{"NGINX_PORT":"8443","NGINX_HOSTNAME":"localhost"}'
+bin/freemarker-cli -t site/template/ -PNGINX_HOSTNAME=www.mydomain.com -m env:///NGINX_CONF#mimetype=application/json
+# == application.properties ==================================================
+server.name=www.mydomain.com
+server.logs=/var/log/nginx
+# == nginx-conf ==============================================================
+server {
+ listen 8443;
+ server_name www.mydomain.com;
+
+ root /usr/share/nginx/www;
+ index index.htm;
+}
+```
+
+Please note that this only works for "top-level" variables, i.e. mimicking enviroment variables or property files.
\ No newline at end of file
diff --git a/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/dataframe/DataFrameTool.java b/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/dataframe/DataFrameTool.java
index d34e705..35fcdfa 100644
--- a/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/dataframe/DataFrameTool.java
+++ b/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/dataframe/DataFrameTool.java
@@ -24,8 +24,10 @@ import de.unknownreality.dataframe.transform.ColumnDataFrameTransform;
import de.unknownreality.dataframe.transform.CountTransformer;
import org.apache.commons.csv.CSVParser;
import org.apache.commons.csv.CSVRecord;
+import org.apache.freemarker.generator.base.table.Table;
import org.apache.freemarker.generator.base.util.Validate;
+import java.io.IOException;
import java.io.Writer;
import java.util.HashMap;
import java.util.List;
@@ -60,25 +62,36 @@ public class DataFrameTool {
* @return data frame
*/
public DataFrame toDataFrame(CSVParser csvParser) {
- Validate.isFalse(csvParser.getHeaderNames().isEmpty(), "CSV headers expected");
-
- final List<String> headerNames = csvParser.getHeaderNames();
+ try {
+ final List<String> headerNames = csvParser.getHeaderNames();
+ final DataFrameBuilder builder = DataFrameBuilder.create();
+ final List<CSVRecord> records = csvParser.getRecords();
+ final CSVRecord firstRecord = records.get(0);
+
+ // build dataframe with headers
+ if (headerNames != null && !headerNames.isEmpty()) {
+ headerNames.forEach(builder::addStringColumn);
+ } else {
+ for (int i = 0; i < firstRecord.size(); i++) {
+ builder.addStringColumn(getAlpha(i + 1));
+ }
+ }
- // build dataframe with headers
- final DataFrameBuilder builder = DataFrameBuilder.create();
- headerNames.forEach(builder::addStringColumn);
- final DataFrame dataFrame = builder.build();
+ final DataFrame dataFrame = builder.build();
- // populate rows
- final String[] currValues = new String[headerNames.size()];
- for (CSVRecord csvRecord : csvParser) {
- for (int i = 0; i < currValues.length; i++) {
- currValues[i] = csvRecord.get(i);
+ // populate rows
+ final String[] currValues = new String[firstRecord.size()];
+ for (CSVRecord csvRecord : records) {
+ for (int i = 0; i < currValues.length; i++) {
+ currValues[i] = csvRecord.get(i);
+ }
+ dataFrame.append(currValues);
}
- dataFrame.append(currValues);
- }
- return dataFrame;
+ return dataFrame;
+ } catch (IOException e) {
+ throw new RuntimeException("Unable to create DataFrame", e);
+ }
}
/**
@@ -90,28 +103,30 @@ public class DataFrameTool {
* @return data frame
*/
public DataFrame toDataFrame(List<Map<String, Object>> list) {
- if (list.isEmpty()) {
+ if (list == null || list.isEmpty()) {
return DataFrameBuilder.createDefault();
}
- final Map<String, Object> firstRow = list.get(0);
+ final Table table = Table.fromMaps(list);
// build dataframe with headers
final DataFrameBuilder builder = DataFrameBuilder.create();
- firstRow.keySet().forEach(builder::addStringColumn);
+ for (int i = 0; i < table.getColumnNames().length; i++) {
+ addColumn(builder, table.getColumnNames()[i], table.getColumnTypes()[i]);
+ }
final DataFrame dataFrame = builder.build();
// populate rows
- list.stream()
- .map(Map::values)
- .map(values -> values.toArray(new Comparable[0]))
- .forEach(dataFrame::append);
+ for (int i = 0; i < table.getNrOfRows(); i++) {
+ final Object[] values = table.getRowValues(i);
+ dataFrame.append(toComparables(values));
+ }
return dataFrame;
}
/**
- * Provide a map with predefined sort orders to be used by templates.
+ * Provide a convinience map with predefined sort orders to be used by templates.
*
* @return available sort orders
*/
@@ -123,7 +138,7 @@ public class DataFrameTool {
}
/**
- * Provide a map with predefined transformers.
+ * Provide a convinience map with predefined transformers.
*
* @return available transformers
*/
@@ -148,8 +163,51 @@ public class DataFrameTool {
return "Bridge to nRo/DataFrame (see https://github.com/nRo/DataFrame)";
}
+ private static DataFrameBuilder addColumn(DataFrameBuilder builder, String name, Class<?> clazz) {
+ switch (clazz.getName()) {
+ case "java.lang.Boolean":
+ return builder.addBooleanColumn(name);
+ case "java.lang.Byte":
+ return builder.addByteColumn(name);
+ case "java.lang.Double":
+ return builder.addDoubleColumn(name);
+ case "java.lang.Float":
+ return builder.addFloatColumn(name);
+ case "java.lang.Integer":
+ return builder.addIntegerColumn(name);
+ case "java.lang.Long":
+ return builder.addLongColumn(name);
+ case "java.lang.Short":
+ return builder.addShortColumn(name);
+ case "java.lang.String":
+ return builder.addStringColumn(name);
+ default:
+ throw new RuntimeException("Unable to add colum for the following type: " + clazz.getName());
+ }
+ }
+
private static CountTransformer countTransformer(boolean ignoreNA) {
return new CountTransformer(ignoreNA);
}
+ private static Comparable<?>[] toComparables(Object[] values) {
+ final Comparable<?>[] comparables = new Comparable<?>[values.length];
+ for (int i = 0; i < values.length; i++) {
+ comparables[i] = (Comparable<?>) values[i];
+ }
+ return comparables;
+ }
+
+ private static String getAlpha(int num) {
+ String result = "";
+ while (num > 0) {
+ num--; // 1 => a, not 0 => a
+ int remainder = num % 26;
+ char digit = (char) (remainder + 65);
+ result = digit + result;
+ num = (num - remainder) / 26;
+ }
+ return result;
+ }
+
}
diff --git a/freemarker-generator-tools/src/test/java/org/apache/freemarker/generator/tools/dataframe/DataFrameToolTest.java b/freemarker-generator-tools/src/test/java/org/apache/freemarker/generator/tools/dataframe/DataFrameToolTest.java
index 20c072d..dd6bd70 100644
--- a/freemarker-generator-tools/src/test/java/org/apache/freemarker/generator/tools/dataframe/DataFrameToolTest.java
+++ b/freemarker-generator-tools/src/test/java/org/apache/freemarker/generator/tools/dataframe/DataFrameToolTest.java
@@ -31,36 +31,52 @@ import static org.apache.commons.csv.CSVFormat.DEFAULT;
public class DataFrameToolTest {
- private static final String CSV_WITH_HEADER = "GENE_ID;FPKM;CHR\n" +
- "A;5;1\n" +
+ private static final String CSV_WITHOUT_HEADER = "A;5;1\n" +
"B;4;2\n" +
"C;6;3\n" +
"D;6;1";
+ private static final String CSV_WITH_HEADER = "GENE_ID;FPKM;CHR\n" +
+ CSV_WITHOUT_HEADER;
+
private static final String JSON_ARRAY = "[\n" +
" {\n" +
" \"Book ID\": \"1\",\n" +
" \"Book Name\": \"Computer Architecture\",\n" +
" \"Category\": \"Computers\",\n" +
- " \"Price\": \"125.60\"\n" +
+ " \"In Stock\": true,\n" +
+ " \"Price\": 125.60\n" +
" },\n" +
" {\n" +
" \"Book ID\": \"2\",\n" +
" \"Book Name\": \"Asp.Net 4 Blue Book\",\n" +
" \"Category\": \"Programming\",\n" +
- " \"Price\": \"56.00\"\n" +
+ " \"In Stock\": null,\n" +
+ " \"Price\": 56.00\n" +
" },\n" +
" {\n" +
" \"Book ID\": \"3\",\n" +
" \"Book Name\": \"Popular Science\",\n" +
" \"Category\": \"Science\",\n" +
- " \"Price\": \"210.40\"\n" +
+ " \"Price\": 210.40\n" +
" }\n" +
"]";
// === CSV ==============================================================
@Test
+ public void shouldParseCsvFileWithoutHeader() {
+ final CSVParser csvParser = csvParser(CSV_WITHOUT_HEADER, DEFAULT.withDelimiter(';'));
+ final DataFrame dataFrame = dataFrameTool().toDataFrame(csvParser);
+
+ assertEquals(3, dataFrame.getColumns().size());
+ assertEquals(4, dataFrame.getRows().size());
+ assertEquals("A", dataFrame.getRow(0).get(0));
+ assertEquals("4", dataFrame.getRow(1).get(1));
+ assertEquals("3", dataFrame.getRow(2).get(2));
+ }
+
+ @Test
public void shouldParseCsvFileWithHeader() {
final CSVParser csvParser = csvParser(CSV_WITH_HEADER, DEFAULT.withHeader().withDelimiter(';'));
final DataFrame dataFrame = dataFrameTool().toDataFrame(csvParser);
@@ -68,6 +84,8 @@ public class DataFrameToolTest {
assertEquals(3, dataFrame.getColumns().size());
assertEquals(4, dataFrame.getRows().size());
assertEquals("A", dataFrame.getColumn("GENE_ID").get(0));
+ assertEquals("4", dataFrame.getColumn("FPKM").get(1));
+ assertEquals("3", dataFrame.getColumn("CHR").get(2));
}
// === JSON =============================================================
@@ -76,12 +94,14 @@ public class DataFrameToolTest {
@SuppressWarnings("unchecked")
public void shouldParseJsonTable() {
final String columnName = "Book ID";
- final List<Map<String, Object>> json = (List) gsonTool().parse(JSON_ARRAY);
+ final List<Map<String, Object>> json = (List<Map<String, Object>>) gsonTool().parse(JSON_ARRAY);
final DataFrame dataFrame = dataFrameTool().toDataFrame(json);
- assertEquals(4, dataFrame.getColumns().size());
+ assertEquals(5, dataFrame.getColumns().size());
assertEquals(3, dataFrame.getRows().size());
assertEquals("1", dataFrame.getColumn(columnName).get(0));
+ assertEquals("2", dataFrame.getColumn(columnName).get(1));
+ assertEquals("3", dataFrame.getColumn(columnName).get(2));
}
private DataFrameTool dataFrameTool() {