You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@freemarker.apache.org by sg...@apache.org on 2020/06/01 06:20:22 UTC
[freemarker-generator] branch FREEMARKER-144 updated:
FREEMARKER-144 Proof Of Concept for providing DataFrames
This is an automated email from the ASF dual-hosted git repository.
sgoeschl pushed a commit to branch FREEMARKER-144
in repository https://gitbox.apache.org/repos/asf/freemarker-generator.git
The following commit(s) were added to refs/heads/FREEMARKER-144 by this push:
new fd2bfe8 FREEMARKER-144 Proof Of Concept for providing DataFrames
fd2bfe8 is described below
commit fd2bfe85bee2dc1d2afeef1c328d97f032efd341
Author: Siegfried Goeschl <si...@gmail.com>
AuthorDate: Mon Jun 1 08:20:06 2020 +0200
FREEMARKER-144 Proof Of Concept for providing DataFrames
---
.../generator/cli/config/DataModelSupplier.java | 2 +-
.../freemarker/generator/cli/ExamplesTest.java | 2 +-
.../freemarker/generator/cli/ManualTest.java | 3 +-
.../templates/dataframe/example.ftl | 45 ++++++++
.../templates/dataframe/html/print.ftl | 49 ++++----
.../templates/json/csv/swagger-endpoints.ftl | 2 +-
.../templates/json/yaml/transform.ftl | 2 +-
.../generator/tools/dataframe/DataFrameTool.java | 125 ++++++++++++++++-----
.../freemarker/generator/tools/gson/GsonTool.java | 65 +++++++++--
.../generator/tools/jsoup/JsoupTool.java | 5 +-
.../generator/tools/system/SystemTool.java | 2 +-
.../tools/commonscsv/CommonsCSVToolTest.java | 1 -
.../tools/dataframe/DataFrameToolTest.java | 80 ++++++++-----
.../generator/tools/gson/GsonToolTest.java | 57 +++++++---
14 files changed, 321 insertions(+), 119 deletions(-)
diff --git a/freemarker-generator-cli/src/main/java/org/apache/freemarker/generator/cli/config/DataModelSupplier.java b/freemarker-generator-cli/src/main/java/org/apache/freemarker/generator/cli/config/DataModelSupplier.java
index d8e3491..e09a74c 100644
--- a/freemarker-generator-cli/src/main/java/org/apache/freemarker/generator/cli/config/DataModelSupplier.java
+++ b/freemarker-generator-cli/src/main/java/org/apache/freemarker/generator/cli/config/DataModelSupplier.java
@@ -86,7 +86,7 @@ public class DataModelSupplier implements Supplier<Map<String, Object>> {
private Map<String, Object> fromJson(DataSource dataSource, boolean isExplodedDataModel) {
final GsonTool gsonTool = new GsonTool();
- final Map<String, Object> map = gsonTool.parse(dataSource);
+ final Map<String, Object> map = gsonTool.toMap(dataSource);
return fromMap(dataSource.getName(), map, isExplodedDataModel);
}
diff --git a/freemarker-generator-cli/src/test/java/org/apache/freemarker/generator/cli/ExamplesTest.java b/freemarker-generator-cli/src/test/java/org/apache/freemarker/generator/cli/ExamplesTest.java
index fedc66c..681dd08 100644
--- a/freemarker-generator-cli/src/test/java/org/apache/freemarker/generator/cli/ExamplesTest.java
+++ b/freemarker-generator-cli/src/test/java/org/apache/freemarker/generator/cli/ExamplesTest.java
@@ -109,7 +109,7 @@ public class ExamplesTest extends AbstractMainTest {
assertValid(execute("-i ${JsoupTool.parse(DataSources.first).select(\"a\")[0]} site/sample/html/dependencies.html"));
assertValid(execute("-i ${GsonTool.toJson(YamlTool.parse(DataSources.get(0)))} site/sample/yaml/swagger-spec.yaml"));
assertValid(execute("-i ${GsonTool.toJson(yaml)} -m yaml=site/sample/yaml/swagger-spec.yaml"));
- assertValid(execute("-i ${YamlTool.toYaml(GsonTool.parse(DataSources.get(0)))} site/sample/json/swagger-spec.json"));
+ assertValid(execute("-i ${YamlTool.toYaml(GsonTool.toMap(DataSources.get(0)))} site/sample/json/swagger-spec.json"));
assertValid(execute("-i ${YamlTool.toYaml(json)} -m json=site/sample/json/swagger-spec.json"));
}
diff --git a/freemarker-generator-cli/src/test/java/org/apache/freemarker/generator/cli/ManualTest.java b/freemarker-generator-cli/src/test/java/org/apache/freemarker/generator/cli/ManualTest.java
index 4a47b1f..acdfd85 100644
--- a/freemarker-generator-cli/src/test/java/org/apache/freemarker/generator/cli/ManualTest.java
+++ b/freemarker-generator-cli/src/test/java/org/apache/freemarker/generator/cli/ManualTest.java
@@ -48,7 +48,8 @@ public class ManualTest {
// private static final String CMD = "-b ./src/test -t templates/demo.ftl -m env=./site/sample/properties/user_0001/user.properties";
// private static final String CMD = "-b ./src/test -t templates/demo.ftl -m ./site/sample/properties/user_0001/user.properties";
// private static final String CMD = "-b ./src/test --data-model post=https://jsonplaceholder.typicode.com/posts/2 -t templates/info.ftl";
- private static final String CMD = "-b ./src/test -t templates/dataframe/html/print.ftl ./site/sample/csv/data_join_a.csv ./site/sample/csv/data_join_b.csv";
+ // private static final String CMD = "-b ./src/test -t templates/dataframe/html/print.ftl ./site/sample/csv/data_join_a.csv ./site/sample/csv/data_join_b.csv";
+ private static final String CMD = "-b ./src/test -t templates/dataframe/example.ftl https://raw.githubusercontent.com/nRo/DataFrame/master/src/test/resources/users.csv";
public static void main(String[] args) {
diff --git a/freemarker-generator-cli/templates/dataframe/example.ftl b/freemarker-generator-cli/templates/dataframe/example.ftl
new file mode 100644
index 0000000..a351297
--- /dev/null
+++ b/freemarker-generator-cli/templates/dataframe/example.ftl
@@ -0,0 +1,45 @@
+<#ftl output_format="plainText">
+<#--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+<#assign cvsFormat = CSVTool.formats["DEFAULT"].withHeader().withDelimiter(';')>
+<#assign csvParser = CSVTool.parse(DataSources.get(0), cvsFormat)>
+<#assign users = DataFrameTool.toDataFrame(csvParser)>
+
+Original Data
+=============================================================================
+${DataFrameTool.print(users)}
+
+Select By Name & Country
+=============================================================================
+<#assign country = "Germany">
+${DataFrameTool.print(users
+ .select("(name == 'Schmitt' || name == 'Meier') && country == '${country}'")
+ .sort("name", DataFrameTool.sortOrder["ASCENDING"]))}
+
+Head of Users
+=============================================================================
+${DataFrameTool.print(users.head(2))}
+
+Count Column Values
+=============================================================================
+${DataFrameTool.print(users.getColumn("country").transform(DataFrameTool.countTransformer(false)))}
+
+Group By Age & Country
+=============================================================================
+${DataFrameTool.print(users.groupBy("age", "country").sort("age"))}
+
+
diff --git a/freemarker-generator-cli/templates/dataframe/html/print.ftl b/freemarker-generator-cli/templates/dataframe/html/print.ftl
index f34e745..e17d3a9 100644
--- a/freemarker-generator-cli/templates/dataframe/html/print.ftl
+++ b/freemarker-generator-cli/templates/dataframe/html/print.ftl
@@ -15,51 +15,40 @@
specific language governing permissions and limitations
under the License.
-->
-<#assign dataSource = DataSources.get(0)>
-<#assign name = dataSource.name>
-<#assign date = .now?iso_utc>
-<#assign dataFrame = DataFrameTool.parse(dataSource, csvReader())>
+<#assign cvsFormat = CSVTool.formats["DEFAULT"].withHeader().withDelimiter(';')>
+<#assign csvParser = CSVTool.parse(DataSources.get(0), cvsFormat)>
+<#assign dataFrame = DataFrameTool.toDataFrame(csvParser)>
<#--------------------------------------------------------------------------->
<!DOCTYPE html>
<html>
<head>
- <title>${name}</title>
+ <title>DataFrame</title>
<meta charset="UTF-8">
<meta name="viewport" content="width=device-width, initial-scale=1">
<link rel="stylesheet" href="https://maxcdn.bootstrapcdn.com/bootstrap/4.1.3/css/bootstrap.min.css">
</head>
<body>
<div class="container-fluid">
- <h1>DataFrame Test <small>${name}, ${date}</small></h1>
- <table class="table table-striped">
- <@writeHeader dataFrame/>
- <@writeRows dataFrame/>
- </table>
+ <h1>DataFrame</h1>
+ <@writeDataFrame dataFrame/>
</div>
</body>
</html>
<#--------------------------------------------------------------------------->
-<#function csvReader>
- <#return DataFrameTool.csvReaderBuilder.containsHeader(true).withSeparator(CSV_IN_DELIMITER!';').build()>
-</#function>
-
-<#--------------------------------------------------------------------------->
-<#macro writeHeader dataFrame>
- <tr>
- <#list dataFrame.columns as column>
- <th>${column.name}</th>
- </#list>
- </tr>
-</#macro>
-
-<#--------------------------------------------------------------------------->
-<#macro writeRows dataFrame>
- <#list dataFrame.iterator() as row>
+<#macro writeDataFrame dataFrame>
+ <table class="table table-striped">
<tr>
- <#list 0..row.size()-1 as idx>
- <td>${row.getString(idx)}</td>
+ <#list dataFrame.columns as column>
+ <th>${column.name}</th>
</#list>
</tr>
- </#list>
-</#macro>
\ No newline at end of file
+ <#list dataFrame.iterator() as row>
+ <tr>
+ <#list 0..row.size()-1 as idx>
+ <td>${row.getString(idx)}</td>
+ </#list>
+ </tr>
+ </#list>
+ </table>
+</#macro>
diff --git a/freemarker-generator-cli/templates/json/csv/swagger-endpoints.ftl b/freemarker-generator-cli/templates/json/csv/swagger-endpoints.ftl
index 5f9d6a0..2c2121e 100644
--- a/freemarker-generator-cli/templates/json/csv/swagger-endpoints.ftl
+++ b/freemarker-generator-cli/templates/json/csv/swagger-endpoints.ftl
@@ -15,7 +15,7 @@
specific language governing permissions and limitations
under the License.
-->
-<#assign map = GsonTool.parse(DataSources.get(0))>
+<#assign map = GsonTool.toMap(DataSources.get(0))>
<#assign basePath = map.basePath!"/">
<#assign paths = map.paths!{}>
diff --git a/freemarker-generator-cli/templates/json/yaml/transform.ftl b/freemarker-generator-cli/templates/json/yaml/transform.ftl
index 80c8d2f..d7c36b6 100644
--- a/freemarker-generator-cli/templates/json/yaml/transform.ftl
+++ b/freemarker-generator-cli/templates/json/yaml/transform.ftl
@@ -14,4 +14,4 @@
specific language governing permissions and limitations
under the License.
-->
-${YamlTool.toYaml(GsonTool.parse(DataSources.get(0)))}
\ No newline at end of file
+${YamlTool.toYaml(GsonTool.toMap(DataSources.get(0)))}
\ No newline at end of file
diff --git a/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/dataframe/DataFrameTool.java b/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/dataframe/DataFrameTool.java
index 749e005..18db73d 100644
--- a/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/dataframe/DataFrameTool.java
+++ b/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/dataframe/DataFrameTool.java
@@ -17,51 +17,122 @@
package org.apache.freemarker.generator.tools.dataframe;
import de.unknownreality.dataframe.DataFrame;
-import de.unknownreality.dataframe.csv.CSVReader;
-import de.unknownreality.dataframe.csv.CSVReaderBuilder;
-import de.unknownreality.dataframe.io.FileFormat;
-import org.apache.freemarker.generator.base.datasource.DataSource;
+import de.unknownreality.dataframe.DataFrameBuilder;
+import de.unknownreality.dataframe.DataFrameWriter;
+import de.unknownreality.dataframe.sort.SortColumn.Direction;
+import de.unknownreality.dataframe.transform.CountTransformer;
+import org.apache.commons.csv.CSVParser;
+import org.apache.commons.csv.CSVRecord;
+import org.apache.freemarker.generator.base.util.Validate;
-import java.io.IOException;
-import java.io.InputStream;
+import java.io.Writer;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import static java.util.Objects.requireNonNull;
+import static org.apache.freemarker.generator.base.FreeMarkerConstants.Model.FREEMARKER_WRITER;
+
+/**
+ * Create and manipulate data frame (tabular data structure). Data frames allow
+ * easy manipulation and transformation of data, e.g. joining two data frames.
+ * For more information see <a href="https://github.com/nRo/DataFrame">nRo/DataFrame</a>.
+ */
public class DataFrameTool {
+ /** Underlying FreeMarker writer for rendering templates */
+ private final Writer writer;
+
+ public DataFrameTool() {
+ this.writer = null;
+ }
+
+ public DataFrameTool(Map<String, Object> settings) {
+ requireNonNull(settings);
+ this.writer = (Writer) settings.getOrDefault(FREEMARKER_WRITER, null);
+ }
+
/**
- * Create a data frame.
+ * Create a data frame from Apache Commons CSVParser.
*
- * @param dataSource data source
+ * @param csvParser CSV Parser
* @return data frame
*/
- public DataFrame parse(DataSource dataSource) {
- try (InputStream is = dataSource.getUnsafeInputStream()) {
- final DataFrame dataFrame = DataFrame.load(is, FileFormat.CSV);
- dataFrame.setName(dataSource.getName());
- return dataFrame;
- } catch (IOException e) {
- throw new RuntimeException("Failed to parse data source: " + dataSource, e);
+ public DataFrame toDataFrame(CSVParser csvParser) {
+ Validate.isFalse(csvParser.getHeaderNames().isEmpty(), "CSV headers expected");
+
+ final List<String> headerNames = csvParser.getHeaderNames();
+
+ // build dataframe with headers
+ final DataFrameBuilder builder = DataFrameBuilder.create();
+ headerNames.forEach(builder::addStringColumn);
+ final DataFrame dataFrame = builder.build();
+
+ // populate rows
+ final String[] currValues = new String[headerNames.size()];
+ for (CSVRecord csvRecord : csvParser) {
+ for (int i = 0; i < currValues.length; i++) {
+ currValues[i] = csvRecord.get(i);
+ }
+ dataFrame.append(currValues);
}
+
+ return dataFrame;
}
/**
- * Create a data frame.
+ * Create a data frame from a list of maps. It is assumed
+ * that the map represent tabular data without missing
+ * values.
*
- * @param dataSource data source
- * @param csvReader CSV format specification to use
+ * @param list map to build the data frame
* @return data frame
*/
- public DataFrame parse(DataSource dataSource, CSVReader csvReader) {
- try (InputStream is = dataSource.getUnsafeInputStream()) {
- final DataFrame dataFrame = DataFrame.load(is, csvReader);
- dataFrame.setName(dataSource.getName());
- return dataFrame;
- } catch (IOException e) {
- throw new RuntimeException("Failed to parse data source: " + dataSource, e);
+ public DataFrame toDataFrame(List<Map<String, Object>> list) {
+ if (list.isEmpty()) {
+ return DataFrameBuilder.createDefault();
}
+
+ final Map<String, Object> firstRow = list.get(0);
+
+ // build dataframe with headers
+ final DataFrameBuilder builder = DataFrameBuilder.create();
+ firstRow.keySet().forEach(builder::addStringColumn);
+ final DataFrame dataFrame = builder.build();
+
+ // populate rows
+ list.stream()
+ .map(Map::values)
+ .map(values -> values.toArray(new Comparable[0]))
+ .forEach(dataFrame::append);
+
+ return dataFrame;
+ }
+
+ /**
+ * Provide a map with predefined sort orders to be used by templates.
+ *
+ * @return available sort orders
+ */
+ public Map<String, Direction> getSortOrder() {
+ final Map<String, Direction> result = new HashMap<>();
+ result.put(Direction.Ascending.name().toUpperCase(), Direction.Ascending);
+ result.put(Direction.Descending.name().toUpperCase(), Direction.Descending);
+ return result;
+ }
+
+ public CountTransformer countTransformer(boolean ignoreNA) {
+ return new CountTransformer(ignoreNA);
}
- public CSVReaderBuilder getCsvReaderBuilder() {
- return CSVReaderBuilder.create();
+ /**
+ * Print the <code>DataFrame</code> to the FreeMarker writer.
+ *
+ * @param dataFrame data frame
+ */
+ public void print(DataFrame dataFrame) {
+ Validate.notNull(writer, "No writer available");
+ DataFrameWriter.write(writer, dataFrame, DataFrameWriter.DEFAULT_PRINT_FORMAT);
}
@Override
diff --git a/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/gson/GsonTool.java b/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/gson/GsonTool.java
index 6a864eb..ba61074 100644
--- a/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/gson/GsonTool.java
+++ b/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/gson/GsonTool.java
@@ -25,25 +25,68 @@ import org.apache.freemarker.generator.base.datasource.DataSource;
import java.io.IOException;
import java.io.InputStreamReader;
import java.lang.reflect.Type;
+import java.util.ArrayList;
+import java.util.List;
import java.util.Map;
public class GsonTool {
private Gson gson;
- private Type type;
- public Map<String, Object> parse(DataSource dataSource) {
+ /**
+ * Parse a data source containing a JSON object.
+ *
+ * @param dataSource data source
+ * @return map representing the JSON
+ */
+ public Map<String, Object> toMap(DataSource dataSource) {
try (JsonReader reader = new JsonReader(new InputStreamReader(dataSource.getUnsafeInputStream()))) {
- return gson().fromJson(reader, type());
+ return gson().fromJson(reader, objectTypeToken());
} catch (IOException e) {
throw new RuntimeException("Failed to parse data source:" + dataSource, e);
}
}
- public Map<String, Object> parse(String json) {
- return gson().fromJson(json, type());
+ /**
+ * Parse a JSON object string.
+ *
+ * @param json Json string
+ * @return map representing the JSON object
+ */
+ public Map<String, Object> toMap(String json) {
+ return gson().fromJson(json, objectTypeToken());
}
+ /**
+ * Parse a data source containing a JSON array.
+ *
+ * @param dataSource data source
+ * @return list of maps
+ */
+ public List<Map<String, Object>> toList(DataSource dataSource) {
+ try (JsonReader reader = new JsonReader(new InputStreamReader(dataSource.getUnsafeInputStream()))) {
+ return gson().fromJson(reader, listTypeToken());
+ } catch (IOException e) {
+ throw new RuntimeException("Failed to parse data source:" + dataSource, e);
+ }
+ }
+
+ /**
+ * Parse a JSON string containing a JSON array.
+ *
+ * @param json Json string
+ * @return list of maps
+ */
+ public List<Map<String, Object>> toList(String json) {
+ return gson().fromJson(json, listTypeToken());
+ }
+
+ /**
+ * Converts to JSON string.
+ *
+ * @param src source object
+ * @return JSON string
+ */
public String toJson(Object src) {
return gson().toJson(src);
}
@@ -60,10 +103,12 @@ public class GsonTool {
return gson;
}
- private synchronized Type type() {
- if (type == null) {
- type = new TypeToken<Map<String, Object>>() {}.getType();
- }
- return type;
+ private static Type objectTypeToken() {
+ return new TypeToken<Map<String, Object>>() {}.getType();
}
+
+ private static Type listTypeToken() {
+ return new TypeToken<ArrayList<Map<String, Object>>>() {}.getType();
+ }
+
}
diff --git a/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/jsoup/JsoupTool.java b/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/jsoup/JsoupTool.java
index 6a9d23e..df252c3 100644
--- a/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/jsoup/JsoupTool.java
+++ b/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/jsoup/JsoupTool.java
@@ -18,13 +18,14 @@ package org.apache.freemarker.generator.tools.jsoup;
import org.apache.freemarker.generator.base.datasource.DataSource;
import org.jsoup.Jsoup;
+import org.jsoup.nodes.Document;
import java.io.IOException;
import java.io.InputStream;
public class JsoupTool {
- public org.jsoup.nodes.Document parse(DataSource dataSource) {
+ public Document parse(DataSource dataSource) {
try (InputStream is = dataSource.getUnsafeInputStream()) {
return Jsoup.parse(is, dataSource.getCharset().name(), "");
} catch (IOException e) {
@@ -32,7 +33,7 @@ public class JsoupTool {
}
}
- public org.jsoup.nodes.Document parse(String html) {
+ public Document parse(String html) {
return Jsoup.parse(html);
}
diff --git a/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/system/SystemTool.java b/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/system/SystemTool.java
index 183b9da..ae5b66a 100644
--- a/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/system/SystemTool.java
+++ b/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/system/SystemTool.java
@@ -34,7 +34,7 @@ import static org.apache.freemarker.generator.base.FreeMarkerConstants.Model.FRE
import static org.apache.freemarker.generator.base.FreeMarkerConstants.Model.FREEMARKER_WRITER;
/**
- * Provides system related functionality, e.g. accessing environment variable,
+ * Provides system related functionality, e.g. accessing environment variables,
* system properties, commandl-line arguments, hostname, FreeMarker writer, etc.
*/
@SuppressWarnings("unchecked")
diff --git a/freemarker-generator-tools/src/test/java/org/apache/freemarker/generator/tools/commonscsv/CommonsCSVToolTest.java b/freemarker-generator-tools/src/test/java/org/apache/freemarker/generator/tools/commonscsv/CommonsCSVToolTest.java
index f76e252..d8e4b6a 100644
--- a/freemarker-generator-tools/src/test/java/org/apache/freemarker/generator/tools/commonscsv/CommonsCSVToolTest.java
+++ b/freemarker-generator-tools/src/test/java/org/apache/freemarker/generator/tools/commonscsv/CommonsCSVToolTest.java
@@ -74,7 +74,6 @@ public class CommonsCSVToolTest {
}
assertEquals(7, keys.size());
- assertEquals(7, keys.size());
assertEquals("C71", keys.get(0));
assertEquals("C72", keys.get(1));
assertEquals("C73", keys.get(2));
diff --git a/freemarker-generator-tools/src/test/java/org/apache/freemarker/generator/tools/dataframe/DataFrameToolTest.java b/freemarker-generator-tools/src/test/java/org/apache/freemarker/generator/tools/dataframe/DataFrameToolTest.java
index 96d6ee3..735e927 100644
--- a/freemarker-generator-tools/src/test/java/org/apache/freemarker/generator/tools/dataframe/DataFrameToolTest.java
+++ b/freemarker-generator-tools/src/test/java/org/apache/freemarker/generator/tools/dataframe/DataFrameToolTest.java
@@ -17,64 +17,90 @@
package org.apache.freemarker.generator.tools.dataframe;
import de.unknownreality.dataframe.DataFrame;
-import de.unknownreality.dataframe.csv.CSVReader;
+import org.apache.commons.csv.CSVFormat;
+import org.apache.commons.csv.CSVParser;
import org.apache.freemarker.generator.base.datasource.DataSource;
import org.apache.freemarker.generator.base.datasource.DataSourceFactory;
+import org.apache.freemarker.generator.tools.commonscsv.CommonsCSVTool;
+import org.apache.freemarker.generator.tools.gson.GsonTool;
import org.junit.Test;
import java.io.File;
+import java.util.List;
+import java.util.Map;
-import static de.unknownreality.dataframe.sort.SortColumn.Direction.Descending;
import static java.nio.charset.StandardCharsets.UTF_8;
-import static junit.framework.Assert.assertEquals;
+import static junit.framework.TestCase.assertEquals;
+import static org.apache.commons.csv.CSVFormat.DEFAULT;
public class DataFrameToolTest {
- private static final File CONTRACT_CSV = new File("./src/test/data/csv/contract.csv");
private static final File DATA_JOIN_A = new File("./src/test/data/csv/data_join_a.csv");
private static final File DATA_JOIN_B = new File("./src/test/data/csv/data_join_b.csv");
+ private static final String JSON_ARRAY = "[\n" +
+ " {\n" +
+ " \"Book ID\": \"1\",\n" +
+ " \"Book Name\": \"Computer Architecture\",\n" +
+ " \"Category\": \"Computers\",\n" +
+ " \"Price\": \"125.60\"\n" +
+ " },\n" +
+ " {\n" +
+ " \"Book ID\": \"2\",\n" +
+ " \"Book Name\": \"Asp.Net 4 Blue Book\",\n" +
+ " \"Category\": \"Programming\",\n" +
+ " \"Price\": \"56.00\"\n" +
+ " },\n" +
+ " {\n" +
+ " \"Book ID\": \"3\",\n" +
+ " \"Book Name\": \"Popular Science\",\n" +
+ " \"Category\": \"Science\",\n" +
+ " \"Price\": \"210.40\"\n" +
+ " }\n" +
+ "]";
+
+ // === CSV ==============================================================
+
@Test
- public void shouldParseCsvFile() {
- final DataFrame dataFrame = dataFrameTool().parse(dataSource(DATA_JOIN_A));
+ public void shouldParseCsvFileWithHeader() {
+ final CSVParser csvParser = csvParser(DATA_JOIN_A, DEFAULT.withHeader().withDelimiter(';'));
+ final DataFrame dataFrame = dataFrameTool().toDataFrame(csvParser);
- assertEquals("data_join_a.csv", dataFrame.getName());
assertEquals(3, dataFrame.getColumns().size());
assertEquals(4, dataFrame.getRows().size());
assertEquals("A", dataFrame.getColumn("GENE_ID").get(0));
}
- @Test
- public void shouldParseCsvFileUsingCSVReader() {
- final DataFrameTool dataFrameTool = dataFrameTool();
- final CSVReader csvReader = dataFrameTool.getCsvReaderBuilder().containsHeader(true).withSeparator(',').build();
- final DataFrame dataFrame = dataFrameTool.parse(dataSource(CONTRACT_CSV), csvReader);
-
- assertEquals("contract.csv", dataFrame.getName());
- assertEquals(32, dataFrame.getColumns().size());
- assertEquals(22, dataFrame.getRows().size());
- assertEquals("C71", dataFrame.getColumn("contract_id").get(0));
- }
+ // === JSON =============================================================
@Test
- public void shouldJoinDataFrames() {
- final String columnName = "GENE_ID";
- final DataFrame dataFrameA = dataFrameTool().parse(dataSource(DATA_JOIN_A));
- final DataFrame dataFrameB = dataFrameTool().parse(dataSource(DATA_JOIN_B));
- final DataFrame dataFrame = dataFrameA.joinInner(dataFrameB, columnName).sort(columnName, Descending);
+ public void shouldParseJsonTable() {
+ final String columnName = "Book ID";
+ final List<Map<String, Object>> json = gsonTool().toList(JSON_ARRAY);
+ final DataFrame dataFrame = dataFrameTool().toDataFrame(json);
- assertEquals(6, dataFrame.getColumns().size());
+ assertEquals(4, dataFrame.getColumns().size());
assertEquals(3, dataFrame.getRows().size());
- assertEquals("B", dataFrame.getColumn(columnName).get(0));
-
- dataFrame.print();
+ assertEquals("1", dataFrame.getColumn(columnName).get(0));
}
private DataFrameTool dataFrameTool() {
return new DataFrameTool();
}
+ private CommonsCSVTool commonsCSVTool() {
+ return new CommonsCSVTool();
+ }
+
+ private GsonTool gsonTool() {
+ return new GsonTool();
+ }
+
private DataSource dataSource(File file) {
return DataSourceFactory.fromFile(file, UTF_8);
}
+
+ private CSVParser csvParser(File file, CSVFormat csvFormat) {
+ return commonsCSVTool().parse(dataSource(file), csvFormat);
+ }
}
diff --git a/freemarker-generator-tools/src/test/java/org/apache/freemarker/generator/tools/gson/GsonToolTest.java b/freemarker-generator-tools/src/test/java/org/apache/freemarker/generator/tools/gson/GsonToolTest.java
index 7a5f5cf..51981db 100644
--- a/freemarker-generator-tools/src/test/java/org/apache/freemarker/generator/tools/gson/GsonToolTest.java
+++ b/freemarker-generator-tools/src/test/java/org/apache/freemarker/generator/tools/gson/GsonToolTest.java
@@ -29,13 +29,13 @@ import static org.apache.commons.io.FileUtils.readFileToString;
public class GsonToolTest {
- private static final String JSON_OBJECT_STRING = "{\n" +
+ private static final String JSON_OBJECT = "{\n" +
" \"id\": 110.0,\n" +
" \"language\": \"Python\",\n" +
" \"price\": 1900.0\n" +
"}";
- private static final String JSON_ARRAY_STRING = "{\n" +
+ private static final String JSON_OBJECT_WITH_ARRAY = "{\n" +
" \"eBooks\": [\n" +
" {\n" +
" \"language\": \"Pascal\",\n" +
@@ -59,11 +59,32 @@ public class GsonToolTest {
" \"color\": \"Red\"\n" +
"}";
+ private static final String JSON_ARRAY = "[\n" +
+ " {\n" +
+ " \"Book ID\": \"1\",\n" +
+ " \"Book Name\": \"Computer Architecture\",\n" +
+ " \"Category\": \"Computers\",\n" +
+ " \"Price\": \"125.60\"\n" +
+ " },\n" +
+ " {\n" +
+ " \"Book ID\": \"2\",\n" +
+ " \"Book Name\": \"Asp.Net 4 Blue Book\",\n" +
+ " \"Category\": \"Programming\",\n" +
+ " \"Price\": \"56.00\"\n" +
+ " },\n" +
+ " {\n" +
+ " \"Book ID\": \"3\",\n" +
+ " \"Book Name\": \"Popular Science\",\n" +
+ " \"Category\": \"Science\",\n" +
+ " \"Price\": \"210.40\"\n" +
+ " }\n" +
+ "]";
+
private final GsonTool gsonTool = gsonTool();
@Test
public void shouldParseJsonObject() {
- final Map<String, Object> map = parse(JSON_OBJECT_STRING);
+ final Map<String, Object> map = gsonTool.toMap(JSON_OBJECT);
assertEquals(3, map.size());
assertEquals("110.0", map.get("id").toString());
@@ -72,39 +93,43 @@ public class GsonToolTest {
}
@Test
- public void shouldParseJsonArray() {
- final Map<String, Object> map = parse(JSON_ARRAY_STRING);
+ public void shouldParseJsonObjectWithArray() {
+ final Map<String, Object> map = gsonTool.toMap(JSON_OBJECT_WITH_ARRAY);
assertEquals(1, map.size());
assertEquals(3, ((List) map.get("eBooks")).size());
-
- return;
}
@Test
public void shouldParseJsonWithComemnts() {
- final Map<String, Object> map = parse(JSON_WITH_COMMENTS);
+ final Map<String, Object> map = gsonTool.toMap(JSON_WITH_COMMENTS);
assertEquals("Apple", map.get("fruit"));
}
@Test
+ public void shouldParseJsonArray() {
+ final List<Map<String, Object>> list = gsonTool.toList(JSON_ARRAY);
+
+ assertEquals(3, list.size());
+ assertEquals("1", list.get(0).get("Book ID"));
+ assertEquals("2", list.get(1).get("Book ID"));
+ assertEquals("3", list.get(2).get("Book ID"));
+ }
+
+ @Test
public void shouldConvertToJson() {
- assertEquals(JSON_OBJECT_STRING, gsonTool.toJson(parse(JSON_OBJECT_STRING)));
- assertEquals(JSON_ARRAY_STRING, gsonTool.toJson(parse(JSON_ARRAY_STRING)));
+ assertEquals(JSON_OBJECT, gsonTool.toJson(gsonTool.toMap(JSON_OBJECT)));
+ assertEquals(JSON_OBJECT_WITH_ARRAY, gsonTool.toJson(gsonTool.toMap(JSON_OBJECT_WITH_ARRAY)));
}
@Test
public void shouldParseComplexJson() throws IOException {
final String json = readFileToString(new File("./src/test/data/json/swagger.json"), UTF_8);
- final Map<String, Object> map = parse(json);
+ final Map<String, Object> map = gsonTool.toMap(json);
assertEquals("petstore.swagger.io", map.get("host"));
- assertEquals(json, gsonTool.toJson(parse(json)));
- }
-
- private Map<String, Object> parse(String json) {
- return gsonTool.parse(json);
+ assertEquals(json, gsonTool.toJson(gsonTool.toMap(json)));
}
private GsonTool gsonTool() {