You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@freemarker.apache.org by sg...@apache.org on 2021/10/23 13:38:47 UTC
[freemarker-generator] branch master updated: FREEMARKER-199 Add
"utah-parser-tool"
This is an automated email from the ASF dual-hosted git repository.
sgoeschl pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/freemarker-generator.git
The following commit(s) were added to refs/heads/master by this push:
new 20d7f13 FREEMARKER-199 Add "utah-parser-tool"
20d7f13 is described below
commit 20d7f13e58788155322b5c7fca4043c769dc2d6f
Author: Siegfried Goeschl <si...@gmail.com>
AuthorDate: Sat Oct 23 15:38:44 2021 +0200
FREEMARKER-199 Add "utah-parser-tool"
---
.../generator/base/datasource/DataSources.java | 17 +-
.../freemarker/generator/base/table/Table.java | 2 +-
.../generator/datasource/DataSourcesTest.java | 33 +++-
freemarker-generator-cli/CHANGELOG.md | 4 +
.../src/app/config/freemarker-generator.properties | 1 +
.../src/app/examples/data/nginx/test1-api.nginx | 30 +++
.../examples/data/nginx/test1-application.nginx | 27 +++
.../utahparser/juniper_bgp_summary_example.txt | 12 ++
.../utahparser/juniper_bgp_summary_template.xml | 83 +++++++++
.../nginx/confluence/nginx-config-parser.ftl | 34 ++++
.../templates/utahparser/csv/transform.ftl | 32 ++++
.../templates/utahparser/json/transform.ftl | 19 ++
.../src/app/scripts/run-examples.bat | 17 ++
.../src/app/scripts/run-examples.sh | 17 ++
.../app/templates/freemarker-generator/info.ftl | 1 -
.../generator/cli/config/DataModelSupplier.java | 12 +-
.../src/site/markdown/cli/concepts/data-models.md | 2 +-
.../src/site/markdown/cli/concepts/data-sources.md | 4 +-
.../src/site/markdown/cli/concepts/named-uris.md | 2 +-
.../site/markdown/cli/usage/running-examples.md | 68 ++-----
.../freemarker/generator/cli/ExamplesTest.java | 19 ++
.../freemarker/generator/cli/ManualTest.java | 2 +-
freemarker-generator-tools/pom.xml | 6 +
.../commonscsv/impl/CommonsCSVPrinterFacade.java | 12 ++
.../generator/tools/utahparser/UtahParserTool.java | 100 ++++++++++
.../tools/utahparser/impl/ParserWrapper.java | 86 +++++++++
.../utahparser/juniper_bgp_summary_example.txt | 12 ++
.../utahparser/juniper_bgp_summary_template.xml | 69 +++++++
.../tools/utahparser/UtahParserToolTest.java | 103 +++++++++++
licences/LICENCE_utahparser.txt | 201 +++++++++++++++++++++
30 files changed, 967 insertions(+), 60 deletions(-)
diff --git a/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/datasource/DataSources.java b/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/datasource/DataSources.java
index 31a6462..7762c41 100644
--- a/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/datasource/DataSources.java
+++ b/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/datasource/DataSources.java
@@ -27,6 +27,8 @@ import java.util.Collections;
import java.util.LinkedHashMap;
import java.util.List;
import java.util.Map;
+import java.util.Map.Entry;
+import java.util.function.Function;
import java.util.stream.Collectors;
import static java.util.function.Function.identity;
@@ -180,7 +182,7 @@ public class DataSources implements Closeable {
}
/**
- * Find data sources based on their metadata key and wildcard string.
+ * Find <code>DataSources</code> based on their metadata key and wildcard string.
*
* @param key metadata key to match
* @param wildcard the wildcard string to match against
@@ -206,6 +208,19 @@ public class DataSources implements Closeable {
return new DataSources(find(key, wildcard));
}
+ /**
+ * Group the <code>DataSources</code> by a metadata value.
+ * @param key metadata key to group by
+ * @return groups of <code>DataSources</code>
+ */
+ public Map<String, DataSources> groupingBy(String key) {
+ final Function<DataSource, String> metadataFunction = dataSource -> dataSource.getMetadata(key);
+ return dataSources.stream()
+ .collect(Collectors.groupingBy(metadataFunction))
+ .entrySet().stream()
+ .collect(Collectors.toMap(Entry::getKey, p -> new DataSources(p.getValue())));
+ }
+
@Override
public void close() {
dataSources.forEach(ClosableUtils::closeQuietly);
diff --git a/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/table/Table.java b/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/table/Table.java
index 1abf21b..b1710b1 100644
--- a/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/table/Table.java
+++ b/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/table/Table.java
@@ -141,7 +141,7 @@ public class Table {
/**
* Create a table from a list of rows representing tabular data
- * where the first row may consists of column headers.
+ * where the first row may consist of column headers.
*
* @param rows row values
* @param withFirstRowAsColumnNames column names as first row?
diff --git a/freemarker-generator-base/src/test/java/org/apache/freemarker/generator/datasource/DataSourcesTest.java b/freemarker-generator-base/src/test/java/org/apache/freemarker/generator/datasource/DataSourcesTest.java
index 4bdf193..9885a81 100644
--- a/freemarker-generator-base/src/test/java/org/apache/freemarker/generator/datasource/DataSourcesTest.java
+++ b/freemarker-generator-base/src/test/java/org/apache/freemarker/generator/datasource/DataSourcesTest.java
@@ -25,6 +25,7 @@ import java.io.File;
import java.net.MalformedURLException;
import java.net.URL;
import java.util.HashMap;
+import java.util.Map;
import static java.nio.charset.StandardCharsets.UTF_8;
import static java.util.Arrays.asList;
@@ -119,6 +120,36 @@ public class DataSourcesTest {
assertEquals(singletonList(DEFAULT_GROUP), dataSources().getGroups());
}
+ @Test
+ public void shouldSupportGroupingByMetadata() {
+ try (DataSources dataSources = dataSources()) {
+ final Map<String, DataSources> map = dataSources.groupingBy("mimeType");
+
+ assertEquals(2, map.size());
+ assertEquals(1, map.get("application/xml").size());
+ assertEquals(2, map.get("text/plain").size());
+ }
+ }
+
+ @Test
+ public void shouldSupportFilteringByMetadata() {
+ try (DataSources dataSources = dataSources().filter("mimeType", "text/plain")) {
+ assertEquals(2, dataSources.size());
+ assertEquals("text/plain", dataSources.get(0).getMimeType());
+ assertEquals("text/plain", dataSources.get(1).getMimeType());
+ }
+
+ try (DataSources dataSources = dataSources().filter("mimeType", "application/xml")) {
+ assertEquals(1, dataSources.size());
+ assertEquals("application/xml", dataSources.get(0).getMimeType());
+ }
+
+ try (DataSources dataSources = dataSources().filter("mimeType", "!text/plain")) {
+ assertEquals(1, dataSources.size());
+ assertEquals("application/xml", dataSources.get(0).getMimeType());
+ }
+ }
+
@Test(expected = IllegalArgumentException.class)
public void shouldThrowExceptionWhenGetDoesNotFindDataSource() {
dataSources().get("file-does-not-exist");
@@ -142,7 +173,7 @@ public class DataSourcesTest {
}
private static DataSource urlDataSource() {
- return DataSourceFactory.fromUrl("server.invalid?foo=bar", "default", toUrl(ANY_URL), "plain/text", UTF_8, new HashMap<>());
+ return DataSourceFactory.fromUrl("server.invalid?foo=bar", "default", toUrl(ANY_URL), "text/plain", UTF_8, new HashMap<>());
}
private static URL toUrl(String value) {
diff --git a/freemarker-generator-cli/CHANGELOG.md b/freemarker-generator-cli/CHANGELOG.md
index d74ca0e..c440886 100644
--- a/freemarker-generator-cli/CHANGELOG.md
+++ b/freemarker-generator-cli/CHANGELOG.md
@@ -4,6 +4,9 @@ All notable changes to this project will be documented in this file. We try to a
## 0.2.0-SNAPSHOT
+### Added
+* [FREEMARKER-199] Add [utah-parser-tool](https://github.com/sonalake/utah-parser) to parse semi-structured text.
+
### Changed
* [FREEMARKER-195] Improve exposure of DataSources using TemplateHashModelEx2
@@ -90,4 +93,5 @@ All notable changes to this project will be documented in this file. We try to a
[FREEMARKER-182]: https://issues.apache.org/jira/browse/FREEMARKER-182
[FREEMARKER-188]: https://issues.apache.org/jira/browse/FREEMARKER-188
[FREEMARKER-195]: https://issues.apache.org/jira/browse/FREEMARKER-195
+[FREEMARKER-199]: https://issues.apache.org/jira/browse/FREEMARKER-199
diff --git a/freemarker-generator-cli/src/app/config/freemarker-generator.properties b/freemarker-generator-cli/src/app/config/freemarker-generator.properties
index 25b7029..d65937b 100644
--- a/freemarker-generator-cli/src/app/config/freemarker-generator.properties
+++ b/freemarker-generator-cli/src/app/config/freemarker-generator.properties
@@ -34,6 +34,7 @@ freemarker.tools.jsonpath=org.apache.freemarker.generator.tools.jsonpath.JsonPat
freemarker.tools.jsoup=org.apache.freemarker.generator.tools.jsoup.JsoupTool
freemarker.tools.properties=org.apache.freemarker.generator.tools.properties.PropertiesTool
freemarker.tools.system=org.apache.freemarker.generator.tools.system.SystemTool
+freemarker.tools.utahparser=org.apache.freemarker.generator.tools.utahparser.UtahParserTool
freemarker.tools.uuid=org.apache.freemarker.generator.tools.uuid.UUIDTool
freemarker.tools.xml=org.apache.freemarker.generator.tools.xml.XmlTool
freemarker.tools.yaml=org.apache.freemarker.generator.tools.snakeyaml.SnakeYamlTool
diff --git a/freemarker-generator-cli/src/app/examples/data/nginx/test1-api.nginx b/freemarker-generator-cli/src/app/examples/data/nginx/test1-api.nginx
new file mode 100644
index 0000000..2c2486f
--- /dev/null
+++ b/freemarker-generator-cli/src/app/examples/data/nginx/test1-api.nginx
@@ -0,0 +1,30 @@
+server {
+ listen 443 ssl;
+ server_name test1-api.company.org;
+
+ access_log /var/log/nginx/test1-api.access.log;
+ error_log /var/log/nginx/test1-api.error.log;
+
+ ssl_certificate /etc/nginx/ssl/wildcard.company.org-public.crt;
+ ssl_certificate_key /etc/nginx/ssl/wildcard.company.org-private.rsa;
+
+ include /etc/nginx/includes/FLKPMM.nginx;
+
+ proxy_buffers 16 64k;
+ proxy_buffer_size 128k;
+ proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504;
+ proxy_redirect off;
+ proxy_set_header Host $host;
+ proxy_set_header X-Real-IP $remote_addr;
+ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+ proxy_set_header X-Forwarded-Proto https;
+ proxy_set_header X-TPP-QWAC $ssl_client_cert;
+
+ ssl_verify_client optional_no_ca;
+
+
+ location / {
+ return 403;
+ }
+
+}
diff --git a/freemarker-generator-cli/src/app/examples/data/nginx/test1-application.nginx b/freemarker-generator-cli/src/app/examples/data/nginx/test1-application.nginx
new file mode 100644
index 0000000..b618991
--- /dev/null
+++ b/freemarker-generator-cli/src/app/examples/data/nginx/test1-application.nginx
@@ -0,0 +1,27 @@
+server {
+ listen 443 ssl;
+ server_name test1-application.company.org;
+
+ access_log /var/log/nginx/test1-application.access.log;
+ error_log /var/log/nginx/test1-application.error.log;
+
+ ssl_certificate /etc/nginx/ssl/wildcard.company.org-public.crt;
+ ssl_certificate_key /etc/nginx/ssl/wildcard.company.org-private.rsa;
+
+ proxy_buffers 16 64k;
+ proxy_buffer_size 128k;
+
+ location / {
+ proxy_pass https://osroutercpssl/;
+ proxy_next_upstream error timeout invalid_header http_500 http_502 http_503 http_504;
+ proxy_redirect off;
+
+ proxy_set_header Host $host;
+ proxy_set_header X-Real-IP $remote_addr;
+ proxy_set_header X-Forwarded-For $proxy_add_x_forwarded_for;
+ proxy_set_header X-Forwarded-Proto https;
+ }
+
+}
+
+
diff --git a/freemarker-generator-cli/src/app/examples/data/text/utahparser/juniper_bgp_summary_example.txt b/freemarker-generator-cli/src/app/examples/data/text/utahparser/juniper_bgp_summary_example.txt
new file mode 100644
index 0000000..c4a209e
--- /dev/null
+++ b/freemarker-generator-cli/src/app/examples/data/text/utahparser/juniper_bgp_summary_example.txt
@@ -0,0 +1,12 @@
+Groups: 3 Peers: 3 Down peers: 0
+Table Tot Paths Act Paths Suppressed History Damp State Pending
+inet.0 947 310 0 0 0 0
+inet6.0 849 807 0 0 0 0
+Peer AS InPkt OutPkt OutQ Flaps Last Up/Dwn State|#Active/Received/Damped...
+10.247.68.182 65550 131725 28179233 0 11 6w3d17h Establ
+ inet.0: 4/5/1
+ inet6.0: 0/0/0
+10.254.166.246 65550 136159 29104942 0 0 6w5d6h Establ
+ inet.0: 0/0/0
+ inet6.0: 7/8/1
+192.0.2.100 65551 1269381 1363320 0 1 9w5d6h 1/2/3 4/5/6
\ No newline at end of file
diff --git a/freemarker-generator-cli/src/app/examples/data/text/utahparser/juniper_bgp_summary_template.xml b/freemarker-generator-cli/src/app/examples/data/text/utahparser/juniper_bgp_summary_template.xml
new file mode 100644
index 0000000..76084aa
--- /dev/null
+++ b/freemarker-generator-cli/src/app/examples/data/text/utahparser/juniper_bgp_summary_template.xml
@@ -0,0 +1,83 @@
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+<config>
+ <searches>
+
+ <!-- in this case, we have a CSV (space delimited file) so we define the line once, and then reuse it over
+ and again for each value -->
+ <search id="QUERY-LINE"><![CDATA[\s*{ipAddress}\s+{numbers}\s+{numbers}\s+{numbers}\s+{numbers}\s+{numbers}\s+{numbersThenText}]]></search>
+ <search id="inetInline"><![CDATA[{inet} {inet}]]></search>
+ <search id="inet4"><![CDATA[inet.0:\s*{inet}]]></search>
+ <search id="inet6"><![CDATA[inet6.0:\s*{inet}]]></search>
+ <search id="inet"><![CDATA[{numbers}/{numbers}/{numbers}]]></search>
+
+ <!-- Some rules for finding text, to make the values a little easier below -->
+ <search id="numbers"><![CDATA[(\d+)]]></search>
+ <search id="numbersThenText"><![CDATA[(\d+\S+)]]></search>
+ <search id="string"><![CDATA[(\S+?)]]></search>
+ <search id="ipAddress"><![CDATA[(\d+(\.\d+){3})]]></search>
+ <search id="EOL"><![CDATA[[\n\r]]]></search>
+ </searches>
+
+ <!-- the record starts with a line with an ip address and ends with either an inet6 line, or where the ids are at
+ the end of the line-->
+ <delim retain="true">{ipAddress}.*(\/\d+)\s*{EOL}</delim>
+ <delim>\s*({inet6})</delim>
+
+ <!--
+ This is the last line of the header
+ -->
+ <header-delim><![CDATA[Peer\s+AS\s+InPkt]]></header-delim>
+
+ <!--
+ Files look like this:
+
+ 10.247.68.182 65550 131725 28179233 0 11 6w3d17h Establ
+ inet.0: 4/5/1
+ inet6.0: 0/0/0
+
+ or
+
+ 192.0.2.100 65551 1269381 1363320 0 1 9w5d6h 2/3/0 0/0/0
+ -->
+ <values>
+ <!-- here we reuse the line pattern, only we pull out different group values -->
+ <value id="remoteIp" group="1"><![CDATA[{QUERY-LINE}]]></value>
+ <value id="uptime" group="8"><![CDATA[{QUERY-LINE}]]></value>
+
+ <!-- here we check for values in the inet* lines and use these -->
+ <value id="activeV4" group="1"><![CDATA[{inet4}]]></value>
+ <value id="receivedV4" group="2"><![CDATA[{inet4}]]></value>
+ <value id="accepted_V4" group="3"><![CDATA[{inet4}]]></value>
+
+ <value id="activeV6" group="1"><![CDATA[{inet6}]]></value>
+ <value id="receivedV6" group="2"><![CDATA[{inet6}]]></value>
+ <value id="accepted_V6" group="3"><![CDATA[{inet6}]]></value>
+
+ <!--
+ here we check for values at the end of the query line, and use these
+ NOTE: since we only set non-null values, these will not overwrite any values set above
+ -->
+ <value id="activeV4" group="9"><![CDATA[{QUERY-LINE}\s*{inetInline}]]></value>
+ <value id="receivedV4" group="10"><![CDATA[{QUERY-LINE}\s*{inetInline}]]></value>
+ <value id="accepted_V4" group="11"><![CDATA[{QUERY-LINE}\s*{inetInline}]]></value>
+ <value id="activeV6" group="12"><![CDATA[{QUERY-LINE}\s*{inetInline}]]></value>
+ <value id="receivedV6" group="13"><![CDATA[{QUERY-LINE}\s*{inetInline}]]></value>
+ <value id="accepted_V6" group="14"><![CDATA[{QUERY-LINE}\s*{inetInline}]]></value>
+
+ </values>
+</config>
\ No newline at end of file
diff --git a/freemarker-generator-cli/src/app/examples/templates/nginx/confluence/nginx-config-parser.ftl b/freemarker-generator-cli/src/app/examples/templates/nginx/confluence/nginx-config-parser.ftl
new file mode 100644
index 0000000..79331a5
--- /dev/null
+++ b/freemarker-generator-cli/src/app/examples/templates/nginx/confluence/nginx-config-parser.ftl
@@ -0,0 +1,34 @@
+<#compress>
+ || FILE || SERVER || ACCESSLOG || SPLUNK ||
+ <#list dataSources as dataSource>
+ <#assign fileName = dataSource.fileName>
+ <#assign serverName = "N.A">
+ <#-- Transform to a single line to avoid matching OS-specific line endings -->
+ <#assign text = dataSource.getText()?replace("\r", "")?replace("\n", " ")>
+ <#assign accessLog = getAccessLog(text)>
+ <#assign serverName = getServerName(text)>
+ | ${fileName} | ${serverName} | ${accessLog} | [${splunkSearchUrl(accessLog)}] |
+ </#list>
+</#compress>
+<#--------------------------------------------------------------------------->
+<#function splunkSearchUrl accessLog>
+ <#return "https://splunk.p.santanderconsumer.at/en-US/app/scbdevteam/search?q=search%20source%3D%22${accessLog?url}%22">
+</#function>
+<#--------------------------------------------------------------------------->
+<#function getAccessLog text>
+ <#assign matches = text?matches(r".*access_log\s*([\w\.\-\/\\]*);.*")>
+ <#if matches>
+ <#return matches?groups[1]?trim>
+ <#else>
+ <#return "N.A.">
+ </#if>
+</#function>
+<#--------------------------------------------------------------------------->
+<#function getServerName text>
+ <#assign matches = text?matches(r".*server_name\s*([\w\.\-\\]*);.*")>
+ <#if matches>
+ <#return matches?groups[1]?trim>
+ <#else>
+ <#return "N.A.">
+ </#if>
+</#function>
\ No newline at end of file
diff --git a/freemarker-generator-cli/src/app/examples/templates/utahparser/csv/transform.ftl b/freemarker-generator-cli/src/app/examples/templates/utahparser/csv/transform.ftl
new file mode 100644
index 0000000..f67729c
--- /dev/null
+++ b/freemarker-generator-cli/src/app/examples/templates/utahparser/csv/transform.ftl
@@ -0,0 +1,32 @@
+<#--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+<#-- Setup Utah-Parser and parse all records to determine the headers -->
+<#assign conf = tools.utahparser.getConfig(dataSources[0])>
+<#assign parser = tools.utahparser.getParser(conf, dataSources[1])>
+<#assign records = parser.toList()>
+<#assign headers = tools.utahparser.getHeaders(records)>
+<#-- Setup CSVPrinter -->
+<#assign defaultCsvformat = tools.csv.formats[CSV_TARGET_FORMAT!"DEFAULT"]>
+<#assign csvDelimiter = tools.csv.toDelimiter(CSV_TARGET_DELIMITER!defaultCsvformat.getDelimiter())>
+<#assign cvsFormat = defaultCsvformat.withHeader(headers).withDelimiter(csvDelimiter)>
+<#assign csvPrinter = tools.csv.printer(cvsFormat)>
+<#-- Print records as CSV -->
+<#compress>
+ <#list records as record>
+ ${csvPrinter.printRecord(record, headers)}
+ </#list>
+</#compress>
\ No newline at end of file
diff --git a/freemarker-generator-cli/src/app/examples/templates/utahparser/json/transform.ftl b/freemarker-generator-cli/src/app/examples/templates/utahparser/json/transform.ftl
new file mode 100644
index 0000000..4c2a53e
--- /dev/null
+++ b/freemarker-generator-cli/src/app/examples/templates/utahparser/json/transform.ftl
@@ -0,0 +1,19 @@
+<#--
+ Licensed to the Apache Software Foundation (ASF) under one
+ or more contributor license agreements. See the NOTICE file
+ distributed with this work for additional information
+ regarding copyright ownership. The ASF licenses this file
+ to you under the Apache License, Version 2.0 (the
+ "License"); you may not use this file except in compliance
+ with the License. You may obtain a copy of the License at
+ http://www.apache.org/licenses/LICENSE-2.0
+ Unless required by applicable law or agreed to in writing,
+ software distributed under the License is distributed on an
+ "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ KIND, either express or implied. See the License for the
+ specific language governing permissions and limitations
+ under the License.
+-->
+<#assign conf = tools.utahparser.getConfig(dataSources[0])>
+<#assign parser = tools.utahparser.getParser(conf, dataSources[1])>
+${tools.gson.toJson(parser.toList())}
\ No newline at end of file
diff --git a/freemarker-generator-cli/src/app/scripts/run-examples.bat b/freemarker-generator-cli/src/app/scripts/run-examples.bat
index 551b5c1..e309473 100644
--- a/freemarker-generator-cli/src/app/scripts/run-examples.bat
+++ b/freemarker-generator-cli/src/app/scripts/run-examples.bat
@@ -162,6 +162,13 @@ echo "examples\templates\json\md\github-users.ftl"
%FREEMARKER_CMD% -t examples\templates\json\md\github-users.ftl examples\data\json\github-users.json > target\out\github-users.md
REM =========================================================================
+REM NGINX
+REM =========================================================================
+
+echo "examples\templates\nginx\confluence\nginx-config-parser.ftl"
+%FREEMARKER_CMD% -t examples\templates\nginx\confluence\nginx-config-parser.ftl -s examples\data\nginx > target\out\nginx-config.txt
+
+REM =========================================================================
REM Properties
REM =========================================================================
@@ -176,6 +183,16 @@ echo "examples\data\template"
%FREEMARKER_CMD% -t examples\data\template -PNGINX_HOSTNAME=localhost -o target\out\template
REM =========================================================================
+REM Utah Parser
+REM =========================================================================
+
+echo "examples\templates\utahparser\csv\transform.ftl"
+%FREEMARKER_CMD% -PCSV_TARGET_FORMAT=EXCEL -PCSV_TARGET_DELIMITER=SEMICOLON -t examples\templates\utahparser\csv\transform.ftl examples\data\text\utahparser\juniper_bgp_summary_template.xml examples\data\text\utahparser\juniper_bgp_summary_example.txt -o target\out\utahparserjuniper_bgp_summary_example.csv
+
+echo "examples\templates\utahparser\json\transform.ftl"
+%FREEMARKER_CMD% -t examples\templates\utahparser\json\transform.ftl examples\data\text\utahparser\juniper_bgp_summary_template.xml examples\data\text\utahparser\juniper_bgp_summary_example.txt -o target\out\utahparser\uniper_bgp_summary_example.json
+
+REM =========================================================================
REM XML
REM =========================================================================
diff --git a/freemarker-generator-cli/src/app/scripts/run-examples.sh b/freemarker-generator-cli/src/app/scripts/run-examples.sh
index 979f0b1..c4a94c1 100755
--- a/freemarker-generator-cli/src/app/scripts/run-examples.sh
+++ b/freemarker-generator-cli/src/app/scripts/run-examples.sh
@@ -183,6 +183,13 @@ echo "examples/templates/json/md/github-users.ftl"
$FREEMARKER_CMD -t examples/templates/json/md/github-users.ftl examples/data/json/github-users.json > target/out/github-users.md || { echo >&2 "Test failed. Aborting."; exit 1; }
#############################################################################
+# NGINX
+#############################################################################
+
+echo "examples/templates/nginx/confluence/nginx-config-parser.ftl"
+$FREEMARKER_CMD -t examples/templates/nginx/confluence/nginx-config-parser.ftl -s examples/data/nginx > target/out/nginx-config.txt || { echo >&2 "Test failed. Aborting."; exit 1; }
+
+#############################################################################
# Properties
#############################################################################
@@ -197,6 +204,16 @@ echo "examples/data/template"
$FREEMARKER_CMD -t examples/data/template -PNGINX_HOSTNAME=localhost -o target/out/template || { echo >&2 "Test failed. Aborting."; exit 1; }
#############################################################################
+# Utah Parser
+#############################################################################
+
+echo "examples/templates/utahparser/csv/transform.ftl"
+$FREEMARKER_CMD -PCSV_TARGET_FORMAT=EXCEL -PCSV_TARGET_DELIMITER=SEMICOLON -t examples/templates/utahparser/csv/transform.ftl examples/data/text/utahparser/juniper_bgp_summary_template.xml examples/data/text/utahparser/juniper_bgp_summary_example.txt -o target/out/utahparserjuniper_bgp_summary_example.csv || { echo >&2 "Test failed. Aborting."; exit 1; }
+
+echo "examples/templates/utahparser/json/transform.ftl"
+$FREEMARKER_CMD -t examples/templates/utahparser/json/transform.ftl examples/data/text/utahparser/juniper_bgp_summary_template.xml examples/data/text/utahparser/juniper_bgp_summary_example.txt -o target/out/utahparser/juniper_bgp_summary_example.json || { echo >&2 "Test failed. Aborting."; exit 1; }
+
+#############################################################################
# XML
#############################################################################
diff --git a/freemarker-generator-cli/src/app/templates/freemarker-generator/info.ftl b/freemarker-generator-cli/src/app/templates/freemarker-generator/info.ftl
index d5725d0..0a3f0ce 100644
--- a/freemarker-generator-cli/src/app/templates/freemarker-generator/info.ftl
+++ b/freemarker-generator-cli/src/app/templates/freemarker-generator/info.ftl
@@ -60,7 +60,6 @@ contentType : ${dataSource.contentType}
fileName : ${dataSource.fileName}
baseName : ${dataSource.baseName}
extension : ${dataSource.extension}
-extension : ${dataSource.extension}
relativeFilePath : ${dataSource.relativeFilePath}
charset : ${dataSource.charset}
mimeType : ${dataSource.mimeType}
diff --git a/freemarker-generator-cli/src/main/java/org/apache/freemarker/generator/cli/config/DataModelSupplier.java b/freemarker-generator-cli/src/main/java/org/apache/freemarker/generator/cli/config/DataModelSupplier.java
index 57591d6..6cc81d0 100644
--- a/freemarker-generator-cli/src/main/java/org/apache/freemarker/generator/cli/config/DataModelSupplier.java
+++ b/freemarker-generator-cli/src/main/java/org/apache/freemarker/generator/cli/config/DataModelSupplier.java
@@ -16,6 +16,7 @@
*/
package org.apache.freemarker.generator.cli.config;
+import org.apache.commons.io.FilenameUtils;
import org.apache.freemarker.generator.base.datasource.DataSource;
import org.apache.freemarker.generator.base.datasource.DataSourceLoader;
import org.apache.freemarker.generator.base.datasource.DataSourceLoaderFactory;
@@ -79,15 +80,22 @@ public class DataModelSupplier implements Supplier<Map<String, Object>> {
if (contentType.startsWith(MIME_APPLICATION_JSON)) {
return fromJson(dataSource, isExplodedDataModel);
+ } else if (isYamlDataSource(dataSource)) {
+ return fromYaml(dataSource, isExplodedDataModel);
} else if (contentType.startsWith(MIME_TEXT_PLAIN)) {
return fromProperties(dataSource, isExplodedDataModel);
- } else if (contentType.startsWith(MIME_TEXT_YAML)) {
- return fromYaml(dataSource, isExplodedDataModel);
} else {
throw new IllegalArgumentException("Don't know how to handle content type: " + contentType);
}
}
+ private static boolean isYamlDataSource(DataSource dataSource) {
+ final String contentType = dataSource.getContentType();
+ final String extension = FilenameUtils.getExtension(dataSource.getUri().toString());
+ return contentType.startsWith(MIME_TEXT_YAML)
+ || (contentType.startsWith(MIME_TEXT_PLAIN)) && "yaml".equalsIgnoreCase(extension);
+ }
+
private static Map<String, Object> fromJson(DataSource dataSource, boolean isExplodedDataModel) {
final GsonTool gsonTool = new GsonTool();
final Object json = gsonTool.parse(dataSource);
diff --git a/freemarker-generator-cli/src/site/markdown/cli/concepts/data-models.md b/freemarker-generator-cli/src/site/markdown/cli/concepts/data-models.md
index bd56d76..42b048f 100644
--- a/freemarker-generator-cli/src/site/markdown/cli/concepts/data-models.md
+++ b/freemarker-generator-cli/src/site/markdown/cli/concepts/data-models.md
@@ -43,7 +43,7 @@ Exposed the JSON data source as variable `post` in FreeMarker's model
post title is: qui est esse
```
-Expose all environment variables as `env` in theFreeMarker model
+Expose all environment variables as `env` in the FreeMarker model
```
> freemarker-generator --data-model env=env:/// -i '<#list env as name,value>${name}=${value}${"\n"}</#list>'
diff --git a/freemarker-generator-cli/src/site/markdown/cli/concepts/data-sources.md b/freemarker-generator-cli/src/site/markdown/cli/concepts/data-sources.md
index 2a7d01e..f04675d 100644
--- a/freemarker-generator-cli/src/site/markdown/cli/concepts/data-sources.md
+++ b/freemarker-generator-cli/src/site/markdown/cli/concepts/data-sources.md
@@ -96,7 +96,7 @@ URI : system:///stdin
### Selecting A DataSource
-After loading one or more `DataSource` they are accessible as `dataSource` map in the FreeMarker model
+After loading one or more `DataSource` they are accessible as `dataSources` map in the FreeMarker model
* `dataSources[0]` or `dataSources?first` selects the first data source
* `dataSources["user.csv"]` selects the data source with the name "user.csv"
@@ -306,8 +306,6 @@ group : default
Iterating Over Properties Of A Datasource
---------------------------------------------------------------------------
-
-
```
diff --git a/freemarker-generator-cli/src/site/markdown/cli/concepts/named-uris.md b/freemarker-generator-cli/src/site/markdown/cli/concepts/named-uris.md
index 5c5d61b..bb84d92 100644
--- a/freemarker-generator-cli/src/site/markdown/cli/concepts/named-uris.md
+++ b/freemarker-generator-cli/src/site/markdown/cli/concepts/named-uris.md
@@ -93,7 +93,7 @@ URI : file:/Users/sgoeschl/work/github/apache/freemarker-generator/freemarker-ge
or use a charset for all files of a directory
```text
-freemarker-generator -t freemarker-generator/info.ftl 'examples/data/csv#charset=UTF-16&mimetype=text/plain'
+freemarker-generator -t freemarker-generator/info.ftl 'examples/data/csv#charset=UTF-16&mimeType=text/plain'
FreeMarker Generator DataSources
------------------------------------------------------------------------------
diff --git a/freemarker-generator-cli/src/site/markdown/cli/usage/running-examples.md b/freemarker-generator-cli/src/site/markdown/cli/usage/running-examples.md
index 9b686a2..ab46cea 100644
--- a/freemarker-generator-cli/src/site/markdown/cli/usage/running-examples.md
+++ b/freemarker-generator-cli/src/site/markdown/cli/usage/running-examples.md
@@ -6,72 +6,44 @@ Run `run-examples.sh` in the `Apache FreeMarker Generator` installation director
```text
./run-examples.sh
-templates/info.ftl
+templates/freemarker-generator/info.ftl
examples/templates/demo.ftl
-templates/csv/html/transform.ftl
-templates/csv/md/transform.ftl
+examples/templates/datasources.ftl
+examples/templates/datasources.ftl
+templates/freemarker-generator/csv/confluence/transform.ftl
+templates/freemarker-generator/csv/html/transform.ftl
+templates/freemarker-generator/csv/md/transform.ftl
examples/templates/csv/shell/curl.ftl
examples/templates/csv/md/filter.ftl
examples/templates/csv/fo/transform.ftl
fop -fo target/out/locker-test-users.fo target/out/locker-test-users.pdf
examples/templates/csv/fo/transactions.ftl
fop -fo target/out/transactions.fo target/out/transactions-fo.pdf
-templates/csv/html/transform.ftl
+examples/templates/csv/html/transform.ftl
wkhtmltopdf -O landscape target/out/transactions.html target/out/transactions-html.pdf
examples/templates/dataframe/example.ftl
examples/templates/accesslog/combined-access.ftl
+examples/templates/logs/csv/serverlog-to-csv.ftl
examples/templates/excel/dataframe/transform.ftl
-templates/excel/html/transform.ftl
-templates/excel/md/transform.ftl
-templates/excel/csv/transform.ftl
+templates/freemarker-generator/excel/html/transform.ftl
+templates/freemarker-generator/excel/md/transform.ftl
+templates/freemarker-generator/excel/csv/transform.ftl
examples/templates/excel/csv/custom.ftl
examples/templates/html/csv/dependencies.ftl
+examples/templates/html/txt/licence.ftl
+examples/templates/javafaker/csv/testdata.ftl
examples/templates/json/csv/swagger-endpoints.ftl
-templates/json/yaml/transform.ftl
+templates/freemarker-generator/json/yaml/transform.ftl
examples/templates/json/md/github-users.ftl
examples/templates/properties/csv/locker-test-users.ftl
examples/data/template
-examples/templates/yaml/txt/transform.ftl
-templates/yaml/json/transform.ftl
+examples/templates/utahparser/csv/transform.ftl
+examples/templates/utahparser/json/transform.ftl
examples/templates/xml/txt/recipients.ftl
-Created the following sample files in ./target/out
-total 1464
--rw-r--r-- 1 sgoeschl staff 646 Jun 27 16:38 combined-access.log.txt
--rw-r--r-- 1 sgoeschl staff 25676 Jun 27 16:38 contract.html
--rw-r--r-- 1 sgoeschl staff 7933 Jun 27 16:38 contract.md
--rw-r--r-- 1 sgoeschl staff 784 Jun 27 16:38 curl.sh
--rw-r--r-- 1 sgoeschl staff 232 Jun 27 16:38 customer.txt
--rw-r--r-- 1 sgoeschl staff 6486 Jun 27 16:38 dataframe.txt
--rw-r--r-- 1 sgoeschl staff 15613 Jun 27 16:38 demo.txt
--rw-r--r-- 1 sgoeschl staff 1310 Jun 27 16:38 dependencies.csv
--rw-r--r-- 1 sgoeschl staff 2029 Jun 27 16:38 github-users-curl.md
--rw-r--r-- 1 sgoeschl staff 2627 Jun 27 16:38 info.txt
--rw-r--r-- 1 sgoeschl staff 8075 Jun 27 16:38 interactive-dataframe.txt
--rw-r--r-- 1 sgoeschl staff 66 Jun 27 16:38 interactive-html.txt
--rw-r--r-- 1 sgoeschl staff 16 Jun 27 16:38 interactive-json.txt
--rw-r--r-- 1 sgoeschl staff 25090 Jun 27 16:38 interactive-swagger.json
--rw-r--r-- 1 sgoeschl staff 16870 Jun 27 16:38 interactive-swagger.yaml
--rw-r--r-- 1 sgoeschl staff 10 Jun 27 16:38 interactive-xml.txt
--rw-r--r-- 1 sgoeschl staff 285 Jun 27 16:38 locker-test-users.csv
--rw-r--r-- 1 sgoeschl staff 6341 Jun 27 16:38 locker-test-users.fo
--rw-r--r-- 1 sgoeschl staff 5526 Jun 27 16:38 locker-test-users.pdf
--rw-r--r-- 1 sgoeschl staff 921 Jun 27 16:38 recipients.txt
--rw-r--r-- 1 sgoeschl staff 910 Jun 27 16:38 sales-records.md
--rw-r--r-- 1 sgoeschl staff 2453 Jun 27 16:38 swagger-spec.csv
--rw-r--r-- 1 sgoeschl staff 25090 Jun 27 16:38 swagger-spec.json
--rw-r--r-- 1 sgoeschl staff 16870 Jun 27 16:38 swagger-spec.yaml
-drwxr-xr-x 4 sgoeschl staff 128 Jun 27 16:38 template
--rw-r--r-- 1 sgoeschl staff 154 Jun 27 16:38 test-multiple-sheets.xlsx.csv
--rw-r--r-- 1 sgoeschl staff 1917 Jun 27 16:38 test-multiple-sheets.xlsx.html
--rw-r--r-- 1 sgoeschl staff 389 Jun 27 16:38 test-multiple-sheets.xlsx.md
--rw-r--r-- 1 sgoeschl staff 155 Jun 27 16:38 test-transform-xls.csv
--rw-r--r-- 1 sgoeschl staff 1439 Jun 27 16:38 test.xls.dataframe.txt
--rw-r--r-- 1 sgoeschl staff 1556 Jun 27 16:38 test.xls.html
--rw-r--r-- 1 sgoeschl staff 1558 Jun 27 16:38 test.xslx.html
--rw-r--r-- 1 sgoeschl staff 25757 Jun 27 16:38 transactions-fo.pdf
--rw-r--r-- 1 sgoeschl staff 66016 Jun 27 16:38 transactions-html.pdf
--rw-r--r-- 1 sgoeschl staff 330128 Jun 27 16:38 transactions.fo
--rw-r--r-- 1 sgoeschl staff 51008 Jun 27 16:38 transactions.html
+examples/templates/yaml/txt/transform.ftl
+templates/freemarker-generator/yaml/json/transform.ftl
+examples/data/csv
+examples/data/csv
```
Please note that generated PDF files are very likely not found since they require `wkhtmltopdf` and `Apache FOP` installation.
diff --git a/freemarker-generator-cli/src/test/java/org/apache/freemarker/generator/cli/ExamplesTest.java b/freemarker-generator-cli/src/test/java/org/apache/freemarker/generator/cli/ExamplesTest.java
index 0aa2399..2e4d70b 100644
--- a/freemarker-generator-cli/src/test/java/org/apache/freemarker/generator/cli/ExamplesTest.java
+++ b/freemarker-generator-cli/src/test/java/org/apache/freemarker/generator/cli/ExamplesTest.java
@@ -82,6 +82,11 @@ public class ExamplesTest extends AbstractMainTest {
}
@Test
+ public void shouldRunNginxParsingExamples() throws IOException {
+ assertValid(execute("-t src/app/examples/templates/nginx/confluence/nginx-config-parser.ftl -s src/app/examples/data/nginx"));
+ }
+
+ @Test
public void shouldRunJsonExamples() throws IOException {
assertValid(execute("-t src/app/examples/templates/json/csv/swagger-endpoints.ftl src/app/examples/data/json/swagger-spec.json"));
assertValid(execute("-t src/app/examples/templates/json/md/github-users.ftl src/app/examples/data/json/github-users.json"));
@@ -126,6 +131,19 @@ public class ExamplesTest extends AbstractMainTest {
}
@Test
+ public void shouldRunUtahParserExamples() throws IOException {
+ assertValid(execute("-PCSV_TARGET_FORMAT=EXCEL " +
+ "-PCSV_TARGET_DELIMITER=SEMICOLON " +
+ "-t src/app/examples/templates/utahparser/csv/transform.ftl " +
+ "src/app/examples/data/text/utahparser/juniper_bgp_summary_template.xml " +
+ "src/app/examples/data/text/utahparser/juniper_bgp_summary_example.txt"));
+
+ assertValid(execute("-t src/app/examples/templates/utahparser/json/transform.ftl " +
+ "src/app/examples/data/text/utahparser/juniper_bgp_summary_template.xml " +
+ "src/app/examples/data/text/utahparser/juniper_bgp_summary_example.txt"));
+ }
+
+ @Test
public void shouldRunInteractiveTemplateExamples() throws IOException {
assertValid(execute("-i ${tools.jsonpath.parse(dataSources[0]).read(\"$.info.title\")} src/app/examples/data/json/swagger-spec.json"));
assertValid(execute("-i ${tools.xml.parse(dataSources[0])[\"recipients/person[1]/name\"]} src/app/examples/data/xml/recipients.xml"));
@@ -215,6 +233,7 @@ public class ExamplesTest extends AbstractMainTest {
}
private static void assertValid(String output) {
+ // System.out.println(output);
assertTrue(output.length() > MIN_OUTPUT_SIZE);
assertFalse(output.contains("Exception"));
assertFalse(output.contains("FreeMarker template error"));
diff --git a/freemarker-generator-cli/src/test/java/org/apache/freemarker/generator/cli/ManualTest.java b/freemarker-generator-cli/src/test/java/org/apache/freemarker/generator/cli/ManualTest.java
index 2f1596b..2f1dd69 100644
--- a/freemarker-generator-cli/src/test/java/org/apache/freemarker/generator/cli/ManualTest.java
+++ b/freemarker-generator-cli/src/test/java/org/apache/freemarker/generator/cli/ManualTest.java
@@ -25,7 +25,7 @@ public class ManualTest extends AbstractMainTest {
// private static final String CMD = "-V";
private static final String CMD =
- "-t src/app/examples/templates/datasources.ftl readme:documentation=README.md src/main/assembly";
+ "-t src/app/examples/templates/nginx/confluence/nginx-config-parser.ftl -s src/app/examples/data/nginx";
// "-t src/app/examples/templates/datasources.ftl";
@Override
diff --git a/freemarker-generator-tools/pom.xml b/freemarker-generator-tools/pom.xml
index 590246f..04ca3f1 100644
--- a/freemarker-generator-tools/pom.xml
+++ b/freemarker-generator-tools/pom.xml
@@ -129,6 +129,12 @@
<artifactId>snakeyaml</artifactId>
<version>1.28</version>
</dependency>
+ <!-- UtahPrserTool -->
+ <dependency>
+ <groupId>com.sonalake</groupId>
+ <artifactId>utah-parser</artifactId>
+ <version>1.0.2</version>
+ </dependency>
<!-- Testing -->
<dependency>
<groupId>junit</groupId>
diff --git a/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/commonscsv/impl/CommonsCSVPrinterFacade.java b/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/commonscsv/impl/CommonsCSVPrinterFacade.java
index 9909d17..958dbdb 100644
--- a/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/commonscsv/impl/CommonsCSVPrinterFacade.java
+++ b/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/commonscsv/impl/CommonsCSVPrinterFacade.java
@@ -25,6 +25,9 @@ import java.io.IOException;
import java.io.StringWriter;
import java.sql.ResultSet;
import java.sql.SQLException;
+import java.util.ArrayList;
+import java.util.List;
+import java.util.Map;
/**
* Wrap <code>CSVPrinter</code> so each print method returns
@@ -91,6 +94,15 @@ public class CommonsCSVPrinterFacade implements Flushable, Closeable {
return getOutput();
}
+ public String printRecord(Map<String, Object> map, String[] headers) throws IOException {
+ final List<String> values = new ArrayList<>(headers.length);
+ for (String header : headers) {
+ final Object value = map.get(header);
+ values.add(value != null ? value.toString() : "");
+ }
+ return printRecord(values);
+ }
+
private String getOutput() {
writer.flush();
final String output = writer.getBuffer().toString();
diff --git a/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/utahparser/UtahParserTool.java b/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/utahparser/UtahParserTool.java
new file mode 100644
index 0000000..82520ef
--- /dev/null
+++ b/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/utahparser/UtahParserTool.java
@@ -0,0 +1,100 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.freemarker.generator.tools.utahparser;
+
+import com.sonalake.utah.Parser;
+import com.sonalake.utah.config.Config;
+import com.sonalake.utah.config.ConfigLoader;
+import org.apache.freemarker.generator.base.datasource.DataSource;
+import org.apache.freemarker.generator.base.datasource.DataSourceLoaderFactory;
+import org.apache.freemarker.generator.tools.utahparser.impl.ParserWrapper;
+
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.InputStreamReader;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Collections;
+import java.util.List;
+import java.util.Map;
+import java.util.stream.Collectors;
+
+public class UtahParserTool {
+
+ /**
+ * Create a Utah Parser config based on the given source.
+ *
+ * @param source source of configuration file
+ * @return config instance
+ */
+ public Config getConfig(String source) {
+ return getConfig(DataSourceLoaderFactory.create().load(source));
+ }
+
+ /**
+ * Create a Utah Parser config based on teh textual content of the data source,
+ *
+ * @param dataSource XML configuration file
+ * @return config instance
+ */
+ public Config getConfig(DataSource dataSource) {
+ try (InputStream is = dataSource.getUnsafeInputStream()) {
+ return loadConfig(is);
+ } catch (Exception e) {
+ throw new RuntimeException("Failed to load parser configuration: " + dataSource, e);
+ }
+ }
+
+ /**
+ * Create a parser instance.
+ *
+ * @param config configuration
+ * @param dataSource data source to be parsed
+ * @return parser
+ */
+ public ParserWrapper getParser(Config config, DataSource dataSource) {
+ final InputStreamReader is = new InputStreamReader(dataSource.getInputStream());
+ final Parser parser = Parser.parse(config, is);
+ return new ParserWrapper(parser);
+ }
+
+ public List<String> getHeaders(Map<String, Object> record) {
+ if (record == null || record.isEmpty()) {
+ return Collections.emptyList();
+ }
+
+ return new ArrayList<>(record.keySet());
+ }
+
+ public List<String> getHeaders(Collection<Map<String, String>> records) {
+ if (records == null || records.isEmpty()) {
+ return Collections.emptyList();
+ }
+
+ return records.stream()
+ .map(Map::keySet)
+ .flatMap(Collection::stream).distinct().sorted().collect(Collectors.toList());
+ }
+
+ public String toString() {
+ return "Parse semi-structured text using regular expressions (see https://github.com/sonalake/utah-parser)";
+ }
+
+ private static Config loadConfig(InputStream is) throws IOException {
+ return new ConfigLoader().loadConfig(new InputStreamReader(is));
+ }
+}
\ No newline at end of file
diff --git a/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/utahparser/impl/ParserWrapper.java b/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/utahparser/impl/ParserWrapper.java
new file mode 100644
index 0000000..9aad9bc
--- /dev/null
+++ b/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/utahparser/impl/ParserWrapper.java
@@ -0,0 +1,86 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.freemarker.generator.tools.utahparser.impl;
+
+import com.sonalake.utah.Parser;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.NoSuchElementException;
+
+import static java.util.Objects.requireNonNull;
+
+/**
+ * Wraps the <code>com.sonalake.utah.Parser</code> to provide convenience
+ * methods such as support of iterators.
+ */
+public class ParserWrapper implements Iterable<Map<String, String>> {
+
+ /** The wrapped parser instance */
+ private final Parser parser;
+
+ public ParserWrapper(Parser parser) {
+ this.parser = requireNonNull(parser);
+ }
+
+ @Override
+ public Iterator<Map<String, String>> iterator() {
+ return new RecordsIterator(parser);
+ }
+
+ /**
+ * Iterates over all records and return a list.
+ *
+ * @return list of records
+ */
+ public List<Map<String, String>> toList() {
+ final List<Map<String, String>> result = new ArrayList<>();
+ for (Map<String, String> record : this) {
+ result.add(record);
+ }
+ return result;
+ }
+
+ private static final class RecordsIterator implements Iterator<Map<String, String>> {
+
+ private final Parser parser;
+ private Map<String, String> nextRecord;
+
+ RecordsIterator(Parser parser) {
+ this.parser = requireNonNull(parser);
+ this.nextRecord = parser.next();
+ }
+
+ @Override
+ public boolean hasNext() {
+ return nextRecord != null;
+ }
+
+ @Override
+ public Map<String, String> next() {
+ if (nextRecord == null) {
+ throw new NoSuchElementException();
+ }
+
+ final Map<String, String> currentRecord = nextRecord;
+ nextRecord = parser.next();
+ return currentRecord;
+ }
+ }
+}
\ No newline at end of file
diff --git a/freemarker-generator-tools/src/test/data/utahparser/juniper_bgp_summary_example.txt b/freemarker-generator-tools/src/test/data/utahparser/juniper_bgp_summary_example.txt
new file mode 100644
index 0000000..c4a209e
--- /dev/null
+++ b/freemarker-generator-tools/src/test/data/utahparser/juniper_bgp_summary_example.txt
@@ -0,0 +1,12 @@
+Groups: 3 Peers: 3 Down peers: 0
+Table Tot Paths Act Paths Suppressed History Damp State Pending
+inet.0 947 310 0 0 0 0
+inet6.0 849 807 0 0 0 0
+Peer AS InPkt OutPkt OutQ Flaps Last Up/Dwn State|#Active/Received/Damped...
+10.247.68.182 65550 131725 28179233 0 11 6w3d17h Establ
+ inet.0: 4/5/1
+ inet6.0: 0/0/0
+10.254.166.246 65550 136159 29104942 0 0 6w5d6h Establ
+ inet.0: 0/0/0
+ inet6.0: 7/8/1
+192.0.2.100 65551 1269381 1363320 0 1 9w5d6h 1/2/3 4/5/6
\ No newline at end of file
diff --git a/freemarker-generator-tools/src/test/data/utahparser/juniper_bgp_summary_template.xml b/freemarker-generator-tools/src/test/data/utahparser/juniper_bgp_summary_template.xml
new file mode 100644
index 0000000..bb1093a
--- /dev/null
+++ b/freemarker-generator-tools/src/test/data/utahparser/juniper_bgp_summary_template.xml
@@ -0,0 +1,69 @@
+<config>
+ <searches>
+
+ <!-- in this case, we have a CSV (space delimited file) so we define the line once, and then reuse it over
+ and again for each value -->
+ <search id="QUERY-LINE"><![CDATA[\s*{ipAddress}\s+{numbers}\s+{numbers}\s+{numbers}\s+{numbers}\s+{numbers}\s+{numbersThenText}]]></search>
+
+
+ <search id="inetInline"><![CDATA[{inet} {inet}]]></search>
+ <search id="inet4"><![CDATA[inet.0:\s*{inet}]]></search>
+ <search id="inet6"><![CDATA[inet6.0:\s*{inet}]]></search>
+ <search id="inet"><![CDATA[{numbers}/{numbers}/{numbers}]]></search>
+
+ <!-- Some rules for finding text, to make the values a little easier below -->
+ <search id="numbers"><![CDATA[(\d+)]]></search>
+ <search id="numbersThenText"><![CDATA[(\d+\S+)]]></search>
+ <search id="string"><![CDATA[(\S+?)]]></search>
+ <search id="ipAddress"><![CDATA[(\d+(\.\d+){3})]]></search>
+ <search id="EOL"><![CDATA[[\n\r]]]></search>
+ </searches>
+
+ <!-- the record starts with a line with an ip address and ends with either an inet6 line, or where the ids are at
+ the end of the line-->
+ <delim retain="true">{ipAddress}.*(\/\d+)\s*{EOL}</delim>
+ <delim>\s*({inet6})</delim>
+
+ <!--
+ This is the last line of the header
+ -->
+ <header-delim><![CDATA[Peer\s+AS\s+InPkt]]></header-delim>
+
+ <!--
+ Files look like this:
+
+ 10.247.68.182 65550 131725 28179233 0 11 6w3d17h Establ
+ inet.0: 4/5/1
+ inet6.0: 0/0/0
+
+ or
+
+ 192.0.2.100 65551 1269381 1363320 0 1 9w5d6h 2/3/0 0/0/0
+ -->
+ <values>
+ <!-- here we reuse the line pattern, only we pull out different group values -->
+ <value id="remoteIp" group="1"><![CDATA[{QUERY-LINE}]]></value>
+ <value id="uptime" group="8"><![CDATA[{QUERY-LINE}]]></value>
+
+ <!-- here we check for values in the inet* lines and use these -->
+ <value id="activeV4" group="1"><![CDATA[{inet4}]]></value>
+ <value id="receivedV4" group="2"><![CDATA[{inet4}]]></value>
+ <value id="accepted_V4" group="3"><![CDATA[{inet4}]]></value>
+
+ <value id="activeV6" group="1"><![CDATA[{inet6}]]></value>
+ <value id="receivedV6" group="2"><![CDATA[{inet6}]]></value>
+ <value id="accepted_V6" group="3"><![CDATA[{inet6}]]></value>
+
+ <!--
+ here we check for values at the end of the query line, and use these
+ NOTE: since we only set non-null values, these will not overwrite any values set above
+ -->
+ <value id="activeV4" group="9"><![CDATA[{QUERY-LINE}\s*{inetInline}]]></value>
+ <value id="receivedV4" group="10"><![CDATA[{QUERY-LINE}\s*{inetInline}]]></value>
+ <value id="accepted_V4" group="11"><![CDATA[{QUERY-LINE}\s*{inetInline}]]></value>
+ <value id="activeV6" group="12"><![CDATA[{QUERY-LINE}\s*{inetInline}]]></value>
+ <value id="receivedV6" group="13"><![CDATA[{QUERY-LINE}\s*{inetInline}]]></value>
+ <value id="accepted_V6" group="14"><![CDATA[{QUERY-LINE}\s*{inetInline}]]></value>
+
+ </values>
+</config>
\ No newline at end of file
diff --git a/freemarker-generator-tools/src/test/java/org/apache/freemarker/generator/tools/utahparser/UtahParserToolTest.java b/freemarker-generator-tools/src/test/java/org/apache/freemarker/generator/tools/utahparser/UtahParserToolTest.java
new file mode 100644
index 0000000..fac9e75
--- /dev/null
+++ b/freemarker-generator-tools/src/test/java/org/apache/freemarker/generator/tools/utahparser/UtahParserToolTest.java
@@ -0,0 +1,103 @@
+package org.apache.freemarker.generator.tools.utahparser;
+
+import com.sonalake.utah.config.Config;
+import org.apache.freemarker.generator.base.datasource.DataSource;
+import org.apache.freemarker.generator.base.datasource.DataSourceFactory;
+import org.apache.freemarker.generator.base.util.MapBuilder;
+import org.apache.freemarker.generator.tools.utahparser.impl.ParserWrapper;
+import org.junit.Test;
+
+import java.io.File;
+import java.util.List;
+import java.util.Map;
+
+import static java.nio.charset.StandardCharsets.UTF_8;
+import static junit.framework.TestCase.assertEquals;
+import static junit.framework.TestCase.assertNotNull;
+import static junit.framework.TestCase.assertTrue;
+
+public class UtahParserToolTest {
+
+ private static final String EXAMPLE_FILE_NAME = "src/test/data/utahparser/juniper_bgp_summary_example.txt";
+ private static final String TEMPLATE_FILE_NAME = "src/test/data/utahparser/juniper_bgp_summary_template.xml";
+
+ @Test
+
+ public void shallLoadConfFromFile() {
+ final Config config = utahParserTool().getConfig(TEMPLATE_FILE_NAME);
+
+ assertNotNull(config);
+ assertTrue(config.isDelimiterValid());
+ }
+
+ @Test
+ public void shallLoadConfFromDataSource() {
+ final DataSource dataSource = dataSource(TEMPLATE_FILE_NAME);
+ final Config config = utahParserTool().getConfig(dataSource);
+
+ assertNotNull(config);
+ assertTrue(config.isDelimiterValid());
+ }
+
+ @Test
+ public void shallGetParserInstance() {
+ final DataSource dataSource = dataSource(EXAMPLE_FILE_NAME);
+ final UtahParserTool utahParserTool = utahParserTool();
+ final Config config = utahParserTool.getConfig(TEMPLATE_FILE_NAME);
+ final ParserWrapper parser = utahParserTool.getParser(config, dataSource);
+
+ assertNotNull(parser);
+ assertNotNull(parser.iterator());
+ }
+
+ @Test
+ public void shallParseAllData() {
+ final DataSource dataSource = dataSource(EXAMPLE_FILE_NAME);
+ final UtahParserTool utahParserTool = utahParserTool();
+ final Config config = utahParserTool.getConfig(TEMPLATE_FILE_NAME);
+ final ParserWrapper parser = utahParserTool.getParser(config, dataSource);
+ final List<Map<String, String>> records = parser.toList();
+
+ assertEquals(3, records.size());
+ }
+
+ @Test
+ public void shallGetHeadersFromRecord() {
+ final UtahParserTool utahParserTool = utahParserTool();
+ final Map<String, Object> record = MapBuilder.toLinkedMap("header1", "foo", "header2", "bar");
+
+ final List<String> headers = utahParserTool.getHeaders(record);
+
+ assertEquals("header1", headers.get(0));
+ assertEquals("header2", headers.get(1));
+ }
+
+ @Test
+ public void shallGetHeadersFromRecords() {
+ final DataSource dataSource = dataSource(EXAMPLE_FILE_NAME);
+ final UtahParserTool utahParserTool = utahParserTool();
+ final Config config = utahParserTool.getConfig(TEMPLATE_FILE_NAME);
+ final ParserWrapper parser = utahParserTool.getParser(config, dataSource);
+ final List<Map<String, String>> records = parser.toList();
+
+ final List<String> headers = utahParserTool.getHeaders(records);
+
+ assertEquals("accepted_V4", headers.get(0));
+ assertEquals("accepted_V6", headers.get(1));
+ assertEquals("activeV4", headers.get(2));
+ assertEquals("activeV6", headers.get(3));
+ assertEquals("receivedV4", headers.get(4));
+ assertEquals("receivedV6", headers.get(5));
+ assertEquals("remoteIp", headers.get(6));
+ assertEquals("uptime", headers.get(7));
+ }
+
+ private static UtahParserTool utahParserTool() {
+ return new UtahParserTool();
+ }
+
+ private static DataSource dataSource(String fileName) {
+ return DataSourceFactory.fromFile(new File(fileName), UTF_8);
+ }
+
+}
diff --git a/licences/LICENCE_utahparser.txt b/licences/LICENCE_utahparser.txt
new file mode 100644
index 0000000..9c8f3ea
--- /dev/null
+++ b/licences/LICENCE_utahparser.txt
@@ -0,0 +1,201 @@
+ Apache License
+ Version 2.0, January 2004
+ http://www.apache.org/licenses/
+
+ TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+ 1. Definitions.
+
+ "License" shall mean the terms and conditions for use, reproduction,
+ and distribution as defined by Sections 1 through 9 of this document.
+
+ "Licensor" shall mean the copyright owner or entity authorized by
+ the copyright owner that is granting the License.
+
+ "Legal Entity" shall mean the union of the acting entity and all
+ other entities that control, are controlled by, or are under common
+ control with that entity. For the purposes of this definition,
+ "control" means (i) the power, direct or indirect, to cause the
+ direction or management of such entity, whether by contract or
+ otherwise, or (ii) ownership of fifty percent (50%) or more of the
+ outstanding shares, or (iii) beneficial ownership of such entity.
+
+ "You" (or "Your") shall mean an individual or Legal Entity
+ exercising permissions granted by this License.
+
+ "Source" form shall mean the preferred form for making modifications,
+ including but not limited to software source code, documentation
+ source, and configuration files.
+
+ "Object" form shall mean any form resulting from mechanical
+ transformation or translation of a Source form, including but
+ not limited to compiled object code, generated documentation,
+ and conversions to other media types.
+
+ "Work" shall mean the work of authorship, whether in Source or
+ Object form, made available under the License, as indicated by a
+ copyright notice that is included in or attached to the work
+ (an example is provided in the Appendix below).
+
+ "Derivative Works" shall mean any work, whether in Source or Object
+ form, that is based on (or derived from) the Work and for which the
+ editorial revisions, annotations, elaborations, or other modifications
+ represent, as a whole, an original work of authorship. For the purposes
+ of this License, Derivative Works shall not include works that remain
+ separable from, or merely link (or bind by name) to the interfaces of,
+ the Work and Derivative Works thereof.
+
+ "Contribution" shall mean any work of authorship, including
+ the original version of the Work and any modifications or additions
+ to that Work or Derivative Works thereof, that is intentionally
+ submitted to Licensor for inclusion in the Work by the copyright owner
+ or by an individual or Legal Entity authorized to submit on behalf of
+ the copyright owner. For the purposes of this definition, "submitted"
+ means any form of electronic, verbal, or written communication sent
+ to the Licensor or its representatives, including but not limited to
+ communication on electronic mailing lists, source code control systems,
+ and issue tracking systems that are managed by, or on behalf of, the
+ Licensor for the purpose of discussing and improving the Work, but
+ excluding communication that is conspicuously marked or otherwise
+ designated in writing by the copyright owner as "Not a Contribution."
+
+ "Contributor" shall mean Licensor and any individual or Legal Entity
+ on behalf of whom a Contribution has been received by Licensor and
+ subsequently incorporated within the Work.
+
+ 2. Grant of Copyright License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ copyright license to reproduce, prepare Derivative Works of,
+ publicly display, publicly perform, sublicense, and distribute the
+ Work and such Derivative Works in Source or Object form.
+
+ 3. Grant of Patent License. Subject to the terms and conditions of
+ this License, each Contributor hereby grants to You a perpetual,
+ worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+ (except as stated in this section) patent license to make, have made,
+ use, offer to sell, sell, import, and otherwise transfer the Work,
+ where such license applies only to those patent claims licensable
+ by such Contributor that are necessarily infringed by their
+ Contribution(s) alone or by combination of their Contribution(s)
+ with the Work to which such Contribution(s) was submitted. If You
+ institute patent litigation against any entity (including a
+ cross-claim or counterclaim in a lawsuit) alleging that the Work
+ or a Contribution incorporated within the Work constitutes direct
+ or contributory patent infringement, then any patent licenses
+ granted to You under this License for that Work shall terminate
+ as of the date such litigation is filed.
+
+ 4. Redistribution. You may reproduce and distribute copies of the
+ Work or Derivative Works thereof in any medium, with or without
+ modifications, and in Source or Object form, provided that You
+ meet the following conditions:
+
+ (a) You must give any other recipients of the Work or
+ Derivative Works a copy of this License; and
+
+ (b) You must cause any modified files to carry prominent notices
+ stating that You changed the files; and
+
+ (c) You must retain, in the Source form of any Derivative Works
+ that You distribute, all copyright, patent, trademark, and
+ attribution notices from the Source form of the Work,
+ excluding those notices that do not pertain to any part of
+ the Derivative Works; and
+
+ (d) If the Work includes a "NOTICE" text file as part of its
+ distribution, then any Derivative Works that You distribute must
+ include a readable copy of the attribution notices contained
+ within such NOTICE file, excluding those notices that do not
+ pertain to any part of the Derivative Works, in at least one
+ of the following places: within a NOTICE text file distributed
+ as part of the Derivative Works; within the Source form or
+ documentation, if provided along with the Derivative Works; or,
+ within a display generated by the Derivative Works, if and
+ wherever such third-party notices normally appear. The contents
+ of the NOTICE file are for informational purposes only and
+ do not modify the License. You may add Your own attribution
+ notices within Derivative Works that You distribute, alongside
+ or as an addendum to the NOTICE text from the Work, provided
+ that such additional attribution notices cannot be construed
+ as modifying the License.
+
+ You may add Your own copyright statement to Your modifications and
+ may provide additional or different license terms and conditions
+ for use, reproduction, or distribution of Your modifications, or
+ for any such Derivative Works as a whole, provided Your use,
+ reproduction, and distribution of the Work otherwise complies with
+ the conditions stated in this License.
+
+ 5. Submission of Contributions. Unless You explicitly state otherwise,
+ any Contribution intentionally submitted for inclusion in the Work
+ by You to the Licensor shall be under the terms and conditions of
+ this License, without any additional terms or conditions.
+ Notwithstanding the above, nothing herein shall supersede or modify
+ the terms of any separate license agreement you may have executed
+ with Licensor regarding such Contributions.
+
+ 6. Trademarks. This License does not grant permission to use the trade
+ names, trademarks, service marks, or product names of the Licensor,
+ except as required for reasonable and customary use in describing the
+ origin of the Work and reproducing the content of the NOTICE file.
+
+ 7. Disclaimer of Warranty. Unless required by applicable law or
+ agreed to in writing, Licensor provides the Work (and each
+ Contributor provides its Contributions) on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+ implied, including, without limitation, any warranties or conditions
+ of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+ PARTICULAR PURPOSE. You are solely responsible for determining the
+ appropriateness of using or redistributing the Work and assume any
+ risks associated with Your exercise of permissions under this License.
+
+ 8. Limitation of Liability. In no event and under no legal theory,
+ whether in tort (including negligence), contract, or otherwise,
+ unless required by applicable law (such as deliberate and grossly
+ negligent acts) or agreed to in writing, shall any Contributor be
+ liable to You for damages, including any direct, indirect, special,
+ incidental, or consequential damages of any character arising as a
+ result of this License or out of the use or inability to use the
+ Work (including but not limited to damages for loss of goodwill,
+ work stoppage, computer failure or malfunction, or any and all
+ other commercial damages or losses), even if such Contributor
+ has been advised of the possibility of such damages.
+
+ 9. Accepting Warranty or Additional Liability. While redistributing
+ the Work or Derivative Works thereof, You may choose to offer,
+ and charge a fee for, acceptance of support, warranty, indemnity,
+ or other liability obligations and/or rights consistent with this
+ License. However, in accepting such obligations, You may act only
+ on Your own behalf and on Your sole responsibility, not on behalf
+ of any other Contributor, and only if You agree to indemnify,
+ defend, and hold each Contributor harmless for any liability
+ incurred by, or claims asserted against, such Contributor by reason
+ of your accepting any such warranty or additional liability.
+
+ END OF TERMS AND CONDITIONS
+
+ APPENDIX: How to apply the Apache License to your work.
+
+ To apply the Apache License to your work, attach the following
+ boilerplate notice, with the fields enclosed by brackets "{}"
+ replaced with your own identifying information. (Don't include
+ the brackets!) The text should be enclosed in the appropriate
+ comment syntax for the file format. We also recommend that a
+ file or class name and description of purpose be included on the
+ same "printed page" as the copyright notice for easier
+ identification within third-party archives.
+
+ Copyright {yyyy} {name of copyright owner}
+
+ Licensed under the Apache License, Version 2.0 (the "License");
+ you may not use this file except in compliance with the License.
+ You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
\ No newline at end of file