You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@freemarker.apache.org by sg...@apache.org on 2020/04/06 10:31:37 UTC
[freemarker-generator] branch feature/FREEMARKER-140 updated:
FREEMARKER-140 freemarker-cli: Expose DataSources directly in the data
model
This is an automated email from the ASF dual-hosted git repository.
sgoeschl pushed a commit to branch feature/FREEMARKER-140
in repository https://gitbox.apache.org/repos/asf/freemarker-generator.git
The following commit(s) were added to refs/heads/feature/FREEMARKER-140 by this push:
new e21e8ac FREEMARKER-140 freemarker-cli: Expose DataSources directly in the data model
e21e8ac is described below
commit e21e8acf185375b81b599225025909e3e97b23ca
Author: Siegfried Goeschl <si...@gmail.com>
AuthorDate: Mon Apr 6 12:31:23 2020 +0200
FREEMARKER-140 freemarker-cli: Expose DataSources directly in the data model
---
.../base/activation/CachingUrlDataSource.java | 26 ++++++++++
.../generator/base/datasource/DataSource.java | 29 +++++++++--
.../base/datasource/DataSourceFactory.java | 3 +-
.../datasource/DataSourceFactoryTest.java | 18 +++++--
.../generator/cli/config/DataModelsSupplier.java | 17 +++----
.../site/markdown/cli/working-with-datamodels.md | 57 ++++++++++++++++++++++
.../freemarker/generator/cli/ManualTest.java | 5 +-
7 files changed, 135 insertions(+), 20 deletions(-)
diff --git a/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/activation/CachingUrlDataSource.java b/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/activation/CachingUrlDataSource.java
new file mode 100644
index 0000000..24e02ac
--- /dev/null
+++ b/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/activation/CachingUrlDataSource.java
@@ -0,0 +1,26 @@
+package org.apache.freemarker.generator.base.activation;
+
+import javax.activation.URLDataSource;
+import java.net.URL;
+
+/**
+ * The standard UrlDataSource actually does network calls when
+ * getting the content type. Try to avoid multiple calls to
+ * determine the content type.
+ */
+public class CachingUrlDataSource extends URLDataSource {
+
+ private String contentType;
+
+ public CachingUrlDataSource(URL url) {
+ super(url);
+ }
+
+ @Override
+ public synchronized String getContentType() {
+ if (contentType == null) {
+ contentType = super.getContentType();
+ }
+ return contentType;
+ }
+}
diff --git a/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/datasource/DataSource.java b/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/datasource/DataSource.java
index d968e54..35cac8e 100644
--- a/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/datasource/DataSource.java
+++ b/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/datasource/DataSource.java
@@ -31,12 +31,18 @@ import java.io.StringWriter;
import java.net.URI;
import java.nio.charset.Charset;
import java.util.List;
+import java.util.Objects;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+import java.util.stream.Stream;
import static java.nio.charset.Charset.forName;
+import static java.nio.charset.StandardCharsets.UTF_8;
import static java.util.Objects.requireNonNull;
import static org.apache.commons.io.IOUtils.lineIterator;
import static org.apache.freemarker.generator.base.FreeMarkerConstants.DATASOURCE_UNKNOWN_LENGTH;
import static org.apache.freemarker.generator.base.util.StringUtils.emptyToNull;
+import static org.apache.freemarker.generator.base.util.StringUtils.firstNonEmpty;
/**
* Data source which encapsulates data to be used for rendering
@@ -45,6 +51,9 @@ import static org.apache.freemarker.generator.base.util.StringUtils.emptyToNull;
*/
public class DataSource implements Closeable {
+ /** Parse something like "application/json; charset=utf-8" */
+ private static final Pattern CHARSET_PATTERN = Pattern.compile("(?i)\\bcharset=\\s*\"?([^\\s;\"]*)");
+
/** Human-readable name of the data source */
private final String name;
@@ -60,7 +69,7 @@ public class DataSource implements Closeable {
/** Optional user-supplied content type */
private final String contentType;
- /** Charset for directly accessing text-based content */
+ /** Optional charset for directly accessing text-based content */
private final Charset charset;
/** Collect all closables handed out to the caller to be closed when the data source is closed itself */
@@ -78,7 +87,7 @@ public class DataSource implements Closeable {
this.uri = requireNonNull(uri);
this.dataSource = requireNonNull(dataSource);
this.contentType = contentType;
- this.charset = requireNonNull(charset);
+ this.charset = charset;
this.closables = new CloseableReaper();
}
@@ -99,11 +108,14 @@ public class DataSource implements Closeable {
}
public Charset getCharset() {
- return charset;
+ return Stream.of(charset, parseCharsetFromContentType(getContentType()))
+ .filter(Objects::nonNull)
+ .findFirst()
+ .orElse(UTF_8);
}
public String getContentType() {
- return contentType != null ? contentType : dataSource.getContentType();
+ return firstNonEmpty(contentType, dataSource.getContentType());
}
public URI getUri() {
@@ -250,4 +262,13 @@ public class DataSource implements Closeable {
", charset=" + charset +
'}';
}
+
+ private Charset parseCharsetFromContentType(String contentType) {
+ final Matcher matcher = CHARSET_PATTERN.matcher(contentType);
+ if (matcher.find()) {
+ final String name = matcher.group(1).trim().toUpperCase();
+ return Charset.forName(name);
+ }
+ return null;
+ }
}
diff --git a/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/datasource/DataSourceFactory.java b/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/datasource/DataSourceFactory.java
index 47bea88..4b5d976 100644
--- a/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/datasource/DataSourceFactory.java
+++ b/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/datasource/DataSourceFactory.java
@@ -18,6 +18,7 @@ package org.apache.freemarker.generator.base.datasource;
import org.apache.freemarker.generator.base.FreeMarkerConstants.Location;
import org.apache.freemarker.generator.base.activation.ByteArrayDataSource;
+import org.apache.freemarker.generator.base.activation.CachingUrlDataSource;
import org.apache.freemarker.generator.base.activation.InputStreamDataSource;
import org.apache.freemarker.generator.base.activation.MimetypesFileTypeMapFactory;
import org.apache.freemarker.generator.base.activation.StringDataSource;
@@ -96,7 +97,7 @@ public class DataSourceFactory {
}
public static DataSource fromUrl(String name, String group, URL url, String contentType, Charset charset) {
- final URLDataSource dataSource = new URLDataSource(url);
+ final URLDataSource dataSource = new CachingUrlDataSource(url);
final URI uri = UriUtils.toURI(url);
return create(name, group, uri, dataSource, contentType, charset);
}
diff --git a/freemarker-generator-base/src/test/java/org/apache/freemarker/generator/datasource/DataSourceFactoryTest.java b/freemarker-generator-base/src/test/java/org/apache/freemarker/generator/datasource/DataSourceFactoryTest.java
index d8b7b4b..035566e 100644
--- a/freemarker-generator-base/src/test/java/org/apache/freemarker/generator/datasource/DataSourceFactoryTest.java
+++ b/freemarker-generator-base/src/test/java/org/apache/freemarker/generator/datasource/DataSourceFactoryTest.java
@@ -26,6 +26,7 @@ import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
+import java.net.URL;
import java.nio.charset.Charset;
import static java.nio.charset.StandardCharsets.UTF_8;
@@ -79,7 +80,7 @@ public class DataSourceFactoryTest {
}
@Test
- public void shouldCreateDataSourceFromBytes() throws IOException {
+ public void shouldCreateDataSourceFromBytes() {
final DataSource dataSource = DataSourceFactory.fromBytes("test.txt", "default", ANY_TEXT.getBytes(UTF_8), "text/plain");
assertEquals("test.txt", dataSource.getName());
@@ -91,7 +92,7 @@ public class DataSourceFactoryTest {
}
@Test
- public void shouldCreateDataSourceFromInputStream() throws IOException {
+ public void shouldCreateDataSourceFromInputStream() {
final InputStream is = new ByteArrayInputStream(ANY_TEXT.getBytes(UTF_8));
final DataSource dataSource = DataSourceFactory.fromInputStream("test.txt", "default", is, "text/plain", UTF_8);
@@ -103,13 +104,22 @@ public class DataSourceFactoryTest {
@Test
public void shouldCreateDataSourceFromURL() throws IOException {
+ final URL url = new URL("https://jsonplaceholder.typicode.com/posts/2");
+ final DataSource dataSource = DataSourceFactory.fromUrl("jsonplaceholder.typicode.com", "default", url, null, null);
+
+ assertEquals("jsonplaceholder.typicode.com", dataSource.getName());
+ assertEquals("application/json; charset=utf-8", dataSource.getContentType());
+ assertEquals(UTF_8, dataSource.getCharset());
+ }
+
+ @Test
+ public void shouldCreateDataSourceFromNamedURL() {
final NamedUri namedUri = NamedUriStringParser.parse(ANY_NAMED_URL_STRING);
- final DataSource dataSource = DataSourceFactory.fromNamedUri(ANY_NAMED_URL_STRING);
+ final DataSource dataSource = DataSourceFactory.fromNamedUri(namedUri);
assertEquals(namedUri.getName(), dataSource.getName());
assertEquals(namedUri.getGroup(), dataSource.getGroup());
assertEquals(UTF_8, dataSource.getCharset());
assertEquals(namedUri.getUri().toString(), dataSource.getUri().toString());
}
-
}
diff --git a/freemarker-generator-cli/src/main/java/org/apache/freemarker/generator/cli/config/DataModelsSupplier.java b/freemarker-generator-cli/src/main/java/org/apache/freemarker/generator/cli/config/DataModelsSupplier.java
index 22251d0..c5e8c59 100644
--- a/freemarker-generator-cli/src/main/java/org/apache/freemarker/generator/cli/config/DataModelsSupplier.java
+++ b/freemarker-generator-cli/src/main/java/org/apache/freemarker/generator/cli/config/DataModelsSupplier.java
@@ -71,15 +71,14 @@ public class DataModelsSupplier implements Supplier<Map<String, Object>> {
final boolean isExplodedDataModel = !namedUri.hasName();
final String contentType = dataSource.getContentType();
- switch (contentType) {
- case MIME_APPLICATION_JSON:
- return fromJson(dataSource, isExplodedDataModel);
- case MIME_TEXT_PLAIN:
- return fromProperties(dataSource, isExplodedDataModel);
- case MIME_TEXT_YAML:
- return fromYaml(dataSource, isExplodedDataModel);
- default:
- throw new IllegalArgumentException("Don't know how to handle :" + contentType);
+ if (contentType.startsWith(MIME_APPLICATION_JSON)) {
+ return fromJson(dataSource, isExplodedDataModel);
+ } else if (contentType.startsWith(MIME_TEXT_PLAIN)) {
+ return fromProperties(dataSource, isExplodedDataModel);
+ } else if (contentType.startsWith(MIME_TEXT_YAML)) {
+ return fromYaml(dataSource, isExplodedDataModel);
+ } else {
+ throw new IllegalArgumentException("Don't know how to handle :" + contentType);
}
}
diff --git a/freemarker-generator-cli/src/site/markdown/cli/working-with-datamodels.md b/freemarker-generator-cli/src/site/markdown/cli/working-with-datamodels.md
new file mode 100644
index 0000000..6b46397
--- /dev/null
+++ b/freemarker-generator-cli/src/site/markdown/cli/working-with-datamodels.md
@@ -0,0 +1,57 @@
+## Working With DataModels
+
+A `DataModel` is an eagerly loaded `DataSource` available in Apache FreeMarker's model (context) when rendering a template.
+
+* The content of the `DataSource` is parsed and a `Map` generated
+* The `Map` is either stored as variable in the model or all entries are copied into the FreeMarker model
+* The parsing is supported for `JSON`, `YAML`, `Properties` and enviroment variables
+
+Expose the fields of the JSON data source in FreeMarker's model
+
+```
+bin/freemarker-cli --data-model https://xkcd.com/info.0.json -i '<a href="${img}">${title}</a>'
+<a href="https://imgs.xkcd.com/comics/scenario_4.png">Scenario 4</a>
+```
+
+Exposed the JSON data source as variable `post` in FreeMarker's model
+
+```
+bin/freemarker-cli --data-model post=https://jsonplaceholder.typicode.com/posts/2 -i 'post title is: ${post.title}'
+post title is: qui est esse
+```
+
+Expose all environment variables as `env` in theFreeMarker model
+
+```
+bin/freemarker-cli --data-model env=env:/// -i '<#list env as name,value>${name}=${value}${"\n"}</#list>'
+HOME=/Users/sgoeschl
+USER=sgoeschl
+```
+
+Expose a single envionment variable in theFreeMarker model
+
+```
+bin/freemarker-cli --data-model NAME=env:///USER -i 'Hello ${NAME}'
+Hello sgoeschl
+```
+
+Alternatively use the short command line options, e.g.
+
+```
+bin/freemarker-cli -m NAME=env:///USER -i 'Hello ${NAME}!'
+Hello sgoeschl!
+```
+
+The following snippet shows a more advanced example
+
+* The environment variable `DB_CONFIG` holds JSON data
+* Use the `config=env:///DB_CONFIG#mimetype=application/json` to parse JSON payload from `DB_CONFIG` into the data model `config`
+
+```
+> export DB_CONFIG='{"db_default_user":"scott","db_default_password":"tiger"}'
+> echo $DB_CONFIG
+{"db_default_user":"scott","db_default_password":"tiger"}
+> bin/freemarker-cli -m config=env:///DB_CONFIG#mimetype=application/json -i '<#list config as name,value>${name}=${value}${"\n"}</#list>'
+db_default_user=scott
+db_default_password=tiger
+```
\ No newline at end of file
diff --git a/freemarker-generator-cli/src/test/java/org/apache/freemarker/generator/cli/ManualTest.java b/freemarker-generator-cli/src/test/java/org/apache/freemarker/generator/cli/ManualTest.java
index 16aec0c..59952de 100644
--- a/freemarker-generator-cli/src/test/java/org/apache/freemarker/generator/cli/ManualTest.java
+++ b/freemarker-generator-cli/src/test/java/org/apache/freemarker/generator/cli/ManualTest.java
@@ -45,8 +45,9 @@ public class ManualTest {
// private static final String CMD = "-b ./src/test -t templates/demo.ftl -m env=env:///";
// private static final String CMD = "-b ./src/test -t templates/demo.ftl -m api=https://httpbin.org/get";
// private static final String CMD = "-b ./src/test -t templates/demo.ftl -m env:///HOME";
- // private static final String CMD = "-b ./src/test -t templates/demo.ftl -m env=./site/sample/properties/user_0001/user.properties";
- private static final String CMD = "-b ./src/test -t templates/demo.ftl -m ./site/sample/properties/user_0001/user.properties";
+ private static final String CMD = "-b ./src/test -t templates/demo.ftl -m env=./site/sample/properties/user_0001/user.properties";
+ // private static final String CMD = "-b ./src/test -t templates/demo.ftl -m ./site/sample/properties/user_0001/user.properties";
+ // private static final String CMD = "-b ./src/test --data-model post=https://jsonplaceholder.typicode.com/posts/2 -t templates/info.ftl";
public static void main(String[] args) {
Main.execute(toArgs(CMD));