You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@freemarker.apache.org by sg...@apache.org on 2020/04/09 08:22:48 UTC
[freemarker-generator] branch master updated: FREEMARKER-140
freemarker-cli: Expose DataSources directly in the data model
This is an automated email from the ASF dual-hosted git repository.
sgoeschl pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/freemarker-generator.git
The following commit(s) were added to refs/heads/master by this push:
new 9cd1bc9 FREEMARKER-140 freemarker-cli: Expose DataSources directly in the data model
9cd1bc9 is described below
commit 9cd1bc91787f46516c31b4040c33a25b95823bac
Author: Siegfried Goeschl <si...@gmail.com>
AuthorDate: Thu Apr 9 10:22:37 2020 +0200
FREEMARKER-140 freemarker-cli: Expose DataSources directly in the data model
---
.../base/activation/CachingUrlDataSource.java | 26 +++
.../generator/base/datasource/DataSource.java | 88 +++++++---
.../base/datasource/DataSourceFactory.java | 62 +++++--
.../generator/base/util/PropertiesFactory.java | 7 +
.../freemarker/generator/base/util/UriUtils.java | 15 +-
.../src/test/data/properties/test.properties | 3 +-
.../datasource/DataSourceFactoryTest.java | 65 ++++++-
.../generator/file/PropertiesFileSupplierTest.java | 3 +-
freemarker-generator-cli/pom.xml | 2 +
.../org/apache/freemarker/generator/cli/Main.java | 14 +-
.../generator/cli/config/DataModelSupplier.java | 128 ++++++++++++++
.../freemarker/generator/cli/config/Settings.java | 25 ++-
.../freemarker/generator/cli/config/Suppliers.java | 4 +
.../generator/cli/task/FreeMarkerTask.java | 29 ++--
.../src/site/markdown/cli/concepts/data-models.md | 57 +++++++
.../src/site/markdown/cli/concepts/named-uris.md | 59 +++++++
.../src/site/markdown/index.md | 7 +-
.../src/test/data/json/environments.json | 4 +
.../src/test/data/properties/test.properties | 3 +-
.../src/test/data/yaml/environments.yaml | 3 +
.../freemarker/generator/cli/ManualTest.java | 10 +-
.../freemarker/generator/cli/PicocliTest.java | 19 ++-
.../cli/config/DataModelSupplierTest.java | 188 +++++++++++++++++++++
.../freemarker/generator/tools/gson/GsonTool.java | 4 +-
pom.xml | 2 +-
25 files changed, 756 insertions(+), 71 deletions(-)
diff --git a/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/activation/CachingUrlDataSource.java b/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/activation/CachingUrlDataSource.java
new file mode 100644
index 0000000..24e02ac
--- /dev/null
+++ b/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/activation/CachingUrlDataSource.java
@@ -0,0 +1,26 @@
+package org.apache.freemarker.generator.base.activation;
+
+import javax.activation.URLDataSource;
+import java.net.URL;
+
+/**
+ * The standard UrlDataSource actually does network calls when
+ * getting the content type. Try to avoid multiple calls to
+ * determine the content type.
+ */
+public class CachingUrlDataSource extends URLDataSource {
+
+ private String contentType;
+
+ public CachingUrlDataSource(URL url) {
+ super(url);
+ }
+
+ @Override
+ public synchronized String getContentType() {
+ if (contentType == null) {
+ contentType = super.getContentType();
+ }
+ return contentType;
+ }
+}
diff --git a/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/datasource/DataSource.java b/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/datasource/DataSource.java
index 9f2ee51..4864dc6 100644
--- a/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/datasource/DataSource.java
+++ b/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/datasource/DataSource.java
@@ -31,20 +31,31 @@ import java.io.StringWriter;
import java.net.URI;
import java.nio.charset.Charset;
import java.util.List;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
import static java.nio.charset.Charset.forName;
+import static java.nio.charset.StandardCharsets.UTF_8;
import static java.util.Objects.requireNonNull;
import static org.apache.commons.io.IOUtils.lineIterator;
import static org.apache.freemarker.generator.base.FreeMarkerConstants.DATASOURCE_UNKNOWN_LENGTH;
import static org.apache.freemarker.generator.base.util.StringUtils.emptyToNull;
+import static org.apache.freemarker.generator.base.util.StringUtils.isNotEmpty;
/**
* Data source which encapsulates data to be used for rendering
* a template. When accessing content it is loaded on demand on not
* kept in memory to allow processing of large volumes of data.
+ * <br>
+ * There is also special support of <code>UrlDataSource</code> since
+ * the content type & charset might be determined using a network
+ * call.
*/
public class DataSource implements Closeable {
+ /** Parse something like "application/json; charset=utf-8" */
+ private static final Pattern CHARSET_PATTERN = Pattern.compile("(?i)\\bcharset=\\s*\"?([^\\s;\"]*)");
+
/** Human-readable name of the data source */
private final String name;
@@ -60,7 +71,7 @@ public class DataSource implements Closeable {
/** Optional user-supplied content type */
private final String contentType;
- /** Charset for directly accessing text-based content */
+ /** Optional charset for directly accessing text-based content */
private final Charset charset;
/** Collect all closables handed out to the caller to be closed when the data source is closed itself */
@@ -77,8 +88,8 @@ public class DataSource implements Closeable {
this.group = emptyToNull(group);
this.uri = requireNonNull(uri);
this.dataSource = requireNonNull(dataSource);
- this.contentType = requireNonNull(contentType);
- this.charset = requireNonNull(charset);
+ this.contentType = contentType;
+ this.charset = charset;
this.closables = new CloseableReaper();
}
@@ -99,11 +110,16 @@ public class DataSource implements Closeable {
}
public Charset getCharset() {
- return charset;
+ return charset != null ? charset : getCharsetFromContentType(contentType(), UTF_8);
}
+ /**
+ * Get the content type without additional parameters, e.g. "charset".
+ *
+ * @return content type
+ */
public String getContentType() {
- return contentType;
+ return stripExtraParameterFronContentType(contentType());
}
public URI getUri() {
@@ -131,9 +147,8 @@ public class DataSource implements Closeable {
* Get an input stream which is closed together with this data source.
*
* @return InputStream
- * @throws IOException Operation failed
*/
- public InputStream getInputStream() throws IOException {
+ public InputStream getInputStream() {
return closables.add(getUnsafeInputStream());
}
@@ -141,21 +156,26 @@ public class DataSource implements Closeable {
* Get an input stream which needs to be closed by the caller.
*
* @return InputStream
- * @throws IOException Operation failed
*/
- public InputStream getUnsafeInputStream() throws IOException {
- return dataSource.getInputStream();
+ public InputStream getUnsafeInputStream() {
+ try {
+ return dataSource.getInputStream();
+ } catch (IOException e) {
+ throw new RuntimeException("Failed to get input stream: " + toString(), e);
+ }
}
- public String getText() throws IOException {
+ public String getText() {
return getText(getCharset().name());
}
- public String getText(String charsetName) throws IOException {
+ public String getText(String charsetName) {
final StringWriter writer = new StringWriter();
try (InputStream is = getUnsafeInputStream()) {
IOUtils.copy(is, writer, forName(charsetName));
return writer.toString();
+ } catch (IOException e) {
+ throw new RuntimeException("Failed to get text: " + toString(), e);
}
}
@@ -164,9 +184,8 @@ public class DataSource implements Closeable {
* one entry per line, using the specified character encoding.
*
* @return the list of Strings, never null
- * @throws IOException if an I/O error occurs
*/
- public List<String> getLines() throws IOException {
+ public List<String> getLines() {
return getLines(getCharset().name());
}
@@ -176,11 +195,12 @@ public class DataSource implements Closeable {
*
* @param charsetName The name of the requested charset
* @return the list of Strings, never null
- * @throws IOException if an I/O error occurs
*/
- public List<String> getLines(String charsetName) throws IOException {
+ public List<String> getLines(String charsetName) {
try (InputStream inputStream = getUnsafeInputStream()) {
return IOUtils.readLines(inputStream, charsetName);
+ } catch (IOException e) {
+ throw new RuntimeException("Failed to get lines: " + toString(), e);
}
}
@@ -190,9 +210,8 @@ public class DataSource implements Closeable {
* the line iterator.
*
* @return line iterator
- * @throws IOException if an I/O error occurs
*/
- public LineIterator getLineIterator() throws IOException {
+ public LineIterator getLineIterator() {
return getLineIterator(getCharset().name());
}
@@ -202,15 +221,20 @@ public class DataSource implements Closeable {
*
* @param charsetName The name of the requested charset
* @return line iterator
- * @throws IOException if an I/O error occurs
*/
- public LineIterator getLineIterator(String charsetName) throws IOException {
- return closables.add(lineIterator(getUnsafeInputStream(), forName(charsetName)));
+ public LineIterator getLineIterator(String charsetName) {
+ try {
+ return closables.add(lineIterator(getUnsafeInputStream(), forName(charsetName)));
+ } catch (IOException e) {
+ throw new RuntimeException("Failed to create line iterator: " + toString(), e);
+ }
}
- public byte[] getBytes() throws IOException {
+ public byte[] getBytes() {
try (InputStream inputStream = getUnsafeInputStream()) {
return IOUtils.toByteArray(inputStream);
+ } catch (IOException e) {
+ throw new RuntimeException("Failed to get bytes: " + toString(), e);
}
}
@@ -238,8 +262,24 @@ public class DataSource implements Closeable {
"name='" + name + '\'' +
", group='" + group + '\'' +
", uri=" + uri +
- ", contentType='" + contentType + '\'' +
- ", charset=" + charset +
'}';
}
+
+ private Charset getCharsetFromContentType(String contentType, Charset def) {
+ final Matcher matcher = CHARSET_PATTERN.matcher(contentType);
+ if (matcher.find()) {
+ final String name = matcher.group(1).trim().toUpperCase();
+ return Charset.forName(name);
+ }
+ return def;
+ }
+
+ private String contentType() {
+ return isNotEmpty(contentType) ? contentType : dataSource.getContentType();
+ }
+
+ private String stripExtraParameterFronContentType(String contentType) {
+ final int end = contentType.indexOf(";");
+ return end > 0 ? contentType.substring(0, end).trim() : contentType;
+ }
}
diff --git a/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/datasource/DataSourceFactory.java b/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/datasource/DataSourceFactory.java
index 95e7e68..76f5142 100644
--- a/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/datasource/DataSourceFactory.java
+++ b/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/datasource/DataSourceFactory.java
@@ -18,71 +18,94 @@ package org.apache.freemarker.generator.base.datasource;
import org.apache.freemarker.generator.base.FreeMarkerConstants.Location;
import org.apache.freemarker.generator.base.activation.ByteArrayDataSource;
+import org.apache.freemarker.generator.base.activation.CachingUrlDataSource;
import org.apache.freemarker.generator.base.activation.InputStreamDataSource;
import org.apache.freemarker.generator.base.activation.MimetypesFileTypeMapFactory;
import org.apache.freemarker.generator.base.activation.StringDataSource;
import org.apache.freemarker.generator.base.uri.NamedUri;
import org.apache.freemarker.generator.base.uri.NamedUriStringParser;
+import org.apache.freemarker.generator.base.util.PropertiesFactory;
import org.apache.freemarker.generator.base.util.StringUtils;
import org.apache.freemarker.generator.base.util.UriUtils;
+import org.apache.freemarker.generator.base.util.Validate;
import javax.activation.FileDataSource;
import javax.activation.URLDataSource;
import java.io.File;
+import java.io.IOException;
import java.io.InputStream;
+import java.io.StringWriter;
import java.net.MalformedURLException;
import java.net.URI;
import java.net.URL;
import java.nio.charset.Charset;
+import java.util.Properties;
import static java.nio.charset.StandardCharsets.UTF_8;
import static org.apache.freemarker.generator.base.FreeMarkerConstants.DEFAULT_GROUP;
+import static org.apache.freemarker.generator.base.activation.Mimetypes.MIME_TEXT_PLAIN;
+import static org.apache.freemarker.generator.base.util.StringUtils.firstNonEmpty;
/**
* Creates a FreeMarker data source from various sources.
*/
public class DataSourceFactory {
+ private static final String NO_MIME_TYPE = null;
+ private static final Charset NO_CHARSET = null;
+
private DataSourceFactory() {
}
// == NamedUri ==========================================================
public static DataSource fromNamedUri(String str) {
+ Validate.notNull(str, "namedUri is null");
+
return fromNamedUri(NamedUriStringParser.parse(str));
}
public static DataSource fromNamedUri(NamedUri namedUri) {
+ Validate.notNull(namedUri, "namedUri is null");
+
final URI uri = namedUri.getUri();
final String group = namedUri.getGroupOrElse(DEFAULT_GROUP);
- final Charset charset = getCharsetOrElse(namedUri, UTF_8);
+ final Charset charset = getCharsetOrElse(namedUri, NO_CHARSET);
+ final String mimeType = getMimeTypeOrElse(namedUri, NO_MIME_TYPE);
if (UriUtils.isHttpURI(uri)) {
final URL url = toURL(uri);
final String name = namedUri.getNameOrElse(url.getHost());
- return fromUrl(name, group, url, charset);
+ return fromUrl(name, group, url, mimeType, charset);
} else if (UriUtils.isFileUri(uri)) {
final File file = namedUri.getFile();
final String name = namedUri.getNameOrElse(file.getName());
return fromFile(name, group, file, charset);
} else if (UriUtils.isEnvUri(uri)) {
final String key = uri.getPath().substring(1);
- final String name = StringUtils.firstNonEmpty(namedUri.getName(), key, "env");
- final String contentType = getMimeTypeOrElse(namedUri, "text/plain");
- return fromEnvironment(name, group, key, contentType);
+ final String contentType = getMimeTypeOrElse(namedUri, MIME_TEXT_PLAIN);
+ final String name = firstNonEmpty(namedUri.getName(), key, Location.ENVIRONMENT);
+ if (StringUtils.isEmpty(key)) {
+ return fromEnvironment(name, group, contentType);
+ } else {
+ return fromEnvironment(name, group, key, contentType);
+ }
} else {
- throw new IllegalArgumentException("Don't knowm how to handle: " + namedUri);
+ // handle things such as "foo=some.file"
+ final File file = namedUri.getFile();
+ final String name = namedUri.getNameOrElse(file.getName());
+ return fromFile(name, group, file, charset);
}
}
// == URL ===============================================================
public static DataSource fromUrl(String name, String group, URL url, Charset charset) {
- return fromUrl(name, group, url, "application/octet-stream", charset);
+ return fromUrl(name, group, url, NO_MIME_TYPE, charset);
}
public static DataSource fromUrl(String name, String group, URL url, String contentType, Charset charset) {
- final URLDataSource dataSource = new URLDataSource(url);
+ final URLDataSource dataSource = new CachingUrlDataSource(url);
final URI uri = UriUtils.toURI(url);
return create(name, group, uri, dataSource, contentType, charset);
}
@@ -106,6 +129,8 @@ public class DataSourceFactory {
}
public static DataSource fromFile(String name, String group, File file, Charset charset) {
+ Validate.isTrue(file.exists(), "File not found: " + file);
+
final FileDataSource dataSource = new FileDataSource(file);
dataSource.setFileTypeMap(MimetypesFileTypeMapFactory.create());
final String contentType = dataSource.getContentType();
@@ -135,9 +160,23 @@ public class DataSourceFactory {
// == Environment =======================================================
+ public static DataSource fromEnvironment(String name, String group, String contentType) {
+ try {
+ final Properties properties = PropertiesFactory.create(System.getenv());
+ final StringWriter writer = new StringWriter();
+ properties.store(writer, null);
+ final StringDataSource dataSource = new StringDataSource(name, writer.toString(), contentType, UTF_8);
+ final URI uri = UriUtils.toURI(Location.ENVIRONMENT, "");
+ return create(name, group, uri, dataSource, contentType, UTF_8);
+ } catch (IOException e) {
+ throw new RuntimeException(e);
+ }
+ }
+
public static DataSource fromEnvironment(String name, String group, String key, String contentType) {
- final String value = System.getenv(key);
- final StringDataSource dataSource = new StringDataSource(name, value, contentType, UTF_8);
+ Validate.notEmpty(System.getenv(key), "Environment variable not found: " + key);
+
+ final StringDataSource dataSource = new StringDataSource(name, System.getenv(key), contentType, UTF_8);
final URI uri = UriUtils.toURI(Location.ENVIRONMENT, key);
return create(name, group, uri, dataSource, contentType, UTF_8);
}
@@ -168,7 +207,8 @@ public class DataSourceFactory {
}
private static Charset getCharsetOrElse(NamedUri namedUri, Charset def) {
- return Charset.forName(namedUri.getParameter(NamedUri.CHARSET, def.name()));
+ final String charsetName = namedUri.getParameter(NamedUri.CHARSET);
+ return StringUtils.isEmpty(charsetName) ? def : Charset.forName(charsetName);
}
private static URL toURL(URI uri) {
diff --git a/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/util/PropertiesFactory.java b/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/util/PropertiesFactory.java
index 561b185..8805150 100644
--- a/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/util/PropertiesFactory.java
+++ b/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/util/PropertiesFactory.java
@@ -19,6 +19,7 @@ package org.apache.freemarker.generator.base.util;
import java.io.IOException;
import java.io.InputStream;
import java.io.StringReader;
+import java.util.Map;
import java.util.Properties;
public class PropertiesFactory {
@@ -42,4 +43,10 @@ public class PropertiesFactory {
throw new RuntimeException("Failed to parse properties: " + value, e);
}
}
+
+ public static Properties create(Map<?, ?> map) {
+ final Properties properties = new Properties();
+ map.forEach((key, value) -> properties.setProperty(key.toString(), value.toString()));
+ return properties;
+ }
}
diff --git a/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/util/UriUtils.java b/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/util/UriUtils.java
index cbc401b..305822c 100644
--- a/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/util/UriUtils.java
+++ b/freemarker-generator-base/src/main/java/org/apache/freemarker/generator/base/util/UriUtils.java
@@ -20,6 +20,8 @@ import java.net.URI;
import java.net.URISyntaxException;
import java.net.URL;
+import static org.apache.freemarker.generator.base.util.StringUtils.isNotEmpty;
+
public class UriUtils {
public static URI toURI(String str) {
@@ -39,18 +41,27 @@ public class UriUtils {
}
public static boolean isUri(String str) {
- return str.contains("://");
+ return isNotEmpty(str) && str.contains("://");
}
public static boolean isHttpURI(URI uri) {
- return uri.getScheme().equalsIgnoreCase("http") || uri.getScheme().equalsIgnoreCase("https");
+ if (uri == null) {
+ return false;
+ }
+ return "http".equalsIgnoreCase(uri.getScheme()) || "https".equalsIgnoreCase(uri.getScheme());
}
public static boolean isFileUri(URI uri) {
+ if (uri == null) {
+ return false;
+ }
return "file".equalsIgnoreCase(uri.getScheme());
}
public static boolean isEnvUri(URI uri) {
+ if (uri == null) {
+ return false;
+ }
return "env".equalsIgnoreCase(uri.getScheme());
}
}
diff --git a/freemarker-generator-base/src/test/data/properties/test.properties b/freemarker-generator-base/src/test/data/properties/test.properties
index 6431653..47def41 100644
--- a/freemarker-generator-base/src/test/data/properties/test.properties
+++ b/freemarker-generator-base/src/test/data/properties/test.properties
@@ -14,4 +14,5 @@
## See the License for the specific language governing permissions and
## limitations under the License.
## ---------------------------------------------------------------------------
-foo=bar
\ No newline at end of file
+FOO=foo
+BAR=bar
diff --git a/freemarker-generator-base/src/test/java/org/apache/freemarker/generator/datasource/DataSourceFactoryTest.java b/freemarker-generator-base/src/test/java/org/apache/freemarker/generator/datasource/DataSourceFactoryTest.java
index d8b7b4b..28062be 100644
--- a/freemarker-generator-base/src/test/java/org/apache/freemarker/generator/datasource/DataSourceFactoryTest.java
+++ b/freemarker-generator-base/src/test/java/org/apache/freemarker/generator/datasource/DataSourceFactoryTest.java
@@ -26,6 +26,7 @@ import java.io.ByteArrayInputStream;
import java.io.File;
import java.io.IOException;
import java.io.InputStream;
+import java.net.URL;
import java.nio.charset.Charset;
import static java.nio.charset.StandardCharsets.UTF_8;
@@ -43,9 +44,8 @@ public class DataSourceFactoryTest {
private static final File ANY_FILE = new File(ANY_FILE_NAME);
private static final String ANY_NAMED_URL_STRING = "content:www=https://www.google.com?foo=bar#contenttype=application/json";
-
@Test
- public void shouldCreateDataSourceFromFile() throws IOException {
+ public void shouldCreateDataSourceFromFile() {
final DataSource dataSource = DataSourceFactory.fromFile(ANY_FILE, ANY_CHAR_SET);
assertEquals(ANY_FILE_NAME, dataSource.getName());
@@ -56,7 +56,7 @@ public class DataSourceFactoryTest {
}
@Test
- public void shouldCreateDataSourceFromFileUri() throws IOException {
+ public void shouldCreateDataSourceFromFileUri() {
final DataSource dataSource = DataSourceFactory.create(ANY_FILE_URI);
assertEquals(ANY_FILE_NAME, dataSource.getName());
@@ -67,49 +67,98 @@ public class DataSourceFactoryTest {
}
@Test
- public void shouldCreateDataSourceFromString() throws IOException {
+ public void shouldCreateDataSourceFromString() {
final DataSource dataSource = DataSourceFactory.fromString("test.txt", "default", ANY_TEXT, "text/plain");
assertEquals("test.txt", dataSource.getName());
assertEquals("default", dataSource.getGroup());
assertEquals(UTF_8, dataSource.getCharset());
+ assertEquals("text/plain", dataSource.getContentType());
assertTrue(dataSource.getUri().toString().startsWith("string:///"));
assertEquals(ANY_TEXT, dataSource.getText());
assertEquals(1, dataSource.getLines().size());
}
@Test
- public void shouldCreateDataSourceFromBytes() throws IOException {
+ public void shouldCreateDataSourceFromBytes() {
final DataSource dataSource = DataSourceFactory.fromBytes("test.txt", "default", ANY_TEXT.getBytes(UTF_8), "text/plain");
assertEquals("test.txt", dataSource.getName());
assertEquals("default", dataSource.getGroup());
assertEquals(UTF_8, dataSource.getCharset());
+ assertEquals("text/plain", dataSource.getContentType());
assertTrue(dataSource.getUri().toString().startsWith("bytes:///"));
assertEquals(ANY_TEXT, dataSource.getText());
assertEquals(1, dataSource.getLines().size());
}
@Test
- public void shouldCreateDataSourceFromInputStream() throws IOException {
+ public void shouldCreateDataSourceFromInputStream() {
final InputStream is = new ByteArrayInputStream(ANY_TEXT.getBytes(UTF_8));
final DataSource dataSource = DataSourceFactory.fromInputStream("test.txt", "default", is, "text/plain", UTF_8);
assertEquals("test.txt", dataSource.getName());
assertEquals(UTF_8, dataSource.getCharset());
+ assertEquals("text/plain", dataSource.getContentType());
assertTrue(dataSource.getUri().toString().startsWith("inputstream:///"));
assertEquals(ANY_TEXT, dataSource.getText());
}
@Test
public void shouldCreateDataSourceFromURL() throws IOException {
+ final URL url = new URL("https://jsonplaceholder.typicode.com/posts/2");
+ final DataSource dataSource = DataSourceFactory.fromUrl("jsonplaceholder.typicode.com", "default", url, null, null);
+
+ assertEquals("jsonplaceholder.typicode.com", dataSource.getName());
+ assertEquals("application/json", dataSource.getContentType());
+ assertEquals(UTF_8, dataSource.getCharset());
+ }
+
+ @Test
+ public void shouldCreateDataSourceFromNamedURL() {
final NamedUri namedUri = NamedUriStringParser.parse(ANY_NAMED_URL_STRING);
- final DataSource dataSource = DataSourceFactory.fromNamedUri(ANY_NAMED_URL_STRING);
+ final DataSource dataSource = DataSourceFactory.fromNamedUri(namedUri);
assertEquals(namedUri.getName(), dataSource.getName());
assertEquals(namedUri.getGroup(), dataSource.getGroup());
- assertEquals(UTF_8, dataSource.getCharset());
+ assertEquals("ISO-8859-1", dataSource.getCharset().toString());
assertEquals(namedUri.getUri().toString(), dataSource.getUri().toString());
}
+ @Test
+ public void shouldCreateDataSourceFromEnviroment() {
+ final NamedUri namedUri = NamedUriStringParser.parse("env:///");
+ final DataSource dataSource = DataSourceFactory.fromNamedUri(namedUri);
+
+ assertEquals("env", dataSource.getName());
+ assertEquals("default", dataSource.getGroup());
+ assertEquals(UTF_8, dataSource.getCharset());
+ assertEquals("env:///", dataSource.getUri().toString());
+ assertEquals("text/plain", dataSource.getContentType());
+ }
+
+ @Test
+ public void shouldCreateDataSourceFromNamedEnviroment() {
+ final NamedUri namedUri = NamedUriStringParser.parse("config=env:///");
+ final DataSource dataSource = DataSourceFactory.fromNamedUri(namedUri);
+
+ assertEquals("config", dataSource.getName());
+ assertEquals("default", dataSource.getGroup());
+ assertEquals(UTF_8, dataSource.getCharset());
+ assertEquals("env:///", dataSource.getUri().toString());
+ assertEquals("text/plain", dataSource.getContentType());
+ }
+
+ @Test
+ public void shouldCreateDataSourceFromEnviromentVariable() {
+ final NamedUri namedUri = NamedUriStringParser.parse("pwd=env:///PWD");
+ final DataSource dataSource = DataSourceFactory.fromNamedUri(namedUri);
+
+ assertEquals("pwd", dataSource.getName());
+ assertEquals("default", dataSource.getGroup());
+ assertEquals(UTF_8, dataSource.getCharset());
+ assertEquals("env:///PWD", dataSource.getUri().toString());
+ assertEquals("text/plain", dataSource.getContentType());
+ }
+
}
diff --git a/freemarker-generator-base/src/test/java/org/apache/freemarker/generator/file/PropertiesFileSupplierTest.java b/freemarker-generator-base/src/test/java/org/apache/freemarker/generator/file/PropertiesFileSupplierTest.java
index 68926e3..82d0305 100644
--- a/freemarker-generator-base/src/test/java/org/apache/freemarker/generator/file/PropertiesFileSupplierTest.java
+++ b/freemarker-generator-base/src/test/java/org/apache/freemarker/generator/file/PropertiesFileSupplierTest.java
@@ -38,7 +38,8 @@ public class PropertiesFileSupplierTest {
final Properties properties = supplier(ANY_PROPERTIES_FILE).get();
assertNotNull(properties);
- assertEquals("bar", properties.getProperty("foo"));
+ assertEquals("foo", properties.getProperty("FOO"));
+ assertEquals("bar", properties.getProperty("BAR"));
}
@Test
diff --git a/freemarker-generator-cli/pom.xml b/freemarker-generator-cli/pom.xml
index 6fafb57..8b74a0d 100644
--- a/freemarker-generator-cli/pom.xml
+++ b/freemarker-generator-cli/pom.xml
@@ -146,6 +146,8 @@
<exclude>src/main/resources/patterns/*</exclude>
<exclude>site/sample/*/**</exclude>
<exclude>src/test/data/encoding/utf8.txt</exclude>
+ <exclude>src/test/data/json/environments.json</exclude>
+ <exclude>src/test/data/yaml/environments.yaml</exclude>
</excludes>
</configuration>
</plugin>
diff --git a/freemarker-generator-cli/src/main/java/org/apache/freemarker/generator/cli/Main.java b/freemarker-generator-cli/src/main/java/org/apache/freemarker/generator/cli/Main.java
index e890120..e3b17b4 100644
--- a/freemarker-generator-cli/src/main/java/org/apache/freemarker/generator/cli/Main.java
+++ b/freemarker-generator-cli/src/main/java/org/apache/freemarker/generator/cli/Main.java
@@ -71,9 +71,6 @@ public class Main implements Callable<Integer> {
@Option(names = { "-b", "--basedir" }, description = "Optional template base directory")
String baseDir;
- @Option(names = { "-d", "--data-source" }, description = "Data source used for rendering")
- List<String> dataSources;
-
@Option(names = { "-D", "--system-property" }, description = "Set system property")
Properties systemProperties;
@@ -86,8 +83,8 @@ public class Main implements Callable<Integer> {
@Option(names = { "-l", "--locale" }, description = "Locale being used for the output, e.g. 'en_US'")
String locale;
- @Option(names = { "-m", "--mode" }, description = "[template|datasource]", defaultValue = "TEMPLATE")
- GeneratorMode mode;
+ @Option(names = { "-m", "--data-model" }, description = "Data model used for rendering")
+ List<String> dataModels;
@Option(names = { "-o", "--output" }, description = "Output file")
String outputFile;
@@ -95,6 +92,9 @@ public class Main implements Callable<Integer> {
@Option(names = { "-P", "--param" }, description = "Set parameter")
Map<String, String> parameters;
+ @Option(names = { "-s", "--data-source" }, description = "Data source used for rendering")
+ List<String> dataSources;
+
@Option(names = { "--config" }, defaultValue = FREEMARKER_CLI_PROPERTY_FILE, description = "FreeMarker CLI configuration file")
String configFile;
@@ -107,6 +107,9 @@ public class Main implements Callable<Integer> {
@Option(names = { "--output-encoding" }, description = "Encoding of output, e.g. UTF-8", defaultValue = "UTF-8")
String outputEncoding;
+ @Option(names = { "--mode" }, description = "[template|datasource]", defaultValue = "TEMPLATE")
+ GeneratorMode mode;
+
@Option(names = { "--stdin" }, description = "Read data source from stdin")
boolean readFromStdin;
@@ -219,6 +222,7 @@ public class Main implements Callable<Integer> {
.setOutputFile(outputFile)
.setParameters(parameters != null ? parameters : new HashMap<>())
.setDataSources(getCombindedDataSources())
+ .setDataModels(dataModels)
.setSystemProperties(systemProperties != null ? systemProperties : new Properties())
.setTemplateDirectories(templateDirectories)
.setTemplateName(templateSourceOptions.template)
diff --git a/freemarker-generator-cli/src/main/java/org/apache/freemarker/generator/cli/config/DataModelSupplier.java b/freemarker-generator-cli/src/main/java/org/apache/freemarker/generator/cli/config/DataModelSupplier.java
new file mode 100644
index 0000000..62ae775
--- /dev/null
+++ b/freemarker-generator-cli/src/main/java/org/apache/freemarker/generator/cli/config/DataModelSupplier.java
@@ -0,0 +1,128 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.freemarker.generator.cli.config;
+
+import org.apache.freemarker.generator.base.datasource.DataSource;
+import org.apache.freemarker.generator.base.datasource.DataSourceFactory;
+import org.apache.freemarker.generator.base.uri.NamedUri;
+import org.apache.freemarker.generator.base.uri.NamedUriStringParser;
+import org.apache.freemarker.generator.base.util.PropertiesFactory;
+import org.apache.freemarker.generator.base.util.StringUtils;
+import org.apache.freemarker.generator.base.util.UriUtils;
+import org.apache.freemarker.generator.tools.gson.GsonTool;
+import org.apache.freemarker.generator.tools.snakeyaml.SnakeYamlTool;
+
+import java.net.URI;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.HashMap;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.Properties;
+import java.util.function.Supplier;
+
+import static java.util.Objects.requireNonNull;
+import static java.util.stream.Collectors.toMap;
+import static org.apache.freemarker.generator.base.activation.Mimetypes.MIME_APPLICATION_JSON;
+import static org.apache.freemarker.generator.base.activation.Mimetypes.MIME_TEXT_PLAIN;
+import static org.apache.freemarker.generator.base.activation.Mimetypes.MIME_TEXT_YAML;
+
+/**
+ * Create a map representing a data model based on a list of sources consisting of
+ * URIs, named URIs or files.
+ */
+public class DataModelSupplier implements Supplier<Map<String, Object>> {
+
+ private final Collection<String> sources;
+
+ /**
+ * Constructor.
+ *
+ * @param sources List of sources
+ */
+ public DataModelSupplier(Collection<String> sources) {
+ this.sources = new ArrayList<>(requireNonNull(sources));
+ }
+
+ @Override
+ public Map<String, Object> get() {
+ return sources.stream()
+ .filter(StringUtils::isNotEmpty)
+ .map(this::toDataModel)
+ .flatMap(map -> map.entrySet().stream())
+ .collect(toMap(Entry::getKey, Entry::getValue));
+ }
+
+ protected Map<String, Object> toDataModel(String source) {
+ final NamedUri namedUri = NamedUriStringParser.parse(source);
+ final DataSource dataSource = DataSourceFactory.fromNamedUri(namedUri);
+ final boolean isExplodedDataModel = !namedUri.hasName();
+ final String contentType = dataSource.getContentType();
+
+ if (contentType.startsWith(MIME_APPLICATION_JSON)) {
+ return fromJson(dataSource, isExplodedDataModel);
+ } else if (contentType.startsWith(MIME_TEXT_PLAIN)) {
+ return fromProperties(dataSource, isExplodedDataModel);
+ } else if (contentType.startsWith(MIME_TEXT_YAML)) {
+ return fromYaml(dataSource, isExplodedDataModel);
+ } else {
+ throw new IllegalArgumentException("Don't know how to handle content type: " + contentType);
+ }
+ }
+
+ private Map<String, Object> fromJson(DataSource dataSource, boolean isExplodedDataModel) {
+ final GsonTool gsonTool = new GsonTool();
+ final Map<String, Object> map = gsonTool.parse(dataSource);
+ return fromMap(dataSource.getName(), map, isExplodedDataModel);
+ }
+
+ private Map<String, Object> fromYaml(DataSource dataSource, boolean isExplodedDataModel) {
+ final SnakeYamlTool snakeYamlTool = new SnakeYamlTool();
+ final Map<String, Object> map = snakeYamlTool.parse(dataSource);
+ return fromMap(dataSource.getName(), map, isExplodedDataModel);
+ }
+
+ private Map<String, Object> fromProperties(DataSource dataSource, boolean isExplodedDataModel) {
+ final Map<String, Object> result = new HashMap<>();
+ final URI uri = dataSource.getUri();
+
+ if (UriUtils.isEnvUri(uri) && !"env:///".equals(uri.toString())) {
+ result.put(dataSource.getName(), dataSource.getText());
+ } else {
+ final Properties properties = PropertiesFactory.create(dataSource.getText());
+ if (isExplodedDataModel) {
+ properties.forEach((key, value) -> result.put(key.toString(), value));
+ } else {
+ result.put(dataSource.getName(), properties);
+ }
+ }
+
+ return result;
+ }
+
+ private Map<String, Object> fromMap(String name, Map<String, Object> map, boolean isExplodedDataModel) {
+ final Map<String, Object> result = new HashMap<>();
+
+ if (isExplodedDataModel) {
+ map.forEach(result::put);
+ } else {
+ result.put(name, map);
+ }
+
+ return result;
+ }
+}
diff --git a/freemarker-generator-cli/src/main/java/org/apache/freemarker/generator/cli/config/Settings.java b/freemarker-generator-cli/src/main/java/org/apache/freemarker/generator/cli/config/Settings.java
index 30f6d9c..51bbae8 100644
--- a/freemarker-generator-cli/src/main/java/org/apache/freemarker/generator/cli/config/Settings.java
+++ b/freemarker-generator-cli/src/main/java/org/apache/freemarker/generator/cli/config/Settings.java
@@ -85,9 +85,12 @@ public class Settings {
/** Expose environment variables globally in the data model? */
private final boolean isEnvironmentExposed;
- /** User-supplied list of source files or directories */
+ /** User-supplied list of data sources or directories */
private final List<String> dataSources;
+ /** User-supplied list of data sources directly exposed in the data model */
+ private List<String> dataModels;
+
/** User-supplied parameters */
private final Map<String, String> parameters;
@@ -113,6 +116,7 @@ public class Settings {
boolean isReadFromStdin,
boolean isEnvironmentExposed,
List<String> dataSources,
+ List<String> dataModels,
Map<String, String> parameters,
Properties sytemProperties,
Writer writer) {
@@ -134,6 +138,7 @@ public class Settings {
this.isReadFromStdin = isReadFromStdin;
this.isEnvironmentExposed = isEnvironmentExposed;
this.dataSources = requireNonNull(dataSources);
+ this.dataModels = requireNonNull(dataModels);
this.parameters = requireNonNull(parameters);
this.sytemProperties = requireNonNull(sytemProperties);
this.configuration = requireNonNull(configuration);
@@ -208,6 +213,10 @@ public class Settings {
return dataSources;
}
+ public List<String> getDataModels() {
+ return dataModels;
+ }
+
public Map<String, String> getParameters() {
return parameters;
}
@@ -289,6 +298,7 @@ public class Settings {
private boolean isReadFromStdin;
private boolean isEnvironmentExposed;
private List<String> dataSources;
+ private List<String> dataModels;
private Map<String, String> parameters;
private Properties systemProperties;
private Properties configuration;
@@ -303,6 +313,7 @@ public class Settings {
this.setInputEncoding(DEFAULT_CHARSET.name());
this.setOutputEncoding(DEFAULT_CHARSET.name());
this.dataSources = emptyList();
+ this.dataModels = emptyList();
this.templateDirectories = emptyList();
}
@@ -381,7 +392,16 @@ public class Settings {
}
public SettingsBuilder setDataSources(List<String> dataSources) {
- this.dataSources = dataSources;
+ if (dataSources != null) {
+ this.dataSources = dataSources;
+ }
+ return this;
+ }
+
+ public SettingsBuilder setDataModels(List<String> dataModels) {
+ if (dataModels != null) {
+ this.dataModels = dataModels;
+ }
return this;
}
@@ -433,6 +453,7 @@ public class Settings {
isReadFromStdin,
isEnvironmentExposed,
dataSources,
+ dataModels,
parameters,
systemProperties,
writer
diff --git a/freemarker-generator-cli/src/main/java/org/apache/freemarker/generator/cli/config/Suppliers.java b/freemarker-generator-cli/src/main/java/org/apache/freemarker/generator/cli/config/Suppliers.java
index 0766ad3..79f6e0a 100644
--- a/freemarker-generator-cli/src/main/java/org/apache/freemarker/generator/cli/config/Suppliers.java
+++ b/freemarker-generator-cli/src/main/java/org/apache/freemarker/generator/cli/config/Suppliers.java
@@ -56,6 +56,10 @@ public class Suppliers {
settings.getInputEncoding());
}
+ public static DataModelSupplier dataModelSupplier(Settings settings) {
+ return new DataModelSupplier(settings.getDataModels());
+ }
+
public static PropertiesSupplier propertiesSupplier(String fileName) {
return new PropertiesSupplier(
new PropertiesFileSystemSupplier(fileName),
diff --git a/freemarker-generator-cli/src/main/java/org/apache/freemarker/generator/cli/task/FreeMarkerTask.java b/freemarker-generator-cli/src/main/java/org/apache/freemarker/generator/cli/task/FreeMarkerTask.java
index 0fa4482..a89bc33 100644
--- a/freemarker-generator-cli/src/main/java/org/apache/freemarker/generator/cli/task/FreeMarkerTask.java
+++ b/freemarker-generator-cli/src/main/java/org/apache/freemarker/generator/cli/task/FreeMarkerTask.java
@@ -43,6 +43,7 @@ import static org.apache.freemarker.generator.base.FreeMarkerConstants.DEFAULT_G
import static org.apache.freemarker.generator.base.FreeMarkerConstants.Location.STDIN;
import static org.apache.freemarker.generator.base.FreeMarkerConstants.Model.DATASOURCES;
import static org.apache.freemarker.generator.cli.config.Suppliers.configurationSupplier;
+import static org.apache.freemarker.generator.cli.config.Suppliers.dataModelSupplier;
import static org.apache.freemarker.generator.cli.config.Suppliers.dataSourcesSupplier;
import static org.apache.freemarker.generator.cli.config.Suppliers.toolsSupplier;
@@ -56,19 +57,22 @@ public class FreeMarkerTask implements Callable<Integer> {
private final Settings settings;
private final Supplier<Map<String, Object>> toolsSupplier;
private final Supplier<List<DataSource>> dataSourcesSupplier;
+ private final Supplier<Map<String, Object>> dataModelsSupplier;
private final Supplier<Configuration> configurationSupplier;
public FreeMarkerTask(Settings settings) {
- this(settings, toolsSupplier(settings), dataSourcesSupplier(settings), configurationSupplier(settings));
+ this(settings, toolsSupplier(settings), dataSourcesSupplier(settings), dataModelSupplier(settings), configurationSupplier(settings));
}
public FreeMarkerTask(Settings settings,
Supplier<Map<String, Object>> toolsSupplier,
Supplier<List<DataSource>> dataSourcesSupplier,
+ Supplier<Map<String, Object>> dataModelsSupplier,
Supplier<Configuration> configurationSupplier) {
this.settings = requireNonNull(settings);
this.toolsSupplier = requireNonNull(toolsSupplier);
this.dataSourcesSupplier = requireNonNull(dataSourcesSupplier);
+ this.dataModelsSupplier = requireNonNull(dataModelsSupplier);
this.configurationSupplier = requireNonNull(configurationSupplier);
}
@@ -76,7 +80,7 @@ public class FreeMarkerTask implements Callable<Integer> {
public Integer call() {
final Template template = template(settings, configurationSupplier);
try (Writer writer = settings.getWriter(); DataSources dataSources = dataSources(settings, dataSourcesSupplier)) {
- final Map<String, Object> dataModel = dataModel(settings, dataSources, toolsSupplier);
+ final Map<String, Object> dataModel = dataModel(settings, dataSources, dataModelsSupplier, toolsSupplier);
template.process(dataModel, writer);
return SUCCESS;
} catch (RuntimeException e) {
@@ -126,20 +130,26 @@ public class FreeMarkerTask implements Callable<Integer> {
}
}
- private static Map<String, Object> dataModel(Settings settings, DataSources dataSources, Supplier<Map<String, Object>> tools) {
- final Map<String, Object> dataModel = new HashMap<>();
+ private static Map<String, Object> dataModel(
+ Settings settings,
+ DataSources dataSources,
+ Supplier<Map<String, Object>> dataModelsSupplier,
+ Supplier<Map<String, Object>> tools) {
+ final Map<String, Object> result = new HashMap<>();
- dataModel.put(DATASOURCES, dataSources);
+ result.putAll(dataModelsSupplier.get());
+ result.put(DATASOURCES, dataSources);
+ // TODO rework based on FREEMARKER-140
if (settings.isEnvironmentExposed()) {
// add all system & user-supplied properties as top-level entries
- dataModel.putAll(System.getenv());
- dataModel.putAll(settings.getParameters());
+ result.putAll(System.getenv());
+ result.putAll(settings.getParameters());
}
- dataModel.putAll(tools.get());
+ result.putAll(tools.get());
- return dataModel;
+ return result;
}
private static boolean isAbsoluteTemplateFile(Settings settings) {
@@ -165,5 +175,4 @@ public class FreeMarkerTask implements Callable<Integer> {
throw new RuntimeException("Failed to load interactive template", e);
}
}
-
}
diff --git a/freemarker-generator-cli/src/site/markdown/cli/concepts/data-models.md b/freemarker-generator-cli/src/site/markdown/cli/concepts/data-models.md
new file mode 100644
index 0000000..c05323c
--- /dev/null
+++ b/freemarker-generator-cli/src/site/markdown/cli/concepts/data-models.md
@@ -0,0 +1,57 @@
+## DataModels
+
+A `DataModel` is an eagerly loaded `DataSource` available in Apache FreeMarker's model (context) when rendering a template.
+
+* The content of the `DataSource` is parsed and a `Map` generated
+* The `Map` is either stored as variable in the model or all entries are copied into the FreeMarker model
+* The parsing is supported for `JSON`, `YAML`, `Properties` and enviroment variables
+
+Expose the fields of the JSON data source in FreeMarker's model
+
+```
+bin/freemarker-cli --data-model https://xkcd.com/info.0.json -i '<a href="${img}">${title}</a>'
+<a href="https://imgs.xkcd.com/comics/scenario_4.png">Scenario 4</a>
+```
+
+Exposed the JSON data source as variable `post` in FreeMarker's model
+
+```
+bin/freemarker-cli --data-model post=https://jsonplaceholder.typicode.com/posts/2 -i 'post title is: ${post.title}'
+post title is: qui est esse
+```
+
+Expose all environment variables as `env` in theFreeMarker model
+
+```
+bin/freemarker-cli --data-model env=env:/// -i '<#list env as name,value>${name}=${value}${"\n"}</#list>'
+HOME=/Users/sgoeschl
+USER=sgoeschl
+```
+
+Expose a single envionment variable in theFreeMarker model
+
+```
+bin/freemarker-cli --data-model NAME=env:///USER -i 'Hello ${NAME}'
+Hello sgoeschl
+```
+
+Alternatively use the short command line options, e.g.
+
+```
+bin/freemarker-cli -m NAME=env:///USER -i 'Hello ${NAME}!'
+Hello sgoeschl!
+```
+
+The following snippet shows a more advanced example
+
+* The environment variable `DB_CONFIG` holds JSON data
+* Use the `config=env:///DB_CONFIG#mimetype=application/json` to parse JSON payload from `DB_CONFIG` into the data model `config`
+
+```
+> export DB_CONFIG='{"db_default_user":"scott","db_default_password":"tiger"}'
+> echo $DB_CONFIG
+{"db_default_user":"scott","db_default_password":"tiger"}
+> bin/freemarker-cli -m config=env:///DB_CONFIG#mimetype=application/json -i '<#list config as name,value>${name}=${value}${"\n"}</#list>'
+db_default_user=scott
+db_default_password=tiger
+```
\ No newline at end of file
diff --git a/freemarker-generator-cli/src/site/markdown/cli/concepts/named-uris.md b/freemarker-generator-cli/src/site/markdown/cli/concepts/named-uris.md
new file mode 100644
index 0000000..2a2469e
--- /dev/null
+++ b/freemarker-generator-cli/src/site/markdown/cli/concepts/named-uris.md
@@ -0,0 +1,59 @@
+# Named URIs
+
+Named URIs allow to identify `DataSources` and pass additional information
+
+A Named URI consists of
+
+* an optional name
+* an URI or simple file name
+
+As a refresher, a URI is made up of the following components (inspired by https://docs.gomplate.ca/datasources/)
+
+```
+ foo://userinfo@example.com:8042/over/there?name=ferret#nose
+ \_/ \_______________________/\_________/ \_________/ \__/
+ | | | | |
+scheme authority path query fragment
+```
+
+For our purposes, the scheme and the path components are especially important, though the other components are used by certain datasources for particular purposes.
+
+| Component | Purpose |
+|-----------|---------|
+| scheme | All datasources require a scheme (except for file when using relative paths) |
+| authority | Used only by remote datasources, and can be omitted in some of those cases. Consists of userinfo (user:pass), host, and port. |
+| path | Can be omitted, but usually used as the basis of the locator for the datasource. |
+| query | Used mainly for HTTP and HTTPS URLs |
+| fragment | Used rarely for providing additional attributes, e.g. `mimetype` of `charset` |
+
+The following Named URI loads a "user.csv" and the data source is available as `my_users`
+
+```
+bin/freemarker-cli -t templates/info.ftl my_users=site/sample/csv/user.csv
+[#1], name=my_users, group=default, contentType=text/csv, charset=UTF-8, length=376 Bytes
+URI : file:site/sample/csv/user.csv
+```
+
+A Named URI allows to pass additional information as part of the fragment, e.g. the charset of the text file
+
+```
+bin/freemarker-cli -t templates/info.ftl my_users=site/sample/csv/user.csv#charset=UTF-16
+[#1], name=my_users, group=default, contentType=text/csv, charset=UTF-16, length=376 Bytes
+URI : file:site/sample/csv/user.csv
+```
+
+In addition to the simplified file syntax full URIs can be used
+
+```
+bin/freemarker-cli -t templates/info.ftl http://google.com?foo=bar
+[#1], name=google.com, group=default, contentType=text/html, charset=ISO-8859-1, length=-1 Bytes
+URI : http://google.com?foo=bar
+```
+
+and also combined with a name
+
+```
+bin/freemarker-cli -t templates/info.ftl page=http://google.com?foo=bar
+[#1], name=page, group=default, contentType=text/html, charset=ISO-8859-1, length=-1 Bytes
+URI : http://google.com?foo=bar
+```
diff --git a/freemarker-generator-cli/src/site/markdown/index.md b/freemarker-generator-cli/src/site/markdown/index.md
index 2fd9f95..c5807bc 100644
--- a/freemarker-generator-cli/src/site/markdown/index.md
+++ b/freemarker-generator-cli/src/site/markdown/index.md
@@ -1 +1,6 @@
-TBD
\ No newline at end of file
+## FreeMarker Generator CLI
+
+### Concepts
+
+* [Named URIs](cli/concepts/named-uris.html)
+* [Data Models](cli/concepts/data-models.html)
\ No newline at end of file
diff --git a/freemarker-generator-cli/src/test/data/json/environments.json b/freemarker-generator-cli/src/test/data/json/environments.json
new file mode 100644
index 0000000..eb11bc4
--- /dev/null
+++ b/freemarker-generator-cli/src/test/data/json/environments.json
@@ -0,0 +1,4 @@
+{
+ "db_default_user": "scott",
+ "db_default_password": "tiger"
+}
\ No newline at end of file
diff --git a/freemarker-generator-base/src/test/data/properties/test.properties b/freemarker-generator-cli/src/test/data/properties/test.properties
similarity index 98%
copy from freemarker-generator-base/src/test/data/properties/test.properties
copy to freemarker-generator-cli/src/test/data/properties/test.properties
index 6431653..47def41 100644
--- a/freemarker-generator-base/src/test/data/properties/test.properties
+++ b/freemarker-generator-cli/src/test/data/properties/test.properties
@@ -14,4 +14,5 @@
## See the License for the specific language governing permissions and
## limitations under the License.
## ---------------------------------------------------------------------------
-foo=bar
\ No newline at end of file
+FOO=foo
+BAR=bar
diff --git a/freemarker-generator-cli/src/test/data/yaml/environments.yaml b/freemarker-generator-cli/src/test/data/yaml/environments.yaml
new file mode 100644
index 0000000..a9c9d0a
--- /dev/null
+++ b/freemarker-generator-cli/src/test/data/yaml/environments.yaml
@@ -0,0 +1,3 @@
+---
+db_default_user: scott
+db_default_password: tiger
diff --git a/freemarker-generator-cli/src/test/java/org/apache/freemarker/generator/cli/ManualTest.java b/freemarker-generator-cli/src/test/java/org/apache/freemarker/generator/cli/ManualTest.java
index 0d8ca31..0dac268 100644
--- a/freemarker-generator-cli/src/test/java/org/apache/freemarker/generator/cli/ManualTest.java
+++ b/freemarker-generator-cli/src/test/java/org/apache/freemarker/generator/cli/ManualTest.java
@@ -40,8 +40,16 @@ public class ManualTest {
// private static final String CMD = "-b ./src/test -t templates/info.ftl -d :user=site/sample/properties -d contract=site/sample/csv/contract.csv";
// private static final String CMD = "-b ./src/test -t site/sample/ftl/nginx/nginx.conf.ftl -d env=site/sample/ftl/nginx/nginx.env";
// private static final String CMD = "-b ./src/test -t templates/info.ftl -d env=site/sample/ftl/nginx/nginx.env";
- private static final String CMD = "-b ./src/test -t templates/json/yaml/transform.ftl site/sample/json/swagger-spec.json";
+ // private static final String CMD = "-b ./src/test -t templates/json/yaml/transform.ftl site/sample/json/swagger-spec.json";
// private static final String CMD = "-b ./src/test -t templates/yaml/json/transform.ftl site/sample/yaml/swagger-spec.yaml";
+ // private static final String CMD = "-b ./src/test -t templates/demo.ftl -m env=env:///";
+ // private static final String CMD = "-b ./src/test -t templates/demo.ftl -m api=https://httpbin.org/get";
+ // private static final String CMD = "-b ./src/test -t templates/demo.ftl -m env:///HOME";
+ // private static final String CMD = "-b ./src/test -t templates/demo.ftl -m env=./site/sample/properties/user_0001/user.properties";
+ // private static final String CMD = "-b ./src/test -t templates/demo.ftl -m ./site/sample/properties/user_0001/user.properties";
+ // private static final String CMD = "-b ./src/test --data-model post=https://jsonplaceholder.typicode.com/posts/2 -t templates/info.ftl";
+ private static final String CMD = "-b ./src/test -t templates/info.ftl google=https://www.google.com";
+
public static void main(String[] args) {
Main.execute(toArgs(CMD));
diff --git a/freemarker-generator-cli/src/test/java/org/apache/freemarker/generator/cli/PicocliTest.java b/freemarker-generator-cli/src/test/java/org/apache/freemarker/generator/cli/PicocliTest.java
index fdd7fc2..d5e15d6 100644
--- a/freemarker-generator-cli/src/test/java/org/apache/freemarker/generator/cli/PicocliTest.java
+++ b/freemarker-generator-cli/src/test/java/org/apache/freemarker/generator/cli/PicocliTest.java
@@ -52,20 +52,35 @@ public class PicocliTest {
@Test
public void testSingleNamedDataSource() {
assertEquals(ANY_FILE, parse("-t", TEMPLATE, ANY_FILE).sources.get(0));
- assertEquals(ANY_FILE, parse("-t", TEMPLATE, "-d", ANY_FILE).dataSources.get(0));
+ assertEquals(ANY_FILE, parse("-t", TEMPLATE, "-s", ANY_FILE).dataSources.get(0));
assertEquals(ANY_FILE, parse("-t", TEMPLATE, "--data-source", ANY_FILE).dataSources.get(0));
assertEquals(ANY_FILE_URI, parse("-t", TEMPLATE, "--data-source", ANY_FILE_URI).dataSources.get(0));
}
@Test
public void testMultipleNamedDataSource() {
- final Main main = parse("-t", TEMPLATE, "-d", ANY_FILE, "--data-source", OTHER_FILE_URI);
+ final Main main = parse("-t", TEMPLATE, "-s", ANY_FILE, "--data-source", OTHER_FILE_URI);
assertEquals(ANY_FILE, main.dataSources.get(0));
assertEquals(OTHER_FILE_URI, main.dataSources.get(1));
assertNull(main.sources);
}
+ @Test
+ public void testSingleDataModel() {
+ assertEquals(ANY_FILE, parse("-t", TEMPLATE, "-m", ANY_FILE).dataModels.get(0));
+ assertEquals(ANY_FILE, parse("-t", TEMPLATE, "--data-model", ANY_FILE).dataModels.get(0));
+ }
+
+ @Test
+ public void testMultipleDataModels() {
+ final Main main = parse("-t", TEMPLATE, "-m", ANY_FILE, "--data-model", OTHER_FILE_URI);
+
+ assertEquals(ANY_FILE, main.dataModels.get(0));
+ assertEquals(OTHER_FILE_URI, main.dataModels.get(1));
+ assertNull(main.sources);
+ }
+
private static Main parse(String... args) {
final Main main = new Main();
new CommandLine(main).parseArgs(args);
diff --git a/freemarker-generator-cli/src/test/java/org/apache/freemarker/generator/cli/config/DataModelSupplierTest.java b/freemarker-generator-cli/src/test/java/org/apache/freemarker/generator/cli/config/DataModelSupplierTest.java
new file mode 100644
index 0000000..0dc0d78
--- /dev/null
+++ b/freemarker-generator-cli/src/test/java/org/apache/freemarker/generator/cli/config/DataModelSupplierTest.java
@@ -0,0 +1,188 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.freemarker.generator.cli.config;
+
+import org.junit.Test;
+
+import java.util.Map;
+
+import static java.util.Collections.singletonList;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+public class DataModelSupplierTest {
+
+ private static final String PWD_VALUE = System.getenv("PWD");
+ private static final int NR_OF_ALL_ENV_VARIABLES = System.getenv().size();
+
+ // === Environment Variables ===
+
+ @Test
+ public void shouldResolveAllEnvironmentVariablesToTopLevelDataModel() {
+ final DataModelSupplier supplier = supplier("env:///");
+
+ final Map<String, Object> model = supplier.get();
+
+ assertEquals(NR_OF_ALL_ENV_VARIABLES, model.size());
+ assertEquals(PWD_VALUE, model.get("PWD"));
+ }
+
+ @Test
+ public void shouldResolveAllEnvironmentVariablesToDataModelVariable() {
+ final DataModelSupplier supplier = supplier("myenv=env:///");
+
+ final Map<String, Object> model = supplier.get();
+
+ assertEquals(1, model.size());
+ assertEquals(NR_OF_ALL_ENV_VARIABLES, toMap(model, "myenv").size());
+ assertEquals(PWD_VALUE, toMap(model, "myenv").get("PWD"));
+ }
+
+ @Test
+ public void shouldResolveSingleEnvironmentVariablesToTopLevelDataModel() {
+ final DataModelSupplier supplier = supplier("env:///PWD");
+
+ final Map<String, Object> model = supplier.get();
+
+ assertEquals(1, model.size());
+ assertEquals(PWD_VALUE, model.get("PWD"));
+ }
+
+ @Test
+ public void shouldResolveSingleEnvironmentVariableToDataModelVariable() {
+ final DataModelSupplier supplier = supplier("mypwd=env:///PWD");
+
+ final Map<String, Object> model = supplier.get();
+
+ assertEquals(1, model.size());
+ assertEquals(PWD_VALUE, model.get("mypwd"));
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void shouldThrowIllegalArgumentExceptionForNonExistingEnvironmentVariable() {
+ supplier("env:///KEY_DOES_NOT_EXIST").get();
+ }
+
+ // === Properties ===
+
+ @Test
+ public void shouldResolvePropertiesFileToTopLevelDataModel() {
+ final DataModelSupplier supplier = supplier("./src/test/data/properties/test.properties");
+
+ final Map<String, Object> model = supplier.get();
+
+ assertEquals(2, model.size());
+ assertEquals("foo", model.get("FOO"));
+ assertEquals("bar", model.get("BAR"));
+ }
+
+ @Test
+ public void shouldResolvePropertiesFileToDataModelVariable() {
+ final DataModelSupplier supplier = supplier("props=./src/test/data/properties/test.properties");
+
+ final Map<String, Object> model = supplier.get();
+
+ assertEquals(1, model.size());
+ assertEquals("foo", toMap(model, "props").get("FOO"));
+ assertEquals("bar", toMap(model, "props").get("BAR"));
+ }
+
+ @Test
+ public void shouldResolvePropertiesUriToDataModelVariable() {
+ final DataModelSupplier supplier = supplier("props=file://./src/test/data/properties/test.properties");
+
+ final Map<String, Object> model = supplier.get();
+
+ assertEquals(1, model.size());
+ assertEquals("foo", toMap(model, "props").get("FOO"));
+ assertEquals("bar", toMap(model, "props").get("BAR"));
+ }
+
+ // === JSON ===
+
+ @Test
+ public void shouldResolveJsonFileToTopLevelDataModel() {
+ final DataModelSupplier supplier = supplier("./src/test/data/json/environments.json");
+
+ final Map<String, Object> model = supplier.get();
+
+ assertEquals(2, model.size());
+ assertEquals("scott", model.get("db_default_user"));
+ assertEquals("tiger", model.get("db_default_password"));
+ }
+
+ @Test
+ public void shouldResolveYamlFileToTopLevelDataModel() {
+ final DataModelSupplier supplier = supplier("./src/test/data/yaml/environments.yaml");
+
+ final Map<String, Object> model = supplier.get();
+
+ assertEquals(2, model.size());
+ assertEquals("scott", model.get("db_default_user"));
+ assertEquals("tiger", model.get("db_default_password"));
+ }
+
+ // == URL ===
+
+ @Test
+ public void shouldResolveUrlToTopLevelDataModel() {
+ final DataModelSupplier supplier = supplier("post=https://jsonplaceholder.typicode.com/posts/2");
+
+ final Map<String, Object> model = supplier.get();
+
+ assertEquals(1, model.size());
+ assertNotNull(model.get("post"));
+ }
+
+ @Test
+ public void shouldResolveUrlToDataModelVariable() {
+ final DataModelSupplier supplier = supplier("https://jsonplaceholder.typicode.com/posts/2");
+
+ final Map<String, Object> model = supplier.get();
+
+ assertTrue(model.size() == 4);
+ }
+
+ @Test(expected = RuntimeException.class)
+ public void shouldResolveUrlToDataModelVariables() {
+ supplier("https://jsonplaceholder.typicode.com/posts/does-not-exist").get();
+ }
+
+ // === Misc ===
+
+ @Test
+ public void shouldReturnEmptyDataModelOnMissingSource() {
+ assertEquals(0, supplier(null).get().size());
+ assertEquals(0, supplier("").get().size());
+ assertEquals(0, supplier(" ").get().size());
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void shouldThrowIllegalArgumentExceptionForNonExistingFile() {
+ supplier("file:///./FILE_DOES_NOT_EXIST.json").get();
+ }
+
+ private static DataModelSupplier supplier(String source) {
+ return new DataModelSupplier(singletonList(source));
+ }
+
+ @SuppressWarnings("unchecked")
+ private static Map<String, Object> toMap(Map<String, Object> model, String key) {
+ return (Map<String, Object>) model.get(key);
+ }
+}
diff --git a/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/gson/GsonTool.java b/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/gson/GsonTool.java
index 9805afd..6a864eb 100644
--- a/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/gson/GsonTool.java
+++ b/freemarker-generator-tools/src/main/java/org/apache/freemarker/generator/tools/gson/GsonTool.java
@@ -32,9 +32,11 @@ public class GsonTool {
private Gson gson;
private Type type;
- public Map<String, Object> parse(DataSource dataSource) throws IOException {
+ public Map<String, Object> parse(DataSource dataSource) {
try (JsonReader reader = new JsonReader(new InputStreamReader(dataSource.getUnsafeInputStream()))) {
return gson().fromJson(reader, type());
+ } catch (IOException e) {
+ throw new RuntimeException("Failed to parse data source:" + dataSource, e);
}
}
diff --git a/pom.xml b/pom.xml
index a02de99..58b46f7 100644
--- a/pom.xml
+++ b/pom.xml
@@ -81,7 +81,7 @@
</modules>
<properties>
- <freemarker.version>2.3.29</freemarker.version>
+ <freemarker.version>2.3.30</freemarker.version>
<maven.compiler.source>1.8</maven.compiler.source>
<maven.compiler.target>1.8</maven.compiler.target>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>