You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@freemarker.apache.org by dd...@apache.org on 2017/07/09 23:07:35 UTC
incubator-freemarker git commit: Continued working on FM2 to FM3
converter...
Repository: incubator-freemarker
Updated Branches:
refs/heads/3 80e4c3424 -> 4763581a7
Continued working on FM2 to FM3 converter...
Project: http://git-wip-us.apache.org/repos/asf/incubator-freemarker/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-freemarker/commit/4763581a
Tree: http://git-wip-us.apache.org/repos/asf/incubator-freemarker/tree/4763581a
Diff: http://git-wip-us.apache.org/repos/asf/incubator-freemarker/diff/4763581a
Branch: refs/heads/3
Commit: 4763581a7c9b42058ca604efef6117639b8e32bd
Parents: 80e4c34
Author: ddekany <dd...@apache.org>
Authored: Mon Jul 10 01:07:20 2017 +0200
Committer: ddekany <dd...@apache.org>
Committed: Mon Jul 10 01:07:20 2017 +0200
----------------------------------------------------------------------
.../apache/freemarker/converter/Converter.java | 116 +++++++++++++++++++
.../freemarker/converter/FM2ToFM3Converter.java | 38 ++++--
.../converter/FM2ToFM3ConverterTest.java | 28 ++++-
.../converter/GenericConverterTest.java | 39 +++++++
4 files changed, 209 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-freemarker/blob/4763581a/freemarker-converter/src/main/java/org/apache/freemarker/converter/Converter.java
----------------------------------------------------------------------
diff --git a/freemarker-converter/src/main/java/org/apache/freemarker/converter/Converter.java b/freemarker-converter/src/main/java/org/apache/freemarker/converter/Converter.java
index 2b3d8eb..2c4fa0b 100644
--- a/freemarker-converter/src/main/java/org/apache/freemarker/converter/Converter.java
+++ b/freemarker-converter/src/main/java/org/apache/freemarker/converter/Converter.java
@@ -27,7 +27,9 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.io.Writer;
+import java.nio.file.Path;
import java.util.HashSet;
+import java.util.Iterator;
import java.util.Set;
import java.util.regex.Pattern;
@@ -36,16 +38,26 @@ import org.apache.freemarker.core.util._StringUtil;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
+/**
+ * Abstract superclass for converters. A converter converts the files in the source directory (and in its
+ * subdirectories, recursively), or of a single source file, and writes the converted files into the the destination
+ * directory.
+ */
public abstract class Converter {
+ public static final String PROPERTY_NAME_INCLUDE = "include";
+ public static final String PROPERTY_NAME_EXCLUDE = "exclude";
public static final String PROPERTY_NAME_SOURCE = "source";
public static final String PROPERTY_NAME_DESTINATION_DIRECTORY = "destinationDirectory";
+
public static final String CONVERSION_MARKERS_FILE_NAME = "__conversion-markers.txt";
public static ThreadLocal<FileConversionContext> FILE_CONVERSION_CONTEXT_TLS = new ThreadLocal<>();
private static final Logger LOG = LoggerFactory.getLogger(Converter.class);
+ private Pattern include;
+ private Pattern exclude;
private File source;
private File destinationDirectory;
private boolean createDestinationDirectory;
@@ -54,30 +66,106 @@ public abstract class Converter {
private Set<File> directoriesKnownToExist = new HashSet<>();
private Writer conversionMarkersWriter;
+ public Converter() {
+ include = getDefaultInclude();
+ }
+
+ /**
+ * Getter pair of {@link #setSource(File)}.
+ */
public File getSource() {
return source;
}
+ /**
+ * Sets the directory that contains the files to be converted (subdirectories will be also searched, and so on
+ * recursively), or the single file that should be converted. Can't be {@code null}.
+ */
public void setSource(File source) {
+ _NullArgumentException.check("source", source);
this.source = source != null ? source.getAbsoluteFile() : null;
}
+ /**
+ * Getter pair of {@link #setDestinationDirectory(File)}.
+ */
public File getDestinationDirectory() {
return destinationDirectory;
}
+ /**
+ * Sets the directory into which the converted files will be written. Unless
+ * {@linkplain #setCreateDestinationDirectory(boolean) createDestinationDirectory property} was set to {@code true},
+ * the directory must already exits.
+ * <p>
+ * Already existing files will be overwritten without warning. Already existing files and subdirectories for which
+ * there's no clashing output will be left as is.
+ */
public void setDestinationDirectory(File destinationDirectory) {
this.destinationDirectory = destinationDirectory != null ? destinationDirectory.getAbsoluteFile() : null;
}
+ /**
+ * Getter pair of {@link #setCreateDestinationDirectory(boolean)}.
+ */
public boolean isCreateDestinationDirectory() {
return createDestinationDirectory;
}
+ /**
+ * Sets whether the destination directory should be created if it doesn't exist. Defaults to {@code false}.
+ * Subdirectories inside the destination directory will be always automatically created, regardless of this
+ * property.
+ */
public void setCreateDestinationDirectory(boolean createDestinationDirectory) {
this.createDestinationDirectory = createDestinationDirectory;
}
+ /**
+ * Getter pair of {@link #setInclude(Pattern)}.
+ */
+ public Pattern getInclude() {
+ return include;
+ }
+
+ /**
+ * Sets what subset of the files selected by the {@linkplain #setSource(File) source property} will be
+ * processed (unless they are excluded by the {@linkplain #setExclude(Pattern) exclude} property.)
+ * Default is {@link #getDefaultInclude()}.
+ *
+ * @param include Matched against the source directory relative path. In the matched path, backslashes are
+ * replaced with slash. If {@code null}, then matches everything.
+ */
+ public void setInclude(Pattern include) {
+ this.include = include;
+ }
+
+ /**
+ * Returns the default value of the {@linkplain #setInclude(Pattern) include property}.
+ */
+ protected abstract Pattern getDefaultInclude();
+
+ /**
+ * Getter pair of {@link #setExclude(Pattern)}.
+ */
+ public Pattern getExclude() {
+ return exclude;
+ }
+
+ /**
+ * Sets what subset of the files selected by the {@linkplain #setSource(File) source property} will not be
+ * processed. Default is {@code null}.
+ *
+ * @param exclude Matched against the source directory relative path. In the matched path, backslashes are
+ * replaced with slash. If {@code null}, then matches noting (nothing will be excluded).
+ */
+ public void setExclude(Pattern exclude) {
+ this.exclude = exclude;
+ }
+
+ /**
+ * Executes the file conversions.
+ */
public final void execute() throws ConverterException {
if (executed) {
throw new IllegalStateException("This converted was already invoked once.");
@@ -143,6 +231,10 @@ public abstract class Converter {
}
private void convertFile(File src, File dstDir) throws ConverterException {
+ if (!isToBeProcessed(src)) {
+ return;
+ }
+
InputStream srcStream;
try {
srcStream = new FileInputStream(src);
@@ -181,6 +273,30 @@ public abstract class Converter {
}
}
+ private boolean isToBeProcessed(File src) {
+ String relSrcPath;
+ File source = getSource();
+ if (source.isFile()) {
+ relSrcPath = source.getName();
+ } else {
+ relSrcPath = pathToStringWithSlashes(source.toPath().relativize(src.toPath()).normalize());
+ }
+
+ return (include == null || include.matcher(relSrcPath).matches())
+ && (exclude == null || !exclude.matcher(relSrcPath).matches());
+ }
+
+ private String pathToStringWithSlashes(Path path) {
+ StringBuilder sb = new StringBuilder();
+ for (Iterator<Path> it = path.iterator(); it.hasNext();) {
+ if (sb.length() != 0) {
+ sb.append('/');
+ }
+ sb.append(it.next());
+ }
+ return sb.toString();
+ }
+
private void storeConversionMarkers(ConversionMarkers conversionMarkers, FileConversionContext ctx)
throws ConverterException {
if (conversionMarkersWriter == null) {
http://git-wip-us.apache.org/repos/asf/incubator-freemarker/blob/4763581a/freemarker-converter/src/main/java/org/apache/freemarker/converter/FM2ToFM3Converter.java
----------------------------------------------------------------------
diff --git a/freemarker-converter/src/main/java/org/apache/freemarker/converter/FM2ToFM3Converter.java b/freemarker-converter/src/main/java/org/apache/freemarker/converter/FM2ToFM3Converter.java
index 1e024e5..0e2e114 100644
--- a/freemarker-converter/src/main/java/org/apache/freemarker/converter/FM2ToFM3Converter.java
+++ b/freemarker-converter/src/main/java/org/apache/freemarker/converter/FM2ToFM3Converter.java
@@ -24,20 +24,37 @@ import java.nio.charset.StandardCharsets;
import java.util.HashMap;
import java.util.Map;
import java.util.Properties;
+import java.util.regex.Pattern;
import org.apache.commons.io.IOUtils;
import org.apache.freemarker.core.util._NullArgumentException;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import freemarker.core.FM2ASTToFM3SourceConverter;
import freemarker.template.Configuration;
import freemarker.template.Template;
import freemarker.template._TemplateAPI;
+/**
+ * Converts FreeMarker 2 templates to FreeMarker 3 templates, as far as it's possible automatically. While the output
+ * will contain syntactically correct FreeMarker 3 templates, the templates will have to be reviewed by humans, as
+ * due to the semantic differences (such as a different treatment of {@code null}).
+ * <p>
+ * This is work in progress... new conversion are mostly only added when the syntactical change was
+ * already implemented. Current conversions:
+ * <ul>
+ * <li>All FreeMarker defined names are converted camel case
+ * <li>Renamed setting names in the {@code ftl} heading and in {@code #setting} tags are replaced with the new name.
+ * <li>Renamed built-in variables are replaced with the new name
+ * <li>From {@code #else}, {@code #elseif} and {@code #recover} tags that end with {@code "/>"} or {@code "/]"} the
+ * "/" characters is removed (as it's now illegal)
+ * <li>{@code #else}, {@code #elseif} and {@code #recover} tags that end with {@code "/>"} or {@code "/]"} the
+ * "/" characters is removed (as it's now illegal)
+ * <li>The last tag in {@code <#attempt>...<#recover>...</#recover>} is replaced with {@code </#attempt>}
+ * </ul>
+ */
public class FM2ToFM3Converter extends Converter {
- public static final Logger LOG = LoggerFactory.getLogger(Converter.class);
+ private static final Pattern DEFAULT_INCLUDE = Pattern.compile("(?i).*\\.(fm|ftl(x|h)?)");
private static final Map<String, String> DEFAULT_REPLACED_FILE_EXTENSIONS;
static {
@@ -55,16 +72,23 @@ public class FM2ToFM3Converter extends Converter {
private Configuration fm2Cfg;
@Override
+ protected Pattern getDefaultInclude() {
+ return DEFAULT_INCLUDE;
+ }
+
+ @Override
protected void prepare() throws ConverterException {
super.prepare();
fm2Cfg = new Configuration(Configuration.VERSION_2_3_19 /* To fix ignored initial unknown tags */);
fm2Cfg.setWhitespaceStripping(false);
fm2Cfg.setTabSize(1);
_TemplateAPI.setPreventStrippings(fm2Cfg, true);
- try {
- fm2Cfg.setSettings(freeMarker2Settings);
- } catch (Exception e) {
- throw new ConverterException("Error while configuring FreeMarker 2", e);
+ if (freeMarker2Settings != null) {
+ try {
+ fm2Cfg.setSettings(freeMarker2Settings);
+ } catch (Exception e) {
+ throw new ConverterException("Error while configuring FreeMarker 2", e);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-freemarker/blob/4763581a/freemarker-converter/src/test/java/org/freemarker/converter/FM2ToFM3ConverterTest.java
----------------------------------------------------------------------
diff --git a/freemarker-converter/src/test/java/org/freemarker/converter/FM2ToFM3ConverterTest.java b/freemarker-converter/src/test/java/org/freemarker/converter/FM2ToFM3ConverterTest.java
index 7d691c9..07af58b 100644
--- a/freemarker-converter/src/test/java/org/freemarker/converter/FM2ToFM3ConverterTest.java
+++ b/freemarker-converter/src/test/java/org/freemarker/converter/FM2ToFM3ConverterTest.java
@@ -426,7 +426,27 @@ public class FM2ToFM3ConverterTest extends ConverterTest {
}
@Test
- public void testFileExtensions() throws IOException, ConverterException {
+ public void testDefaultIncludes() throws IOException, ConverterException {
+ FileUtils.write(new File(srcDir, "t.txt"), "x", UTF_8);
+ FileUtils.write(new File(srcDir, "t.fm"), "x", UTF_8);
+ FileUtils.write(new File(srcDir, "t.ftl"), "x", UTF_8);
+ FileUtils.write(new File(srcDir, "t.ftlfoo"), "x", UTF_8);
+ FileUtils.write(new File(srcDir, "U.FTLH"), "x", UTF_8);
+
+ FM2ToFM3Converter converter = new FM2ToFM3Converter();
+ converter.setSource(srcDir);
+ converter.setDestinationDirectory(dstDir);
+ converter.execute();
+
+ assertFalse(new File(dstDir, "t.txt").exists());
+ assertTrue(new File(dstDir, "t.fm3").exists());
+ assertTrue(new File(dstDir, "t.fm3").exists());
+ assertFalse(new File(dstDir, "t.ftlfoo").exists());
+ assertTrue(new File(dstDir, "U.fm3h").exists());
+ }
+
+ @Test
+ public void testFileExtensionConversion() throws IOException, ConverterException {
FileUtils.write(new File(srcDir, "t1"), "x", UTF_8);
FileUtils.write(new File(srcDir, "t2.foo"), "x", UTF_8);
FileUtils.write(new File(srcDir, "t3.ftl"), "x", UTF_8);
@@ -441,10 +461,7 @@ public class FM2ToFM3ConverterTest extends ConverterTest {
FM2ToFM3Converter converter = new FM2ToFM3Converter();
converter.setSource(srcDir);
converter.setDestinationDirectory(dstDir);
- Properties properties = new Properties();
- properties.setProperty(Configuration.DEFAULT_ENCODING_KEY, UTF_8.name());
- converter.setFreeMarker2Settings(properties);
-
+ converter.setInclude(null);
converter.execute();
assertTrue(new File(dstDir, "t1").exists());
@@ -496,6 +513,7 @@ public class FM2ToFM3ConverterTest extends ConverterTest {
FM2ToFM3Converter converter = new FM2ToFM3Converter();
converter.setSource(srcFile);
converter.setDestinationDirectory(dstDir);
+ converter.setInclude(null);
Properties properties = new Properties();
properties.setProperty(Configuration.DEFAULT_ENCODING_KEY, UTF_8.name());
if (squareBracketTagSyntax) {
http://git-wip-us.apache.org/repos/asf/incubator-freemarker/blob/4763581a/freemarker-converter/src/test/java/org/freemarker/converter/GenericConverterTest.java
----------------------------------------------------------------------
diff --git a/freemarker-converter/src/test/java/org/freemarker/converter/GenericConverterTest.java b/freemarker-converter/src/test/java/org/freemarker/converter/GenericConverterTest.java
index 7fcee1a..0873133 100644
--- a/freemarker-converter/src/test/java/org/freemarker/converter/GenericConverterTest.java
+++ b/freemarker-converter/src/test/java/org/freemarker/converter/GenericConverterTest.java
@@ -27,6 +27,7 @@ import static org.junit.Assert.*;
import java.io.File;
import java.io.IOException;
import java.nio.charset.StandardCharsets;
+import java.util.regex.Pattern;
import org.apache.commons.io.IOUtils;
import org.apache.freemarker.converter.ConversionMarkers;
@@ -212,9 +213,47 @@ public class GenericConverterTest extends ConverterTest {
}
}
+ @Test
+ public void testIncludeAndExclude() throws IOException, ConverterException {
+ write(new File(srcDir, "included.txt"), "x", UTF_8);
+ write(new File(srcDir, "not-included.doc"), "x", UTF_8);
+ write(new File(srcDir, "~excluded.txt"), "x", UTF_8);
+
+ ToUpperCaseConverter converter = new ToUpperCaseConverter();
+ converter.setSource(srcDir);
+ converter.setDestinationDirectory(dstDir);
+ converter.setInclude(Pattern.compile("(?i).*\\.txt"));
+ converter.setExclude(Pattern.compile("(.*/)?~[^/]*"));
+ converter.execute();
+
+ assertTrue(new File(dstDir, "included.txt.uc").exists());
+ assertFalse(new File(dstDir, "not-included.doc.uc").exists());
+ assertFalse(new File(dstDir, "~excluded.txt.uc").exists());
+ }
+
+ @Test
+ public void testIncludeAndExclude2() throws IOException, ConverterException {
+ write(new File(srcDir, "included.txt"), "x", UTF_8);
+ write(new File(srcDir, "included2.doc"), "x", UTF_8);
+
+ ToUpperCaseConverter converter = new ToUpperCaseConverter();
+ converter.setSource(srcDir);
+ converter.setDestinationDirectory(dstDir);
+ converter.setInclude(null);
+ converter.execute();
+
+ assertTrue(new File(dstDir, "included.txt.uc").exists());
+ assertTrue(new File(dstDir, "included2.doc.uc").exists());
+ }
+
public static class ToUpperCaseConverter extends Converter {
@Override
+ protected Pattern getDefaultInclude() {
+ return Pattern.compile("(?i).*\\.txt");
+ }
+
+ @Override
protected void convertFile(FileConversionContext ctx) throws ConverterException, IOException {
String content = IOUtils.toString(ctx.getSourceStream(), StandardCharsets.UTF_8);
ctx.setDestinationFileName(ctx.getSourceFileName() + ".uc");