You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hop.apache.org by ha...@apache.org on 2026/08/06 13:09:33 UTC
(hop) branch main updated: issue #3212: Parquet output option for compression before file extension (#7803)
This is an automated email from the ASF dual-hosted git repository.
hansva pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/hop.git
The following commit(s) were added to refs/heads/main by this push:
new af95f52002 issue #3212: Parquet output option for compression before file extension (#7803)
af95f52002 is described below
commit af95f52002fc55bbc68ca006e7d1f14180abc270
Author: Matt Casters <ma...@gmail.com>
AuthorDate: Thu Aug 6 15:09:27 2026 +0200
issue #3212: Parquet output option for compression before file extension (#7803)
Add a checkbox so filenames can use Spark-style naming (file.snappy.parquet).
New transforms default to the new order; existing pipelines keep the old
file.parquet.snappy behavior when the property is absent.
---
.../pipeline/transforms/parquet-file-output.adoc | 5 ++
.../parquet/transforms/output/ParquetOutput.java | 15 ++++--
.../transforms/output/ParquetOutputDialog.java | 23 +++++++++
.../transforms/output/ParquetOutputMeta.java | 27 +++++++++++
.../output/messages/messages_en_US.properties | 1 +
.../transforms/output/ParquetOutputMetaTest.java | 7 +++
.../transforms/output/ParquetOutputTest.java | 54 ++++++++++++++++++++++
7 files changed, 129 insertions(+), 3 deletions(-)
diff --git a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/parquet-file-output.adoc b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/parquet-file-output.adoc
index 4535b6f89e..1eccd4e01b 100644
--- a/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/parquet-file-output.adoc
+++ b/docs/hop-user-manual/modules/ROOT/pages/pipeline/transforms/parquet-file-output.adoc
@@ -92,6 +92,11 @@ Specify a split size larger than 0 and this is then the number of rows per file.
The file part (split) number will be included in the filename to make sure that the same file is not being overwritten.
The split number is formatted with mask `0000`
+|Include compression codec before extension?
+|When enabled (the default for new transforms), the compression codec is placed before the file extension (for example `file.snappy.parquet`), which matches the naming used by Spark and other Parquet tools.
+When disabled, the compression codec is appended after the extension (for example `file.parquet.snappy`).
+Existing pipelines that do not have this option set keep the previous behavior.
+
|Compression codec
|Here you can indicate which compression codec you want to use.
The default is SNAPPY for Apache Snappy compression.
diff --git a/plugins/tech/parquet/src/main/java/org/apache/hop/parquet/transforms/output/ParquetOutput.java b/plugins/tech/parquet/src/main/java/org/apache/hop/parquet/transforms/output/ParquetOutput.java
index 704b28047a..9c7fef4f21 100644
--- a/plugins/tech/parquet/src/main/java/org/apache/hop/parquet/transforms/output/ParquetOutput.java
+++ b/plugins/tech/parquet/src/main/java/org/apache/hop/parquet/transforms/output/ParquetOutput.java
@@ -295,7 +295,7 @@ public class ParquetOutput extends BaseTransform<ParquetOutputMeta, ParquetOutpu
}
}
- private String buildFilename(Date date) {
+ String buildFilename(Date date) {
String filename = resolve(meta.getFilenameBase());
if (meta.isFilenameIncludingDate()) {
filename += "-" + new SimpleDateFormat("yyyyMMdd").format(date);
@@ -316,8 +316,17 @@ public class ParquetOutput extends BaseTransform<ParquetOutputMeta, ParquetOutpu
if (data.isBeamContext()) {
filename += "_" + getLogChannelId() + "_" + data.getBeamBundleNr();
}
- filename += "." + Const.NVL(resolve(meta.getFilenameExtension()), "parquet");
- filename += meta.getCompressionCodec().getExtension();
+ String extension = Const.NVL(resolve(meta.getFilenameExtension()), "parquet");
+ String compressionExtension = meta.getCompressionCodec().getExtension();
+ if (meta.isFilenameCompressionBeforeExtension()) {
+ // Spark-style: file.snappy.parquet
+ filename += compressionExtension;
+ filename += "." + extension;
+ } else {
+ // Backward compatible: file.parquet.snappy
+ filename += "." + extension;
+ filename += compressionExtension;
+ }
return filename;
}
diff --git a/plugins/tech/parquet/src/main/java/org/apache/hop/parquet/transforms/output/ParquetOutputDialog.java b/plugins/tech/parquet/src/main/java/org/apache/hop/parquet/transforms/output/ParquetOutputDialog.java
index b1cd5baf68..fd6d570618 100644
--- a/plugins/tech/parquet/src/main/java/org/apache/hop/parquet/transforms/output/ParquetOutputDialog.java
+++ b/plugins/tech/parquet/src/main/java/org/apache/hop/parquet/transforms/output/ParquetOutputDialog.java
@@ -62,6 +62,7 @@ public class ParquetOutputDialog extends BaseTransformDialog {
private Label wlFilenameSplitSize;
private TextVar wFilenameSplitSize;
private Button wFilenameCreateFolders;
+ private Button wFilenameCompressionBeforeExtension;
private Combo wCompressionCodec;
private Combo wVersion;
private TextVar wRowGroupSize;
@@ -302,6 +303,26 @@ public class ParquetOutputDialog extends BaseTransformDialog {
fdFilenameCreateFolders.top = new FormAttachment(wlFilenameCreateFolders, 0, SWT.CENTER);
fdFilenameCreateFolders.right = new FormAttachment(100, 0);
wFilenameCreateFolders.setLayoutData(fdFilenameCreateFolders);
+ lastControl = wlFilenameCreateFolders;
+
+ Label wlFilenameCompressionBeforeExtension = new Label(wFileGroup, SWT.RIGHT);
+ wlFilenameCompressionBeforeExtension.setText(
+ BaseMessages.getString(
+ PKG, "ParquetOutputDialog.FilenameCompressionBeforeExtension.Label"));
+ PropsUi.setLook(wlFilenameCompressionBeforeExtension);
+ FormData fdlFilenameCompressionBeforeExtension = new FormData();
+ fdlFilenameCompressionBeforeExtension.left = new FormAttachment(0, 0);
+ fdlFilenameCompressionBeforeExtension.right = new FormAttachment(middle, -margin);
+ fdlFilenameCompressionBeforeExtension.top = new FormAttachment(lastControl, margin);
+ wlFilenameCompressionBeforeExtension.setLayoutData(fdlFilenameCompressionBeforeExtension);
+ wFilenameCompressionBeforeExtension = new Button(wFileGroup, SWT.CHECK);
+ PropsUi.setLook(wFilenameCompressionBeforeExtension);
+ FormData fdFilenameCompressionBeforeExtension = new FormData();
+ fdFilenameCompressionBeforeExtension.left = new FormAttachment(middle, 0);
+ fdFilenameCompressionBeforeExtension.top =
+ new FormAttachment(wlFilenameCompressionBeforeExtension, 0, SWT.CENTER);
+ fdFilenameCompressionBeforeExtension.right = new FormAttachment(100, 0);
+ wFilenameCompressionBeforeExtension.setLayoutData(fdFilenameCompressionBeforeExtension);
// End of the file group
//
@@ -478,6 +499,7 @@ public class ParquetOutputDialog extends BaseTransformDialog {
wFilenameIncludeSplitNr.setSelection(input.isFilenameIncludingSplitNr());
wFilenameSplitSize.setText(Const.NVL(input.getFileSplitSize(), ""));
wFilenameCreateFolders.setSelection(input.isFilenameCreatingParentFolders());
+ wFilenameCompressionBeforeExtension.setSelection(input.isFilenameCompressionBeforeExtension());
wCompressionCodec.setText(input.getCompressionCodec().name());
wVersion.setText(input.getVersion().getDescription());
wRowGroupSize.setText(Const.NVL(input.getRowGroupSize(), ""));
@@ -506,6 +528,7 @@ public class ParquetOutputDialog extends BaseTransformDialog {
input.setFilenameIncludingSplitNr(wFilenameIncludeSplitNr.getSelection());
input.setFileSplitSize(wFilenameSplitSize.getText());
input.setFilenameCreatingParentFolders(wFilenameCreateFolders.getSelection());
+ input.setFilenameCompressionBeforeExtension(wFilenameCompressionBeforeExtension.getSelection());
CompressionCodecName codec = CompressionCodecName.UNCOMPRESSED;
try {
diff --git a/plugins/tech/parquet/src/main/java/org/apache/hop/parquet/transforms/output/ParquetOutputMeta.java b/plugins/tech/parquet/src/main/java/org/apache/hop/parquet/transforms/output/ParquetOutputMeta.java
index 410f15237a..2cfb84df76 100644
--- a/plugins/tech/parquet/src/main/java/org/apache/hop/parquet/transforms/output/ParquetOutputMeta.java
+++ b/plugins/tech/parquet/src/main/java/org/apache/hop/parquet/transforms/output/ParquetOutputMeta.java
@@ -65,6 +65,15 @@ public class ParquetOutputMeta extends BaseTransformMeta<ParquetOutput, ParquetO
@HopMetadataProperty(key = "filename_create_parent_folders")
private boolean filenameCreatingParentFolders;
+ /**
+ * When true, the compression codec extension is placed before the file extension (e.g.
+ * file.snappy.parquet). When false, it is appended after the file extension (e.g.
+ * file.parquet.snappy). New transforms default to true; pipelines loaded without this property
+ * keep false for backward compatibility.
+ */
+ @HopMetadataProperty(key = "filename_compression_before_extension")
+ private boolean filenameCompressionBeforeExtension;
+
@HopMetadataProperty(key = "compression_codec")
private CompressionCodecName compressionCodec;
@@ -94,6 +103,7 @@ public class ParquetOutputMeta extends BaseTransformMeta<ParquetOutput, ParquetO
filenameIncludingCopyNr = true;
filenameIncludingSplitNr = true;
filenameCreatingParentFolders = true;
+ filenameCompressionBeforeExtension = true;
fileSplitSize = "1000000";
}
@@ -108,6 +118,7 @@ public class ParquetOutputMeta extends BaseTransformMeta<ParquetOutput, ParquetO
this.filenameIncludingSplitNr = m.filenameIncludingSplitNr;
this.fileSplitSize = m.fileSplitSize;
this.filenameCreatingParentFolders = m.filenameCreatingParentFolders;
+ this.filenameCompressionBeforeExtension = m.filenameCompressionBeforeExtension;
this.compressionCodec = m.compressionCodec;
this.version = m.version;
this.rowGroupSize = m.rowGroupSize;
@@ -276,6 +287,22 @@ public class ParquetOutputMeta extends BaseTransformMeta<ParquetOutput, ParquetO
this.filenameCreatingParentFolders = filenameCreatingParentFolders;
}
+ /**
+ * Gets filenameCompressionBeforeExtension
+ *
+ * @return value of filenameCompressionBeforeExtension
+ */
+ public boolean isFilenameCompressionBeforeExtension() {
+ return filenameCompressionBeforeExtension;
+ }
+
+ /**
+ * @param filenameCompressionBeforeExtension The filenameCompressionBeforeExtension to set
+ */
+ public void setFilenameCompressionBeforeExtension(boolean filenameCompressionBeforeExtension) {
+ this.filenameCompressionBeforeExtension = filenameCompressionBeforeExtension;
+ }
+
/**
* Gets compressionCodec
*
diff --git a/plugins/tech/parquet/src/main/resources/org/apache/hop/parquet/transforms/output/messages/messages_en_US.properties b/plugins/tech/parquet/src/main/resources/org/apache/hop/parquet/transforms/output/messages/messages_en_US.properties
index 95f202a77b..db680bb9cb 100644
--- a/plugins/tech/parquet/src/main/resources/org/apache/hop/parquet/transforms/output/messages/messages_en_US.properties
+++ b/plugins/tech/parquet/src/main/resources/org/apache/hop/parquet/transforms/output/messages/messages_en_US.properties
@@ -25,6 +25,7 @@ ParquetOutputDialog.FieldsColumn.SourceField.Label=Source field
ParquetOutputDialog.FieldsColumn.TargetField.Label=Target field
ParquetOutputDialog.FilenameBase.Label=Base file name
ParquetOutputDialog.FilenameCreateFolders.Label=Create parent folders
+ParquetOutputDialog.FilenameCompressionBeforeExtension.Label=Include compression codec before extension
ParquetOutputDialog.FilenameDateTimeFormat.Label=Date time format
ParquetOutputDialog.FilenameExtension.Label=Extension
ParquetOutputDialog.FilenameGroup.Label=Filename
diff --git a/plugins/tech/parquet/src/test/java/org/apache/hop/parquet/transforms/output/ParquetOutputMetaTest.java b/plugins/tech/parquet/src/test/java/org/apache/hop/parquet/transforms/output/ParquetOutputMetaTest.java
index b0bdf44f15..6924a66926 100644
--- a/plugins/tech/parquet/src/test/java/org/apache/hop/parquet/transforms/output/ParquetOutputMetaTest.java
+++ b/plugins/tech/parquet/src/test/java/org/apache/hop/parquet/transforms/output/ParquetOutputMetaTest.java
@@ -51,6 +51,7 @@ class ParquetOutputMetaTest {
assertFalse(meta.isFilenameIncludingDate());
assertFalse(meta.isFilenameIncludingTime());
assertFalse(meta.isFilenameIncludingDateTime());
+ assertTrue(meta.isFilenameCompressionBeforeExtension());
assertTrue(meta.getFields().isEmpty());
}
@@ -60,6 +61,7 @@ class ParquetOutputMetaTest {
original.setFilenameBase("/tmp/output");
original.setFilenameExtension("pq");
original.setFilenameIncludingDate(true);
+ original.setFilenameCompressionBeforeExtension(true);
original.setCompressionCodec(CompressionCodecName.SNAPPY);
original.setVersion(ParquetVersion.Version1);
original.setFields(List.of(new ParquetField("id", "id_out")));
@@ -68,6 +70,7 @@ class ParquetOutputMetaTest {
assertEquals("/tmp/output", copy.getFilenameBase());
assertEquals("pq", copy.getFilenameExtension());
assertTrue(copy.isFilenameIncludingDate());
+ assertTrue(copy.isFilenameCompressionBeforeExtension());
assertEquals(CompressionCodecName.SNAPPY, copy.getCompressionCodec());
assertEquals(ParquetVersion.Version1, copy.getVersion());
assertEquals(1, copy.getFields().size());
@@ -87,6 +90,7 @@ class ParquetOutputMetaTest {
meta.setFilenameIncludingSplitNr(false);
meta.setFileSplitSize("500");
meta.setFilenameCreatingParentFolders(false);
+ meta.setFilenameCompressionBeforeExtension(true);
meta.setCompressionCodec(CompressionCodecName.GZIP);
meta.setVersion(ParquetVersion.Version1);
meta.setRowGroupSize("1024");
@@ -122,6 +126,9 @@ class ParquetOutputMetaTest {
assertEquals(expected.getFileSplitSize(), actual.getFileSplitSize());
assertEquals(
expected.isFilenameCreatingParentFolders(), actual.isFilenameCreatingParentFolders());
+ assertEquals(
+ expected.isFilenameCompressionBeforeExtension(),
+ actual.isFilenameCompressionBeforeExtension());
assertEquals(expected.getCompressionCodec(), actual.getCompressionCodec());
assertEquals(expected.getVersion(), actual.getVersion());
assertEquals(expected.getRowGroupSize(), actual.getRowGroupSize());
diff --git a/plugins/tech/parquet/src/test/java/org/apache/hop/parquet/transforms/output/ParquetOutputTest.java b/plugins/tech/parquet/src/test/java/org/apache/hop/parquet/transforms/output/ParquetOutputTest.java
index 251571a5ce..2780c0fc77 100644
--- a/plugins/tech/parquet/src/test/java/org/apache/hop/parquet/transforms/output/ParquetOutputTest.java
+++ b/plugins/tech/parquet/src/test/java/org/apache/hop/parquet/transforms/output/ParquetOutputTest.java
@@ -25,6 +25,9 @@ import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.when;
+import java.util.Calendar;
+import java.util.Date;
+import java.util.GregorianCalendar;
import org.apache.hop.core.exception.HopException;
import org.apache.hop.core.logging.ILoggingObject;
import org.apache.hop.core.row.RowMeta;
@@ -37,6 +40,7 @@ import org.apache.hop.pipeline.engines.local.LocalPipelineEngine;
import org.apache.hop.pipeline.transform.TransformMeta;
import org.apache.hop.pipeline.transforms.mock.TransformMockHelper;
import org.apache.parquet.column.ParquetProperties;
+import org.apache.parquet.hadoop.metadata.CompressionCodecName;
import org.junit.jupiter.api.AfterEach;
import org.junit.jupiter.api.BeforeEach;
import org.junit.jupiter.api.Test;
@@ -168,6 +172,56 @@ class ParquetOutputTest {
assertEquals("name", data.outputFields.get(1).getTargetFieldName());
}
+ @Test
+ void testBuildFilenameCompressionBeforeExtensionByDefault() {
+ ParquetOutputMeta meta = new ParquetOutputMeta();
+ meta.setFilenameBase("/tmp/output");
+ meta.setFilenameExtension("parquet");
+ meta.setFilenameIncludingCopyNr(false);
+ meta.setFilenameIncludingSplitNr(false);
+ meta.setCompressionCodec(CompressionCodecName.SNAPPY);
+
+ ParquetOutputData data = new ParquetOutputData();
+ ParquetOutput output = createTransform(meta, data);
+
+ assertEquals("/tmp/output.snappy.parquet", output.buildFilename(fixedDate()));
+ }
+
+ @Test
+ void testBuildFilenameCompressionAfterExtensionWhenDisabled() {
+ ParquetOutputMeta meta = new ParquetOutputMeta();
+ meta.setFilenameBase("/tmp/output");
+ meta.setFilenameExtension("parquet");
+ meta.setFilenameIncludingCopyNr(false);
+ meta.setFilenameIncludingSplitNr(false);
+ meta.setFilenameCompressionBeforeExtension(false);
+ meta.setCompressionCodec(CompressionCodecName.SNAPPY);
+
+ ParquetOutputData data = new ParquetOutputData();
+ ParquetOutput output = createTransform(meta, data);
+
+ assertEquals("/tmp/output.parquet.snappy", output.buildFilename(fixedDate()));
+ }
+
+ @Test
+ void testBuildFilenameUncompressedHasNoCodecExtension() {
+ ParquetOutputMeta meta = new ParquetOutputMeta();
+ meta.setFilenameBase("/tmp/output");
+ meta.setFilenameExtension("parquet");
+ meta.setFilenameIncludingCopyNr(false);
+ meta.setFilenameIncludingSplitNr(false);
+ meta.setCompressionCodec(CompressionCodecName.UNCOMPRESSED);
+
+ ParquetOutputData data = new ParquetOutputData();
+ ParquetOutput output = createTransform(meta, data);
+
+ assertEquals("/tmp/output.parquet", output.buildFilename(fixedDate()));
+ }
+
+ private static Date fixedDate() {
+ return new GregorianCalendar(2024, Calendar.JANUARY, 15, 10, 30, 0).getTime();
+ }
+
private ParquetOutput createTransform(ParquetOutputMeta meta, ParquetOutputData data) {
PipelineMeta pipelineMeta = new PipelineMeta();
TransformMeta transformMeta = new TransformMeta("Parquet Output", meta);