You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@paimon.apache.org by lz...@apache.org on 2023/03/22 04:24:00 UTC
[incubator-paimon] branch master updated: [test] add test for CoreOptions.testCreateFileFormat method (#678)
This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-paimon.git
The following commit(s) were added to refs/heads/master by this push:
new ae312c105 [test] add test for CoreOptions.testCreateFileFormat method (#678)
ae312c105 is described below
commit ae312c105c6c3a60d1ff3a317d19f17249466de8
Author: JunZhang <zh...@126.com>
AuthorDate: Wed Mar 22 12:23:55 2023 +0800
[test] add test for CoreOptions.testCreateFileFormat method (#678)
---
.../src/main/java/org/apache/paimon/CoreOptions.java | 5 ++---
.../org/apache/paimon/format/FileFormatDiscover.java | 4 ++--
.../src/test/java/org/apache/paimon/FileFormatTest.java | 16 ++++++++++++++++
.../java/org/apache/paimon/format/orc/OrcFileFormat.java | 5 +++++
.../org/apache/paimon/format/orc/OrcFileFormatTest.java | 10 +++-------
.../paimon/format/parquet/ParquetFileFormatTest.java | 11 ++++-------
6 files changed, 32 insertions(+), 19 deletions(-)
diff --git a/paimon-core/src/main/java/org/apache/paimon/CoreOptions.java b/paimon-core/src/main/java/org/apache/paimon/CoreOptions.java
index acb986244..6da5090d3 100644
--- a/paimon-core/src/main/java/org/apache/paimon/CoreOptions.java
+++ b/paimon-core/src/main/java/org/apache/paimon/CoreOptions.java
@@ -21,7 +21,7 @@ package org.apache.paimon;
import org.apache.paimon.annotation.Documentation.ExcludeFromDocumentation;
import org.apache.paimon.annotation.Documentation.Immutable;
import org.apache.paimon.format.FileFormat;
-import org.apache.paimon.format.FileFormatFactory;
+import org.apache.paimon.format.FileFormatFactory.FormatContext;
import org.apache.paimon.fs.Path;
import org.apache.paimon.options.ConfigOption;
import org.apache.paimon.options.MemorySize;
@@ -585,8 +585,7 @@ public class CoreOptions implements Serializable {
int readBatchSize = options.get(READ_BATCH_SIZE);
return FileFormat.fromIdentifier(
formatIdentifier,
- new FileFormatFactory.FormatContext(
- options.removePrefix(formatIdentifier + "."), readBatchSize));
+ new FormatContext(options.removePrefix(formatIdentifier + "."), readBatchSize));
}
public Map<Integer, String> fileCompressionPerLevel() {
diff --git a/paimon-core/src/main/java/org/apache/paimon/format/FileFormatDiscover.java b/paimon-core/src/main/java/org/apache/paimon/format/FileFormatDiscover.java
index 34f9f8df4..8ce1e2d15 100644
--- a/paimon-core/src/main/java/org/apache/paimon/format/FileFormatDiscover.java
+++ b/paimon-core/src/main/java/org/apache/paimon/format/FileFormatDiscover.java
@@ -19,6 +19,7 @@
package org.apache.paimon.format;
import org.apache.paimon.CoreOptions;
+import org.apache.paimon.format.FileFormatFactory.FormatContext;
import java.util.HashMap;
import java.util.Map;
@@ -38,8 +39,7 @@ public interface FileFormatDiscover {
private FileFormat create(String identifier) {
return FileFormat.fromIdentifier(
identifier,
- new FileFormatFactory.FormatContext(
- options.toConfiguration(), options.readBatchSize()));
+ new FormatContext(options.toConfiguration(), options.readBatchSize()));
}
};
}
diff --git a/paimon-core/src/test/java/org/apache/paimon/FileFormatTest.java b/paimon-core/src/test/java/org/apache/paimon/FileFormatTest.java
index cda1f8940..1ff601443 100644
--- a/paimon-core/src/test/java/org/apache/paimon/FileFormatTest.java
+++ b/paimon-core/src/test/java/org/apache/paimon/FileFormatTest.java
@@ -23,6 +23,7 @@ import org.apache.paimon.data.InternalRow;
import org.apache.paimon.format.FileFormat;
import org.apache.paimon.format.FormatWriter;
import org.apache.paimon.format.FormatWriterFactory;
+import org.apache.paimon.format.orc.OrcFileFormat;
import org.apache.paimon.fs.Path;
import org.apache.paimon.fs.PositionOutputStream;
import org.apache.paimon.fs.local.LocalFileIO;
@@ -39,6 +40,7 @@ import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
+import static org.apache.paimon.format.orc.OrcFileFormatFactory.IDENTIFIER;
import static org.assertj.core.api.Assertions.assertThat;
/** Test for {@link FileFormat}. */
@@ -85,6 +87,20 @@ public class FileFormatTest {
"Unrecognized codec: _unsupported");
}
+ @Test
+ public void testCreateFileFormat() {
+ Options tableOptions = new Options();
+ tableOptions.set(CoreOptions.FILE_FORMAT, IDENTIFIER);
+ tableOptions.set(CoreOptions.READ_BATCH_SIZE, 1024);
+ tableOptions.setString(IDENTIFIER + ".hello", "world");
+ FileFormat fileFormat = CoreOptions.createFileFormat(tableOptions, CoreOptions.FILE_FORMAT);
+ Assertions.assertTrue(fileFormat instanceof OrcFileFormat);
+
+ OrcFileFormat orcFileFormat = (OrcFileFormat) fileFormat;
+ assertThat(orcFileFormat.formatContext().formatOptions().get("hello")).isEqualTo("world");
+ assertThat(orcFileFormat.formatContext().readBatchSize()).isEqualTo(1024);
+ }
+
public FileFormat createFileFormat(String codec) {
Options tableOptions = new Options();
tableOptions.set(CoreOptions.FILE_FORMAT, "avro");
diff --git a/paimon-format/src/main/java/org/apache/paimon/format/orc/OrcFileFormat.java b/paimon-format/src/main/java/org/apache/paimon/format/orc/OrcFileFormat.java
index 8ac95f2bb..8480e2aa3 100644
--- a/paimon-format/src/main/java/org/apache/paimon/format/orc/OrcFileFormat.java
+++ b/paimon-format/src/main/java/org/apache/paimon/format/orc/OrcFileFormat.java
@@ -80,6 +80,11 @@ public class OrcFileFormat extends FileFormat {
return orcProperties;
}
+ @VisibleForTesting
+ public FormatContext formatContext() {
+ return formatContext;
+ }
+
@Override
public Optional<FileStatsExtractor> createStatsExtractor(RowType type) {
return Optional.of(new OrcFileStatsExtractor(type));
diff --git a/paimon-format/src/test/java/org/apache/paimon/format/orc/OrcFileFormatTest.java b/paimon-format/src/test/java/org/apache/paimon/format/orc/OrcFileFormatTest.java
index 42fe80ce4..fd5699894 100644
--- a/paimon-format/src/test/java/org/apache/paimon/format/orc/OrcFileFormatTest.java
+++ b/paimon-format/src/test/java/org/apache/paimon/format/orc/OrcFileFormatTest.java
@@ -18,7 +18,7 @@
package org.apache.paimon.format.orc;
-import org.apache.paimon.format.FileFormatFactory;
+import org.apache.paimon.format.FileFormatFactory.FormatContext;
import org.apache.paimon.options.Options;
import org.junit.jupiter.api.Test;
@@ -33,9 +33,7 @@ public class OrcFileFormatTest {
public void testAbsent() {
Options options = new Options();
options.setString("haha", "1");
- OrcFileFormat orc =
- new OrcFileFormatFactory()
- .create(new FileFormatFactory.FormatContext(options, 1024));
+ OrcFileFormat orc = new OrcFileFormatFactory().create(new FormatContext(options, 1024));
assertThat(orc.orcProperties().getProperty(IDENTIFIER + ".haha", "")).isEqualTo("1");
assertThat(orc.orcProperties().getProperty(IDENTIFIER + ".compress", "")).isEqualTo("lz4");
}
@@ -45,9 +43,7 @@ public class OrcFileFormatTest {
Options options = new Options();
options.setString("haha", "1");
options.setString("compress", "zlib");
- OrcFileFormat orc =
- new OrcFileFormatFactory()
- .create(new FileFormatFactory.FormatContext(options, 1024));
+ OrcFileFormat orc = new OrcFileFormatFactory().create(new FormatContext(options, 1024));
assertThat(orc.orcProperties().getProperty(IDENTIFIER + ".haha", "")).isEqualTo("1");
assertThat(orc.orcProperties().getProperty(IDENTIFIER + ".compress", "")).isEqualTo("zlib");
}
diff --git a/paimon-format/src/test/java/org/apache/paimon/format/parquet/ParquetFileFormatTest.java b/paimon-format/src/test/java/org/apache/paimon/format/parquet/ParquetFileFormatTest.java
index f67a91abb..404965782 100644
--- a/paimon-format/src/test/java/org/apache/paimon/format/parquet/ParquetFileFormatTest.java
+++ b/paimon-format/src/test/java/org/apache/paimon/format/parquet/ParquetFileFormatTest.java
@@ -18,7 +18,7 @@
package org.apache.paimon.format.parquet;
-import org.apache.paimon.format.FileFormatFactory;
+import org.apache.paimon.format.FileFormatFactory.FormatContext;
import org.apache.paimon.options.ConfigOption;
import org.apache.paimon.options.ConfigOptions;
import org.apache.paimon.options.Options;
@@ -40,8 +40,7 @@ public class ParquetFileFormatTest {
public void testAbsent() {
Options options = new Options();
ParquetFileFormat parquet =
- new ParquetFileFormatFactory()
- .create(new FileFormatFactory.FormatContext(options, 1024));
+ new ParquetFileFormatFactory().create(new FormatContext(options, 1024));
assertThat(parquet.formatOptions().getString(KEY1)).isEqualTo("absent");
}
@@ -50,8 +49,7 @@ public class ParquetFileFormatTest {
Options options = new Options();
options.setString(KEY1.key(), "v1");
ParquetFileFormat parquet =
- new ParquetFileFormatFactory()
- .create(new FileFormatFactory.FormatContext(options, 1024));
+ new ParquetFileFormatFactory().create(new FormatContext(options, 1024));
assertThat(parquet.formatOptions().getString(KEY1)).isEqualTo("v1");
}
@@ -72,8 +70,7 @@ public class ParquetFileFormatTest {
private String getCompressionCodec(Options conf) {
Options formatOptions = conf.removePrefix(IDENTIFIER + ".");
ParquetFileFormat parquet =
- new ParquetFileFormatFactory()
- .create(new FileFormatFactory.FormatContext(formatOptions, 1024));
+ new ParquetFileFormatFactory().create(new FormatContext(formatOptions, 1024));
return getParquetConfiguration(parquet.formatOptions())
.getString(ParquetOutputFormat.COMPRESSION, null);
}