You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@flink.apache.org by GitBox <gi...@apache.org> on 2022/03/31 12:29:36 UTC
[GitHub] [flink] alpreu commented on a change in pull request #19224: [hotfix][test][formats] Detach TableCsvFormatITCase from JsonPlanTestBase
alpreu commented on a change in pull request #19224:
URL: https://github.com/apache/flink/pull/19224#discussion_r839542592
##########
File path: flink-formats/flink-csv/src/test/java/org/apache/flink/formats/csv/TableCsvFormatITCase.java
##########
@@ -18,32 +18,78 @@
package org.apache.flink.formats.csv;
+import org.apache.flink.connector.file.table.FileSystemConnectorOptions;
+import org.apache.flink.connector.file.table.FileSystemTableFactory;
import org.apache.flink.formats.common.TimeFormats;
+import org.apache.flink.table.api.EnvironmentSettings;
+import org.apache.flink.table.api.Schema;
+import org.apache.flink.table.api.TableDescriptor;
+import org.apache.flink.table.api.TableEnvironment;
+import org.apache.flink.table.planner.factories.TestValuesTableFactory;
import org.apache.flink.table.planner.runtime.utils.TestData;
import org.apache.flink.table.planner.utils.JavaScalaConversionUtil;
-import org.apache.flink.table.planner.utils.JsonPlanTestBase;
+import org.apache.flink.test.util.AbstractTestBase;
+import org.apache.flink.types.Row;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+import javax.annotation.Nullable;
import java.io.File;
import java.io.IOException;
import java.nio.file.Files;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import static org.apache.flink.table.api.DataTypes.BIGINT;
+import static org.apache.flink.table.api.DataTypes.INT;
+import static org.apache.flink.table.api.DataTypes.STRING;
+import static org.apache.flink.table.api.DataTypes.TIMESTAMP;
import static org.apache.flink.table.utils.DateTimeUtils.toLocalDateTime;
+import static org.apache.flink.util.Preconditions.checkNotNull;
+import static org.assertj.core.api.Assertions.assertThat;
/** Tests for the CSV file format. */
-public class TableCsvFormatITCase extends JsonPlanTestBase {
+public class TableCsvFormatITCase extends AbstractTestBase {
+
+ @Rule public ExpectedException exception = ExpectedException.none();
+
+ private TableEnvironment tableEnv;
+
+ @Before
+ public void setup() throws Exception {
+ tableEnv = TableEnvironment.create(EnvironmentSettings.inStreamingMode());
+ }
+
+ @After
+ public void after() {
+ TestValuesTableFactory.clearAllData();
+ }
@Test
public void testProjectPushDown() throws Exception {
List<String> data = Arrays.asList("1,1,hi", "2,1,hello", "3,2,hello world");
- createSourceTable("MyTable", data, "a bigint", "b int not null", "c varchar");
- File sinkPath = createSinkTable("MySink", "a bigint", "c varchar");
+
+ Schema sourceSchema =
+ Schema.newBuilder()
+ .column("a", BIGINT())
+ .column("b", INT())
+ .column("c", STRING())
+ .build();
+
+ createSourceTable("MyTable", data, sourceSchema);
+
+ Schema sinkSchema = Schema.newBuilder().column("a", BIGINT()).column("c", STRING()).build();
+
+ File sinkPath = createSinkTable("MySink", sinkSchema);
Review comment:
Can we just reuse sourceSchema here?
##########
File path: flink-formats/flink-csv/src/test/java/org/apache/flink/formats/csv/TableCsvFormatITCase.java
##########
@@ -165,30 +269,87 @@ private static String formatSqlTimestamp(long timestamp) {
return TimeFormats.SQL_TIMESTAMP_FORMAT.format(toLocalDateTime(timestamp));
}
- private void createSourceTable(String tableName, List<String> data, String... fieldNameAndTypes)
+ private void createSourceTable(String tableName, List<String> data, Schema schema)
throws IOException {
+
File sourceFile = TEMPORARY_FOLDER.newFile();
Collections.shuffle(data);
Files.write(sourceFile.toPath(), String.join("\n", data).getBytes());
- Map<String, String> properties = new HashMap<>();
- properties.put("connector", "filesystem");
- properties.put("path", sourceFile.getAbsolutePath());
- properties.put("format", "csv");
-
- createTestSourceTable(tableName, fieldNameAndTypes, null, properties);
+ tableEnv.createTemporaryTable(
+ tableName,
+ TableDescriptor.forConnector(FileSystemTableFactory.IDENTIFIER)
+ .option(FileSystemConnectorOptions.PATH, sourceFile.getAbsolutePath())
+ .format(CsvCommons.IDENTIFIER)
+ .schema(schema)
+ .build());
}
- private File createSinkTable(String tableName, String... fieldNameAndTypes) throws IOException {
+ private File createSinkTable(String tableName, Schema schema) throws IOException {
File sinkPath = TEMPORARY_FOLDER.newFolder();
- Map<String, String> properties = new HashMap<>();
- properties.put("connector", "filesystem");
- properties.put("path", sinkPath.getAbsolutePath());
- properties.put("format", "csv");
- properties.put("csv.disable-quote-character", "true");
+ tableEnv.createTemporaryTable(
+ tableName,
+ TableDescriptor.forConnector(FileSystemTableFactory.IDENTIFIER)
+ .option(FileSystemConnectorOptions.PATH, sinkPath.getAbsolutePath())
+ .option("csv.disable-quote-character", "true")
+ .format(CsvCommons.IDENTIFIER)
+ .schema(schema)
+ .build());
- createTestSinkTable(tableName, fieldNameAndTypes, null, properties);
return sinkPath;
}
+
+ private void createTestValuesSourceTable(
+ String tableName,
+ List<Row> data,
+ Schema schema,
+ Map<String, String> extraProperties,
+ @Nullable String... partitionFields) {
+
+ String dataId = TestValuesTableFactory.registerData(data);
+ Map<String, String> properties = new HashMap<>();
+ properties.put("data-id", dataId);
+ properties.put("bounded", "true");
+ properties.put("disable-lookup", "true");
+ properties.putAll(extraProperties);
+
+ TableDescriptor.Builder descriptor =
+ TableDescriptor.forConnector("values")
+ .schema(schema)
+ .partitionedBy(partitionFields);
+
+ for (Map.Entry<String, String> entry : properties.entrySet()) {
+ descriptor.option(entry.getKey(), entry.getValue());
+ }
+
+ tableEnv.createTemporaryTable(tableName, descriptor.build());
+ }
+
+ private void assertResult(List<String> expected, File resultFile) throws IOException {
+ List<String> actual = readLines(resultFile);
+ assertResult(expected, actual);
+ }
+
+ private void assertResult(List<String> expected, List<String> actual) {
+ Collections.sort(expected);
+ Collections.sort(actual);
+ assertThat(actual).isEqualTo(expected);
+ }
Review comment:
Using assertJ could be reduced to `assertThat(actual).hasSameElementsAs(expected);`
--
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.
To unsubscribe, e-mail: issues-unsubscribe@flink.apache.org
For queries about this service, please contact Infrastructure at:
users@infra.apache.org