You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@shardingsphere.apache.org by zh...@apache.org on 2022/11/16 15:08:49 UTC
[shardingsphere] branch master updated: Refactor : add fileloader support and reafctor the parser code (#22185)
This is an automated email from the ASF dual-hosted git repository.
zhangliang pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/shardingsphere.git
The following commit(s) were added to refs/heads/master by this push:
new 023564386b0 Refactor : add fileloader support and reafctor the parser code (#22185)
023564386b0 is described below
commit 023564386b064248e58812395b10ce6e57c62344
Author: 孙念君 Nianjun Sun <su...@apache.org>
AuthorDate: Wed Nov 16 23:08:42 2022 +0800
Refactor : add fileloader support and reafctor the parser code (#22185)
* Refactor : Add fileloader support and reafctor the parser code
* Refactor : set cosntant for all config keys
* Fix : fix a checkstyle issue
---
.github/workflows/nightly-build.yml | 11 +++-----
.../external/engine/SQLParserParameterizedIT.java | 2 +-
.../env/SQLParserExternalITEnvironment.java | 17 +++++++++---
.../sql/parser/external/loader/SQLCaseLoader.java | 27 ++++++++++---------
.../strategy/impl/GitHubSQLCaseLoadStrategy.java | 7 ++---
.../impl/LocalFileSQLCaseLoadStrategy.java | 16 ++++++++---
.../external/loader/summary/FileSummary.java | 2 +-
.../result/type/csv/CsvSQLParseResultReporter.java | 31 +++++++++-------------
...r.external.result.SQLParseResultReporterCreator | 17 ------------
9 files changed, 62 insertions(+), 68 deletions(-)
diff --git a/.github/workflows/nightly-build.yml b/.github/workflows/nightly-build.yml
index efc9804034c..ab0a12959e3 100644
--- a/.github/workflows/nightly-build.yml
+++ b/.github/workflows/nightly-build.yml
@@ -62,9 +62,10 @@ jobs:
remote_port: ${{ secrets.NIGHTLIES_RSYNC_PORT }}
remote_user: ${{ secrets.NIGHTLIES_RSYNC_USER }}
remote_key: ${{ secrets.NIGHTLIES_RSYNC_KEY }}
-
+
build-proxy-image:
if: github.repository == 'apache/shardingsphere'
+ name: Build Proxy Image
runs-on: ubuntu-latest
permissions:
contents: read
@@ -103,12 +104,8 @@ jobs:
permissions:
contents: read
packages: write
- name: GraalVM Native Image on ${{ matrix.os }}
- runs-on: ${{ matrix.os }}
- strategy:
- matrix:
- os: [ubuntu-latest]
- fail-fast: false
+ name: Build GraalVM Native Image
+ runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
with:
diff --git a/test/parser/src/main/java/org/apache/shardingsphere/test/sql/parser/external/engine/SQLParserParameterizedIT.java b/test/parser/src/main/java/org/apache/shardingsphere/test/sql/parser/external/engine/SQLParserParameterizedIT.java
index b3f84d76b8a..6da9b0e6b6b 100644
--- a/test/parser/src/main/java/org/apache/shardingsphere/test/sql/parser/external/engine/SQLParserParameterizedIT.java
+++ b/test/parser/src/main/java/org/apache/shardingsphere/test/sql/parser/external/engine/SQLParserParameterizedIT.java
@@ -40,7 +40,7 @@ public abstract class SQLParserParameterizedIT {
private final SQLParseResultReporter resultReporter;
- public SQLParserParameterizedIT(final String sqlCaseId, final String sql, final String databaseType, final String reportType) {
+ protected SQLParserParameterizedIT(final String sqlCaseId, final String sql, final String databaseType, final String reportType) {
this.sqlCaseId = sqlCaseId;
this.sql = sql;
this.databaseType = databaseType;
diff --git a/test/parser/src/main/java/org/apache/shardingsphere/test/sql/parser/external/env/SQLParserExternalITEnvironment.java b/test/parser/src/main/java/org/apache/shardingsphere/test/sql/parser/external/env/SQLParserExternalITEnvironment.java
index e03616eb046..8ddf13cf0cd 100644
--- a/test/parser/src/main/java/org/apache/shardingsphere/test/sql/parser/external/env/SQLParserExternalITEnvironment.java
+++ b/test/parser/src/main/java/org/apache/shardingsphere/test/sql/parser/external/env/SQLParserExternalITEnvironment.java
@@ -27,23 +27,32 @@ import java.util.Properties;
/**
* SQL parser external IT environment.
*/
-@Getter
public final class SQLParserExternalITEnvironment {
+ private static final String SQL_PARSER_EXTERNAL_IT_ENABLED_KEY = "sql.parser.external.it.enabled";
+
+ private static final String SQL_PARSER_EXTERNAL_IT_REPORT_PATH = "sql.parser.external.it.report.path";
+
+ private static final String SQL_PARSER_EXTERNAL_IT_REPORT_TYPE = "sql.parser.external.it.report.type";
+
+ @Getter
private static final SQLParserExternalITEnvironment INSTANCE = new SQLParserExternalITEnvironment();
+ @Getter
private final boolean sqlParserITEnabled;
+ @Getter
private final String resultPath;
+ @Getter
private final String resultProcessorType;
private SQLParserExternalITEnvironment() {
Properties props = loadProperties();
sqlParserITEnabled = Boolean.parseBoolean(
- null == System.getProperty("sql.parser.external.it.enabled") ? props.get("sql.parser.external.it.enabled").toString() : System.getProperty("sql.parser.external.it.enabled"));
- resultPath = props.getOrDefault("sql.parser.external.it.report.path", "/tmp/").toString();
- resultProcessorType = props.getOrDefault("sql.parser.external.it.report.type", "LOG").toString();
+ null == System.getProperty(SQL_PARSER_EXTERNAL_IT_ENABLED_KEY) ? props.get(SQL_PARSER_EXTERNAL_IT_ENABLED_KEY).toString() : System.getProperty(SQL_PARSER_EXTERNAL_IT_ENABLED_KEY));
+ resultPath = props.getOrDefault(SQL_PARSER_EXTERNAL_IT_REPORT_PATH, "/tmp/").toString();
+ resultProcessorType = props.getOrDefault(SQL_PARSER_EXTERNAL_IT_REPORT_TYPE, "LOG").toString();
}
/**
diff --git a/test/parser/src/main/java/org/apache/shardingsphere/test/sql/parser/external/loader/SQLCaseLoader.java b/test/parser/src/main/java/org/apache/shardingsphere/test/sql/parser/external/loader/SQLCaseLoader.java
index eeff339a236..6329af3bf19 100644
--- a/test/parser/src/main/java/org/apache/shardingsphere/test/sql/parser/external/loader/SQLCaseLoader.java
+++ b/test/parser/src/main/java/org/apache/shardingsphere/test/sql/parser/external/loader/SQLCaseLoader.java
@@ -60,8 +60,8 @@ public final class SQLCaseLoader {
Map<String, FileSummary> resultFileSummaries = loadStrategy.loadSQLCaseFileSummaries(resultURI).stream().collect(Collectors.toMap(FileSummary::getFileName, v -> v, (k, v) -> v));
for (Entry<String, FileSummary> entry : sqlCaseFileSummaries.entrySet()) {
String fileName = entry.getKey();
- String sqlCaseFileContent = loadContent(URI.create(entry.getValue().getAccessURL()));
- String resultFileContent = resultFileSummaries.containsKey(fileName) ? loadContent(URI.create(resultFileSummaries.get(fileName).getAccessURL())) : "";
+ String sqlCaseFileContent = loadContent(URI.create(entry.getValue().getAccessURI()));
+ String resultFileContent = resultFileSummaries.containsKey(fileName) ? loadContent(URI.create(resultFileSummaries.get(fileName).getAccessURI())) : "";
result.addAll(createSQLCases(fileName, sqlCaseFileContent, resultFileContent));
}
if (result.isEmpty()) {
@@ -71,9 +71,10 @@ public final class SQLCaseLoader {
}
private String loadContent(final URI uri) {
- try {
- InputStreamReader in = new InputStreamReader(uri.toURL().openStream());
- return new BufferedReader(in).lines().collect(Collectors.joining(System.lineSeparator()));
+ try (
+ InputStreamReader in = new InputStreamReader(uri.toURL().openStream());
+ BufferedReader reader = new BufferedReader(in)) {
+ return reader.lines().collect(Collectors.joining(System.lineSeparator()));
} catch (final IOException ex) {
log.warn("Load failed, reason is: ", ex);
return "";
@@ -82,20 +83,20 @@ public final class SQLCaseLoader {
private Collection<Object[]> createSQLCases(final String sqlCaseFileName, final String sqlCaseFileContent, final String resultFileContent) {
Collection<Object[]> result = new LinkedList<>();
- String[] caseCaseLines = sqlCaseFileContent.split("\n");
- String[] resultLines = resultFileContent.split("\n");
+ String[] rawsCaseLines = sqlCaseFileContent.split("\n");
+ String[] rawResultLines = resultFileContent.split("\n");
String completedSQL = "";
int sqlCaseEnum = 1;
int statementLines = 0;
int resultIndex = 0;
boolean inProcedure = false;
- for (String each : caseCaseLines) {
+ for (String each : rawsCaseLines) {
inProcedure = isInProcedure(inProcedure, each.trim());
completedSQL = getStatement(completedSQL, each.trim(), inProcedure);
statementLines = completedSQL.isEmpty() ? 0 : statementLines + 1;
if (completedSQL.contains(";") && !inProcedure) {
- resultIndex = searchResult(resultIndex, resultLines, completedSQL, statementLines);
- if (resultIndex >= resultLines.length || !resultLines[resultIndex].contains("ERROR")) {
+ resultIndex = searchInResultContent(resultIndex, rawResultLines, completedSQL, statementLines);
+ if (resultIndex >= rawResultLines.length || !rawResultLines[resultIndex].contains("ERROR")) {
String sqlCaseId = sqlCaseFileName + sqlCaseEnum;
result.add(new Object[]{sqlCaseId, completedSQL});
sqlCaseEnum++;
@@ -114,15 +115,15 @@ public final class SQLCaseLoader {
return (statementLines.contains("{") || statementLines.contains("}") || statementLines.contains("$$")) != inProcedure;
}
- private static String getStatement(final String completedSQL, final String sqlLine, final boolean inProcedure) {
- return (sqlLine.isEmpty() || isComment(sqlLine)) && !inProcedure ? "" : completedSQL + sqlLine + " ";
+ private static String getStatement(final String completedSQL, final String rawSQLLine, final boolean inProcedure) {
+ return (rawSQLLine.isEmpty() || isComment(rawSQLLine)) && !inProcedure ? "" : completedSQL + rawSQLLine + " ";
}
private static boolean isComment(final String statement) {
return statement.startsWith("#") || statement.startsWith("/") || statement.startsWith("--") || statement.startsWith(":") || statement.startsWith("\\");
}
- private static int searchResult(final int resultIndex, final String[] resultLines, final String completedSQL, final int statementLines) {
+ private static int searchInResultContent(final int resultIndex, final String[] resultLines, final String completedSQL, final int statementLines) {
int index = resultIndex;
while (index < resultLines.length && !completedSQL.startsWith(resultLines[index].trim())) {
index++;
diff --git a/test/parser/src/main/java/org/apache/shardingsphere/test/sql/parser/external/loader/strategy/impl/GitHubSQLCaseLoadStrategy.java b/test/parser/src/main/java/org/apache/shardingsphere/test/sql/parser/external/loader/strategy/impl/GitHubSQLCaseLoadStrategy.java
index d693396b985..c47bcd4a117 100644
--- a/test/parser/src/main/java/org/apache/shardingsphere/test/sql/parser/external/loader/strategy/impl/GitHubSQLCaseLoadStrategy.java
+++ b/test/parser/src/main/java/org/apache/shardingsphere/test/sql/parser/external/loader/strategy/impl/GitHubSQLCaseLoadStrategy.java
@@ -73,9 +73,10 @@ public final class GitHubSQLCaseLoadStrategy implements SQLCaseLoadStrategy {
}
private String loadContent(final URI casesURI) {
- try {
- InputStreamReader in = new InputStreamReader(casesURI.toURL().openStream());
- return new BufferedReader(in).lines().collect(Collectors.joining(System.lineSeparator()));
+ try (
+ InputStreamReader in = new InputStreamReader(casesURI.toURL().openStream());
+ BufferedReader reader = new BufferedReader(in)) {
+ return reader.lines().collect(Collectors.joining(System.lineSeparator()));
} catch (final IOException ex) {
log.warn("Load failed, reason is: ", ex);
return "";
diff --git a/test/parser/src/main/java/org/apache/shardingsphere/test/sql/parser/external/loader/strategy/impl/LocalFileSQLCaseLoadStrategy.java b/test/parser/src/main/java/org/apache/shardingsphere/test/sql/parser/external/loader/strategy/impl/LocalFileSQLCaseLoadStrategy.java
index 8d75c68d58a..5839629b481 100644
--- a/test/parser/src/main/java/org/apache/shardingsphere/test/sql/parser/external/loader/strategy/impl/LocalFileSQLCaseLoadStrategy.java
+++ b/test/parser/src/main/java/org/apache/shardingsphere/test/sql/parser/external/loader/strategy/impl/LocalFileSQLCaseLoadStrategy.java
@@ -17,21 +17,31 @@
package org.apache.shardingsphere.test.sql.parser.external.loader.strategy.impl;
+import lombok.SneakyThrows;
import org.apache.shardingsphere.test.sql.parser.external.loader.strategy.SQLCaseLoadStrategy;
import org.apache.shardingsphere.test.sql.parser.external.loader.summary.FileSummary;
import java.net.URI;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.Paths;
import java.util.Collection;
-import java.util.Collections;
+import java.util.LinkedList;
+import java.util.stream.Stream;
/**
* SQL case loader with local file.
*/
public final class LocalFileSQLCaseLoadStrategy implements SQLCaseLoadStrategy {
+ @SneakyThrows
@Override
public Collection<FileSummary> loadSQLCaseFileSummaries(final URI uri) {
- // TODO
- return Collections.emptyList();
+ final Collection<FileSummary> result = new LinkedList<>();
+ try (Stream<Path> stream = Files.walk(Paths.get(uri))) {
+ stream.filter(each -> each.toString().endsWith(".sql"))
+ .forEach(each -> result.add(new FileSummary(each.getFileName().toString(), each.toUri().toString())));
+ }
+ return result;
}
}
diff --git a/test/parser/src/main/java/org/apache/shardingsphere/test/sql/parser/external/loader/summary/FileSummary.java b/test/parser/src/main/java/org/apache/shardingsphere/test/sql/parser/external/loader/summary/FileSummary.java
index f71e6abcf08..1dcbfaf825e 100644
--- a/test/parser/src/main/java/org/apache/shardingsphere/test/sql/parser/external/loader/summary/FileSummary.java
+++ b/test/parser/src/main/java/org/apache/shardingsphere/test/sql/parser/external/loader/summary/FileSummary.java
@@ -29,5 +29,5 @@ public final class FileSummary {
private final String fileName;
- private final String accessURL;
+ private final String accessURI;
}
diff --git a/test/parser/src/main/java/org/apache/shardingsphere/test/sql/parser/external/result/type/csv/CsvSQLParseResultReporter.java b/test/parser/src/main/java/org/apache/shardingsphere/test/sql/parser/external/result/type/csv/CsvSQLParseResultReporter.java
index cec7b6e1268..b1d62ab86d1 100644
--- a/test/parser/src/main/java/org/apache/shardingsphere/test/sql/parser/external/result/type/csv/CsvSQLParseResultReporter.java
+++ b/test/parser/src/main/java/org/apache/shardingsphere/test/sql/parser/external/result/type/csv/CsvSQLParseResultReporter.java
@@ -17,6 +17,7 @@
package org.apache.shardingsphere.test.sql.parser.external.result.type.csv;
+import lombok.SneakyThrows;
import org.apache.commons.csv.CSVFormat;
import org.apache.commons.csv.CSVPrinter;
import org.apache.shardingsphere.test.sql.parser.external.env.SQLParserExternalITEnvironment;
@@ -24,7 +25,6 @@ import org.apache.shardingsphere.test.sql.parser.external.result.SQLParseResultR
import java.io.File;
import java.io.FileWriter;
-import java.io.IOException;
/**
* SQL parse result reporter for CSV.
@@ -33,35 +33,28 @@ public final class CsvSQLParseResultReporter implements SQLParseResultReporter {
private final CSVPrinter printer;
+ @SneakyThrows
public CsvSQLParseResultReporter(final String databaseType) {
- try {
- File csvFile = new File(SQLParserExternalITEnvironment.getInstance().getResultPath() + databaseType + "-result.csv");
- printHeader(csvFile);
- printer = new CSVPrinter(new FileWriter(csvFile, true), CSVFormat.DEFAULT.builder().setSkipHeaderRecord(true).build());
- } catch (final IOException ex) {
- throw new RuntimeException("Create CSV file failed.", ex);
- }
+ File csvFile = new File(SQLParserExternalITEnvironment.getInstance().getResultPath() + databaseType + "-result.csv");
+ printHeader(csvFile);
+ printer = new CSVPrinter(new FileWriter(csvFile, true), CSVFormat.DEFAULT.builder().setSkipHeaderRecord(true).build());
}
+ @SneakyThrows
private void printHeader(final File csvFile) {
if (csvFile.exists()) {
return;
}
- try (CSVPrinter printer = new CSVPrinter(new FileWriter(csvFile), CSVFormat.DEFAULT.builder().setSkipHeaderRecord(false).build())) {
- printer.printRecord("SQLCaseId", "DatabaseType", "Result", "SQL");
- printer.flush();
- } catch (final IOException ex) {
- throw new RuntimeException("Create CSV file header failed.", ex);
+ try (CSVPrinter csvHeaderPrinter = new CSVPrinter(new FileWriter(csvFile), CSVFormat.DEFAULT.builder().setSkipHeaderRecord(false).build())) {
+ csvHeaderPrinter.printRecord("SQLCaseId", "DatabaseType", "Result", "SQL");
+ csvHeaderPrinter.flush();
}
}
+ @SneakyThrows
@Override
public void printResult(final String sqlCaseId, final String databaseType, final boolean isSuccess, final String sql) {
- try {
- printer.printRecord(sqlCaseId, databaseType, isSuccess ? "success" : "failed", sql);
- printer.flush();
- } catch (final IOException ex) {
- throw new RuntimeException("Write CSV file failed.", ex);
- }
+ printer.printRecord(sqlCaseId, databaseType, isSuccess ? "success" : "failed", sql);
+ printer.flush();
}
}
diff --git a/test/parser/src/main/resources/META-INF/services/org.apache.shardingsphere.test.sql.parser.external.result.SQLParseResultReporterCreator b/test/parser/src/main/resources/META-INF/services/org.apache.shardingsphere.test.sql.parser.external.result.SQLParseResultReporterCreator
index b62d4ba1f4f..11854e3314a 100644
--- a/test/parser/src/main/resources/META-INF/services/org.apache.shardingsphere.test.sql.parser.external.result.SQLParseResultReporterCreator
+++ b/test/parser/src/main/resources/META-INF/services/org.apache.shardingsphere.test.sql.parser.external.result.SQLParseResultReporterCreator
@@ -15,22 +15,5 @@
# limitations under the License.
#
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
org.apache.shardingsphere.test.sql.parser.external.result.type.log.LogSQLParseResultReporterCreator
org.apache.shardingsphere.test.sql.parser.external.result.type.csv.CsvSQLParseResultReporterCreator