You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@asterixdb.apache.org by AsterixDB Code Review <do...@asterix-gerrit.ics.uci.edu> on 2021/06/14 20:34:42 UTC
Change in asterixdb[master]: [ASTERIXDB-2915][SQL] Fix UDF parsing issues
From Cameron Samak <cs...@apache.org>:
Cameron Samak has uploaded this change for review. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/11943 )
Change subject: [ASTERIXDB-2915][SQL] Fix UDF parsing issues
......................................................................
[ASTERIXDB-2915][SQL] Fix UDF parsing issues
- user model changes: no
- storage format changes: no
- interface changes: yes
Details:
SqlppParserFactory.createParser(Reader) created parsers with an invalid
state. Only UDF code used this method and many queries still parse fine,
so tests didn't catch issues.
It's still possible to create a parser without a ScopeChecker, but now
that requires bypassing the ParserFactory and even then it will fail
on nearly all queries (so we can't accidentally do this again).
ScopeChecker currently also serves as a parser util class and should be
broken up (left for a future change).
Change-Id: Iea8b66d064386124725de450e0faf7cb6dc0fbcc
---
M asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/base/IParserFactory.java
M asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/parser/ScopeChecker.java
M asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/FunctionUtil.java
M asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/parser/SqlppParserFactory.java
M asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj
M asterixdb/asterix-tools/src/test/java/org/apache/asterix/tools/datagen/AdgClientDriver.java
M asterixdb/asterix-tools/src/test/java/org/apache/asterix/tools/datagen/AdmDataGen.java
7 files changed, 77 insertions(+), 87 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/43/11943/1
diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/base/IParserFactory.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/base/IParserFactory.java
index 1775fbb..76c78a88 100644
--- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/base/IParserFactory.java
+++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/base/IParserFactory.java
@@ -18,13 +18,9 @@
*/
package org.apache.asterix.lang.common.base;
-import java.io.Reader;
-
public interface IParserFactory {
IParser createParser(String query);
- IParser createParser(Reader reader);
-
String getLanguage();
}
diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/parser/ScopeChecker.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/parser/ScopeChecker.java
index f5aa489..dbd3c67 100644
--- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/parser/ScopeChecker.java
+++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/parser/ScopeChecker.java
@@ -31,24 +31,21 @@
public class ScopeChecker {
- protected static String quot = "\"";
-
- protected String eol = System.getProperty("line.separator", "\n");
-
protected Counter varCounter = new Counter(-1);
- protected Stack<Scope> scopeStack = new Stack<>();
-
- protected Stack<Scope> forbiddenScopeStack = new Stack<>();
-
- protected String[] inputLines;
+ private static final String EOL = System.getProperty("line.separator", "\n");
+ private final Stack<Scope> scopeStack = new Stack<>();
+ private final Stack<Scope> forbiddenScopeStack = new Stack<>();
+ private String[] inputLines;
public ScopeChecker() {
scopeStack.push(RootScopeFactory.createRootScope(this));
}
- protected void setInput(String s) {
- inputLines = s.split("\n|\r\n?");
+ // TODO: Methods requiring the input String have nothing to do with scope checking. Split this class.
+ public ScopeChecker(String input) {
+ scopeStack.push(RootScopeFactory.createRootScope(this));
+ inputLines = input.split("\n|\r\n?");
}
// Forbidden scopes are used to disallow, in a limit clause, variables
@@ -77,7 +74,7 @@
return scope;
}
- protected final Scope extendCurrentScopeNoPush(boolean maskParentScope) {
+ public final Scope extendCurrentScopeNoPush(boolean maskParentScope) {
Scope parent = scopeStack.peek();
return new Scope(this, parent, maskParentScope);
}
@@ -175,7 +172,7 @@
return false;
}
- protected int appendExpected(StringBuilder expected, int[][] expectedTokenSequences, String[] tokenImage) {
+ public int appendExpected(StringBuilder expected, int[][] expectedTokenSequences, String[] tokenImage) {
int maxSize = 0;
for (int i = 0; i < expectedTokenSequences.length; i++) {
if (maxSize < expectedTokenSequences[i].length) {
@@ -188,7 +185,7 @@
if (expectedTokenSequences[i][expectedTokenSequences[i].length - 1] != 0) {
append(expected, "...");
}
- append(expected, eol);
+ append(expected, EOL);
append(expected, " ");
}
return maxSize;
@@ -200,17 +197,17 @@
}
}
- protected static String fixQuotes(String token) {
+ public static String fixQuotes(String token) {
final String stripped = stripQuotes(token);
return stripped != null ? "'" + stripped + "'" : token;
}
- protected static String stripQuotes(String token) {
+ public static String stripQuotes(String token) {
final int last = token.length() - 1;
return token.charAt(0) == '"' && token.charAt(last) == '"' ? token.substring(1, last) : null;
}
- protected static String addEscapes(String str) {
+ public static String addEscapes(String str) {
StringBuilder escaped = new StringBuilder();
for (int i = 0; i < str.length(); i++) {
appendChar(escaped, str.charAt(i));
@@ -305,12 +302,12 @@
return res.toString();
}
- protected String getLine(int line) {
+ public String getLine(int line) {
int idx = line - 1;
return idx >= 0 && idx < inputLines.length ? inputLines[idx] : "";
}
- protected String extractFragment(int beginLine, int beginColumn, int endLine, int endColumn) {
+ public String extractFragment(int beginLine, int beginColumn, int endLine, int endColumn) {
StringBuilder extract = new StringBuilder();
if (beginLine == endLine) {
// special case that we need to handle separately
diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/FunctionUtil.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/FunctionUtil.java
index ded91af..05dfe12 100644
--- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/FunctionUtil.java
+++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/FunctionUtil.java
@@ -19,7 +19,6 @@
package org.apache.asterix.lang.common.util;
-import java.io.StringReader;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
@@ -366,7 +365,7 @@
throw new CompilationException(ErrorCode.COMPILATION_INCOMPATIBLE_FUNCTION_LANGUAGE, sourceLoc,
function.getLanguage(), function.getSignature().toString(), parserFactory.getLanguage());
}
- IParser parser = parserFactory.createParser(new StringReader(function.getFunctionBody()));
+ IParser parser = parserFactory.createParser(function.getFunctionBody());
try {
FunctionDecl functionDecl =
parser.parseFunctionBody(function.getSignature(), function.getParameterNames(), true);
diff --git a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/parser/SqlppParserFactory.java b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/parser/SqlppParserFactory.java
index 5b724f2..3aec49b 100644
--- a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/parser/SqlppParserFactory.java
+++ b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/parser/SqlppParserFactory.java
@@ -18,10 +18,9 @@
*/
package org.apache.asterix.lang.sqlpp.parser;
-import java.io.Reader;
-
import org.apache.asterix.lang.common.base.IParser;
import org.apache.asterix.lang.common.base.IParserFactory;
+import org.apache.asterix.lang.common.parser.ScopeChecker;
public class SqlppParserFactory implements IParserFactory {
@@ -30,12 +29,7 @@
@Override
public IParser createParser(String query) {
- return new SQLPPParser(query);
- }
-
- @Override
- public IParser createParser(Reader reader) {
- return new SQLPPParser(reader);
+ return new SQLPPParser(query, new ScopeChecker(query));
}
@Override
diff --git a/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj b/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj
index c4c86f4..a895bc9 100644
--- a/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj
+++ b/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj
@@ -216,7 +216,7 @@
import org.apache.hyracks.util.LogRedactionUtil;
import org.apache.hyracks.util.StringUtil;
-class SQLPPParser extends ScopeChecker implements IParser {
+class SQLPPParser implements IParser {
// tokens parsed as identifiers
private static final String CUBE = "CUBE";
@@ -246,6 +246,8 @@
private static final String RETURNS = "RETURNS";
private static final String CONFIG = "CONFIG";
+ private static final String QUOT = "\"";
+ private static final String EOL = System.getProperty("line.separator", "\n");
private static final String INT_TYPE_NAME = "int";
private static final String UDF_VARARGS_PARAM_NAME = "args"; // Note: this value is stored in the function metadata
@@ -263,6 +265,9 @@
private final Map<SourceLocation, String> hintCollector = new HashMap<SourceLocation, String>();
+ // This cannot be final and unassigned since generated constructors do not assign it.
+ private ScopeChecker scopeChecker = null;
+
private static class IndexParams {
public IndexType type;
public int gramLength;
@@ -333,9 +338,9 @@
}
}
- public SQLPPParser(String s) {
+ public SQLPPParser(String s, ScopeChecker scopeChecker) {
this(new StringReader(s));
- super.setInput(s);
+ this.scopeChecker = scopeChecker;
}
public static void main(String args[]) throws ParseException, TokenMgrError, IOException, FileNotFoundException, CompilationException {
@@ -367,7 +372,7 @@
}
private static Expression parseExpression(String text) throws CompilationException {
- return new SQLPPParser(text).parseExpression();
+ return new SQLPPParser(text, new ScopeChecker(text)).parseExpression();
}
@Override
@@ -390,7 +395,7 @@
}
private static List<String> parseParenthesizedIdentifierList(String text) throws CompilationException {
- return new SQLPPParser(text).parseParenthesizedIdentifierList();
+ return new SQLPPParser(text, new ScopeChecker(text)).parseParenthesizedIdentifierList();
}
@Override
@@ -401,13 +406,13 @@
public FunctionDecl parse() throws ParseException {
DataverseName dataverse = defaultDataverse;
defaultDataverse = signature.getDataverseName();
- createNewScope();
+ scopeChecker.createNewScope();
List<VarIdentifier> paramVars = new ArrayList<VarIdentifier>(paramNames.size());
for (String paramName : paramNames) {
paramVars.add(SqlppVariableUtil.toInternalVariableIdentifier(paramName));
}
Expression functionBodyExpr = SQLPPParser.this.FunctionBody();
- removeCurrentScope();
+ scopeChecker.removeCurrentScope();
defaultDataverse = dataverse;
return new FunctionDecl(signature, paramVars, functionBodyExpr, isStored);
}
@@ -469,26 +474,26 @@
}
int[][] expectedTokenSequences = pe.expectedTokenSequences;
String[] tokenImage = pe.tokenImage;
- String sep = REPORT_EXPECTED_TOKENS ? eol : " ";
+ String sep = REPORT_EXPECTED_TOKENS ? EOL : " ";
StringBuilder expected = REPORT_EXPECTED_TOKENS ? new StringBuilder() : null;
- int maxSize = appendExpected(expected, expectedTokenSequences, tokenImage);
+ int maxSize = scopeChecker.appendExpected(expected, expectedTokenSequences, tokenImage);
Token tok = currentToken.next;
int line = tok.beginLine;
StringBuilder message = new StringBuilder(128);
- message.append("In line ").append(line).append(" >>").append(getLine(line)).append("<<").append(sep).append("Encountered ");
+ message.append("In line ").append(line).append(" >>").append(scopeChecker.getLine(line)).append("<<").append(sep).append("Encountered ");
for (int i = 0; i < maxSize; i++) {
if (i != 0) {
message.append(' ');
}
if (tok.kind == 0) {
- message.append(fixQuotes(tokenImage[0]));
+ message.append(scopeChecker.fixQuotes(tokenImage[0]));
break;
}
final String fixedTokenImage = tokenImage[tok.kind];
- if (! tok.image.equalsIgnoreCase(stripQuotes(fixedTokenImage))) {
- message.append(fixQuotes(fixedTokenImage)).append(' ');
+ if (! tok.image.equalsIgnoreCase(scopeChecker.stripQuotes(fixedTokenImage))) {
+ message.append(scopeChecker.fixQuotes(fixedTokenImage)).append(' ');
}
- message.append(quot).append(addEscapes(tok.image)).append(quot);
+ message.append(QUOT).append(scopeChecker.addEscapes(tok.image)).append(QUOT);
tok = tok.next;
}
message.append(" at column ").append(currentToken.next.beginColumn).append('.').append(sep);
@@ -695,7 +700,6 @@
List<Statement> Statement() throws ParseException:
{
- scopeStack.push(RootScopeFactory.createRootScope(this));
List<Statement> decls = new ArrayList<Statement>();
Statement stmt = null;
}
@@ -1427,16 +1431,16 @@
<LEFTBRACE>
{
beginPos = token;
- createNewScope();
+ scopeChecker.createNewScope();
}
functionBodyExpr = FunctionBody()
<RIGHTBRACE>
{
endPos = token;
- String functionBody = extractFragment(beginPos.beginLine, beginPos.beginColumn, endPos.beginLine,
+ String functionBody = scopeChecker.extractFragment(beginPos.beginLine, beginPos.beginColumn, endPos.beginLine,
endPos.beginColumn);
- getCurrentScope().addFunctionDescriptor(signature, false);
- removeCurrentScope();
+ scopeChecker.getCurrentScope().addFunctionDescriptor(signature, false);
+ scopeChecker.removeCurrentScope();
ensureNoTypeDeclsInFunction(fctName.function, params, returnType, startStmtToken);
stmt = new CreateFunctionStatement(signature, params, functionBody, functionBodyExpr, orReplace, ifNotExists);
}
@@ -2175,7 +2179,7 @@
}
query.setTopLevel(true);
InsertStatement stmt = new InsertStatement(nameComponents.first, nameComponents.second.getValue(), query,
- getVarCounter(), var, returnExpression);
+ scopeChecker.getVarCounter(), var, returnExpression);
return addSourceLocation(stmt, startToken);
}
}
@@ -2199,7 +2203,7 @@
}
query.setTopLevel(true);
UpsertStatement stmt = new UpsertStatement(nameComponents.first, nameComponents.second.getValue(), query,
- getVarCounter(), var, returnExpression);
+ scopeChecker.getVarCounter(), var, returnExpression);
return addSourceLocation(stmt, startToken);
}
}
@@ -2221,7 +2225,7 @@
addSourceLocation(var, startToken);
}
DeleteStatement stmt = new DeleteStatement(var, nameComponents.first, nameComponents.second.getValue(),
- condition, getVarCounter());
+ condition, scopeChecker.getVarCounter());
return addSourceLocation(stmt, startToken);
}
}
@@ -2460,12 +2464,12 @@
{
if (whereClause != null) {
endPos = token;
- whereClauseBody = extractFragment(beginPos.endLine, beginPos.endColumn, endPos.endLine, endPos.endColumn + 1);
+ whereClauseBody = scopeChecker.extractFragment(beginPos.endLine, beginPos.endColumn, endPos.endLine, endPos.endColumn + 1);
}
}
{
stmt = new ConnectFeedStatement(feedNameComponents, datasetNameComponents, appliedFunctions,
- policy, whereClauseBody, getVarCounter());
+ policy, whereClauseBody, scopeChecker.getVarCounter());
}
)
{
@@ -2868,7 +2872,7 @@
{
<QUOTED_STRING>
{
- return removeQuotesAndEscapes(token.image);
+ return scopeChecker.removeQuotesAndEscapes(token.image);
}
}
@@ -2878,7 +2882,7 @@
{
<STRING_LITERAL>
{
- return removeQuotesAndEscapes(token.image);
+ return scopeChecker.removeQuotesAndEscapes(token.image);
}
}
@@ -2969,7 +2973,7 @@
String functionName;
Pair<Integer, List<Pair<VarIdentifier,TypeExpression>>> paramsWithArity = null;
Expression funcBody;
- createNewScope();
+ scopeChecker.createNewScope();
}
{
<DECLARE> { startToken = token; } <FUNCTION>
@@ -2982,14 +2986,14 @@
int arity = paramsWithArity.first;
List<Pair<VarIdentifier,TypeExpression>> paramList = paramsWithArity.second;
FunctionSignature signature = new FunctionSignature(defaultDataverse, functionName, arity);
- getCurrentScope().addFunctionDescriptor(signature, false);
+ scopeChecker.getCurrentScope().addFunctionDescriptor(signature, false);
ensureNoTypeDeclsInFunction(functionName, paramList, null, startToken);
List<VarIdentifier> params = new ArrayList<VarIdentifier>(paramList.size());
for (Pair<VarIdentifier,TypeExpression> p: paramList) {
params.add(p.getFirst());
}
FunctionDecl stmt = new FunctionDecl(signature, params, funcBody, false);
- removeCurrentScope();
+ scopeChecker.removeCurrentScope();
return addSourceLocation(stmt, startToken);
}
}
@@ -3607,8 +3611,8 @@
{
id = VariableIdentifier()
{
- Identifier ident = lookupSymbol(id);
- if (isInForbiddenScopes(id)) {
+ Identifier ident = scopeChecker.lookupSymbol(id);
+ if (scopeChecker.isInForbiddenScopes(id)) {
throw new SqlppParseException(getSourceLocation(token),
"Inside limit clauses, it is disallowed to reference a variable having the same name as any variable bound in the same scope as the limit clause.");
}
@@ -3630,7 +3634,7 @@
{
id = VariableIdentifier()
{
- Identifier ident = lookupSymbol(id);
+ Identifier ident = scopeChecker.lookupSymbol(id);
VariableExpr varExp = new VariableExpr(new VarIdentifier(id));
if (ident != null) { // exist such ident
varExp.setIsNewVar(false);
@@ -3688,7 +3692,7 @@
(
<DOLLAR_IDENTIFIER> { name = token.image.substring(1); }
| <DOLLAR_INTEGER_LITERAL> { name = token.image.substring(1); }
- | <DOLLAR_QUOTED_STRING> { name = removeQuotesAndEscapes(token.image.substring(1)); }
+ | <DOLLAR_QUOTED_STRING> { name = scopeChecker.removeQuotesAndEscapes(token.image.substring(1)); }
| <QUES> { name = String.valueOf(++externalVarCounter); }
)
{
@@ -3845,7 +3849,7 @@
}
String fqFunctionName = funcName.library == null ? name : funcName.library + "#" + name;
int arity = argList.size();
- FunctionSignature signature = lookupFunctionSignature(funcName.dataverse, fqFunctionName, arity);
+ FunctionSignature signature = scopeChecker.lookupFunctionSignature(funcName.dataverse, fqFunctionName, arity);
if (signature == null) {
signature = new FunctionSignature(funcName.dataverse, fqFunctionName, arity);
}
@@ -4134,7 +4138,7 @@
SelectSetOperation selectSetOperation;
OrderbyClause orderbyClause = null;
LimitClause limitClause = null;
- createNewScope();
+ scopeChecker.createNewScope();
}
{
( letClauses = LetClause() )?
@@ -4383,7 +4387,7 @@
{
Token startToken = null;
List<FromTerm> fromTerms = new ArrayList<FromTerm>();
- extendCurrentScope();
+ scopeChecker.extendCurrentScope();
}
{
{
@@ -4582,8 +4586,7 @@
}
{
{
- Scope newScope = extendCurrentScopeNoPush(true);
- // extendCurrentScope(true);
+ Scope newScope = scopeChecker.extendCurrentScopeNoPush(true);
}
<GROUP>
{
@@ -4616,7 +4619,7 @@
gbc.setWithVarMap(new HashMap<Expression, VariableExpr>());
gbc.setGroupVar(groupVar);
gbc.setGroupFieldList(groupFieldList);
- replaceCurrentScope(newScope);
+ scopeChecker.replaceCurrentScope(newScope);
return addSourceLocation(gbc, startToken);
}
}
@@ -4750,14 +4753,14 @@
{
(
(
- <LIMIT> { startToken = token; pushForbiddenScope(getCurrentScope()); } limitExpr = Expression()
+ <LIMIT> { startToken = token; scopeChecker.pushForbiddenScope(scopeChecker.getCurrentScope()); } limitExpr = Expression()
( <OFFSET> offsetExpr = Expression() )?
- { popForbiddenScope(); }
+ { scopeChecker.popForbiddenScope(); }
)
|
(
- <OFFSET> { startToken = token; pushForbiddenScope(getCurrentScope()); } offsetExpr = Expression()
- { popForbiddenScope(); }
+ <OFFSET> { startToken = token; scopeChecker.pushForbiddenScope(scopeChecker.getCurrentScope()); } offsetExpr = Expression()
+ { scopeChecker.popForbiddenScope(); }
)
)
{
@@ -4778,7 +4781,7 @@
}
{
{
- createNewScope();
+ scopeChecker.createNewScope();
}
( ((<ANY>|<SOME>) { startToken = token; qc.setQuantifier(QuantifiedExpression.Quantifier.SOME); })
@@ -4799,7 +4802,7 @@
{
qc.setSatisfiesExpr(satisfiesExpr);
qc.setQuantifiedList(quantifiedList);
- removeCurrentScope();
+ scopeChecker.removeCurrentScope();
return addSourceLocation(qc, startToken);
}
}
@@ -4808,7 +4811,7 @@
{
VariableExpr varExp;
Expression beExp;
- extendCurrentScope();
+ scopeChecker.extendCurrentScope();
}
{
varExp = Variable() <EQ> beExp = Expression()
@@ -4823,7 +4826,7 @@
{
VariableExpr varExp;
Expression beExp;
- extendCurrentScope();
+ scopeChecker.extendCurrentScope();
}
{
varExp = Variable() <AS> beExp = Expression()
diff --git a/asterixdb/asterix-tools/src/test/java/org/apache/asterix/tools/datagen/AdgClientDriver.java b/asterixdb/asterix-tools/src/test/java/org/apache/asterix/tools/datagen/AdgClientDriver.java
index 8a1adb0..296d632 100644
--- a/asterixdb/asterix-tools/src/test/java/org/apache/asterix/tools/datagen/AdgClientDriver.java
+++ b/asterixdb/asterix-tools/src/test/java/org/apache/asterix/tools/datagen/AdgClientDriver.java
@@ -19,6 +19,7 @@
package org.apache.asterix.tools.datagen;
import java.io.File;
+import java.nio.file.Path;
import org.kohsuke.args4j.Argument;
import org.kohsuke.args4j.CmdLineParser;
@@ -30,7 +31,7 @@
public static class AdgClientConfig {
@Argument(index = 0, required = true, metaVar = "ARG1", usage = "The file containing the annotated schema.")
- private File schemaFile;
+ private Path schemaFile;
@Argument(index = 1, required = true, metaVar = "ARG2", usage = "The output directory path.")
private File outputDir;
diff --git a/asterixdb/asterix-tools/src/test/java/org/apache/asterix/tools/datagen/AdmDataGen.java b/asterixdb/asterix-tools/src/test/java/org/apache/asterix/tools/datagen/AdmDataGen.java
index aa86916..fb02243 100644
--- a/asterixdb/asterix-tools/src/test/java/org/apache/asterix/tools/datagen/AdmDataGen.java
+++ b/asterixdb/asterix-tools/src/test/java/org/apache/asterix/tools/datagen/AdmDataGen.java
@@ -21,9 +21,11 @@
import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileOutputStream;
-import java.io.FileReader;
import java.io.IOException;
import java.io.PrintStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -922,23 +924,21 @@
}
- private final File schemaFile;
+ private final Path schemaFile;
private final File outputDir;
private Map<TypeSignature, IAType> typeMap;
private Map<TypeSignature, TypeDataGen> typeAnnotMap;
private DataGeneratorContext dgCtx;
private final IParserFactory parserFactory = new SqlppParserFactory();
- public AdmDataGen(File schemaFile, File outputDir) {
+ public AdmDataGen(Path schemaFile, File outputDir) {
this.schemaFile = schemaFile;
this.outputDir = outputDir;
}
public void init() throws IOException, ACIDException, AlgebricksException {
- FileReader aql = new FileReader(schemaFile);
- IParser parser = parserFactory.createParser(aql);
+ IParser parser = parserFactory.createParser(Files.readString(schemaFile, StandardCharsets.UTF_8));
List<Statement> statements = parser.parse();
- aql.close();
// TODO: Need to fix how to use transactions here.
MetadataTransactionContext mdTxnCtx = new MetadataTransactionContext(new TxnId(-1));
ADGenDmlTranslator dmlt = new ADGenDmlTranslator(mdTxnCtx, statements);
--
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/11943
To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Change-Id: Iea8b66d064386124725de450e0faf7cb6dc0fbcc
Gerrit-Change-Number: 11943
Gerrit-PatchSet: 1
Gerrit-Owner: Cameron Samak <cs...@apache.org>
Gerrit-MessageType: newchange
Change in asterixdb[master]: [ASTERIXDB-2915][SQL] Fix UDF parsing issues
Posted by AsterixDB Code Review <do...@asterix-gerrit.ics.uci.edu>.
From Cameron Samak <cs...@apache.org>:
Cameron Samak has uploaded this change for review. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/11943 )
Change subject: [ASTERIXDB-2915][SQL] Fix UDF parsing issues
......................................................................
[ASTERIXDB-2915][SQL] Fix UDF parsing issues
- user model changes: no
- storage format changes: no
- interface changes: yes
Details:
SqlppParserFactory.createParser(Reader) created parsers with an invalid
state. Only UDF code used this method and many queries still parse fine,
so tests didn't catch issues.
It's still possible to create a parser without a ScopeChecker, but now
that requires bypassing the ParserFactory and even then it will fail
on nearly all queries (so we can't accidentally do this again).
ScopeChecker currently also serves as a parser util class and should be
broken up (left for a future change).
Change-Id: Iea8b66d064386124725de450e0faf7cb6dc0fbcc
---
M asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/base/IParserFactory.java
M asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/parser/ScopeChecker.java
M asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/FunctionUtil.java
M asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/parser/SqlppParserFactory.java
M asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj
M asterixdb/asterix-tools/src/test/java/org/apache/asterix/tools/datagen/AdgClientDriver.java
M asterixdb/asterix-tools/src/test/java/org/apache/asterix/tools/datagen/AdmDataGen.java
7 files changed, 77 insertions(+), 87 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/43/11943/1
diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/base/IParserFactory.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/base/IParserFactory.java
index 1775fbb..76c78a88 100644
--- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/base/IParserFactory.java
+++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/base/IParserFactory.java
@@ -18,13 +18,9 @@
*/
package org.apache.asterix.lang.common.base;
-import java.io.Reader;
-
public interface IParserFactory {
IParser createParser(String query);
- IParser createParser(Reader reader);
-
String getLanguage();
}
diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/parser/ScopeChecker.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/parser/ScopeChecker.java
index f5aa489..dbd3c67 100644
--- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/parser/ScopeChecker.java
+++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/parser/ScopeChecker.java
@@ -31,24 +31,21 @@
public class ScopeChecker {
- protected static String quot = "\"";
-
- protected String eol = System.getProperty("line.separator", "\n");
-
protected Counter varCounter = new Counter(-1);
- protected Stack<Scope> scopeStack = new Stack<>();
-
- protected Stack<Scope> forbiddenScopeStack = new Stack<>();
-
- protected String[] inputLines;
+ private static final String EOL = System.getProperty("line.separator", "\n");
+ private final Stack<Scope> scopeStack = new Stack<>();
+ private final Stack<Scope> forbiddenScopeStack = new Stack<>();
+ private String[] inputLines;
public ScopeChecker() {
scopeStack.push(RootScopeFactory.createRootScope(this));
}
- protected void setInput(String s) {
- inputLines = s.split("\n|\r\n?");
+ // TODO: Methods requiring the input String have nothing to do with scope checking. Split this class.
+ public ScopeChecker(String input) {
+ scopeStack.push(RootScopeFactory.createRootScope(this));
+ inputLines = input.split("\n|\r\n?");
}
// Forbidden scopes are used to disallow, in a limit clause, variables
@@ -77,7 +74,7 @@
return scope;
}
- protected final Scope extendCurrentScopeNoPush(boolean maskParentScope) {
+ public final Scope extendCurrentScopeNoPush(boolean maskParentScope) {
Scope parent = scopeStack.peek();
return new Scope(this, parent, maskParentScope);
}
@@ -175,7 +172,7 @@
return false;
}
- protected int appendExpected(StringBuilder expected, int[][] expectedTokenSequences, String[] tokenImage) {
+ public int appendExpected(StringBuilder expected, int[][] expectedTokenSequences, String[] tokenImage) {
int maxSize = 0;
for (int i = 0; i < expectedTokenSequences.length; i++) {
if (maxSize < expectedTokenSequences[i].length) {
@@ -188,7 +185,7 @@
if (expectedTokenSequences[i][expectedTokenSequences[i].length - 1] != 0) {
append(expected, "...");
}
- append(expected, eol);
+ append(expected, EOL);
append(expected, " ");
}
return maxSize;
@@ -200,17 +197,17 @@
}
}
- protected static String fixQuotes(String token) {
+ public static String fixQuotes(String token) {
final String stripped = stripQuotes(token);
return stripped != null ? "'" + stripped + "'" : token;
}
- protected static String stripQuotes(String token) {
+ public static String stripQuotes(String token) {
final int last = token.length() - 1;
return token.charAt(0) == '"' && token.charAt(last) == '"' ? token.substring(1, last) : null;
}
- protected static String addEscapes(String str) {
+ public static String addEscapes(String str) {
StringBuilder escaped = new StringBuilder();
for (int i = 0; i < str.length(); i++) {
appendChar(escaped, str.charAt(i));
@@ -305,12 +302,12 @@
return res.toString();
}
- protected String getLine(int line) {
+ public String getLine(int line) {
int idx = line - 1;
return idx >= 0 && idx < inputLines.length ? inputLines[idx] : "";
}
- protected String extractFragment(int beginLine, int beginColumn, int endLine, int endColumn) {
+ public String extractFragment(int beginLine, int beginColumn, int endLine, int endColumn) {
StringBuilder extract = new StringBuilder();
if (beginLine == endLine) {
// special case that we need to handle separately
diff --git a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/FunctionUtil.java b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/FunctionUtil.java
index ded91af..05dfe12 100644
--- a/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/FunctionUtil.java
+++ b/asterixdb/asterix-lang-common/src/main/java/org/apache/asterix/lang/common/util/FunctionUtil.java
@@ -19,7 +19,6 @@
package org.apache.asterix.lang.common.util;
-import java.io.StringReader;
import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
@@ -366,7 +365,7 @@
throw new CompilationException(ErrorCode.COMPILATION_INCOMPATIBLE_FUNCTION_LANGUAGE, sourceLoc,
function.getLanguage(), function.getSignature().toString(), parserFactory.getLanguage());
}
- IParser parser = parserFactory.createParser(new StringReader(function.getFunctionBody()));
+ IParser parser = parserFactory.createParser(function.getFunctionBody());
try {
FunctionDecl functionDecl =
parser.parseFunctionBody(function.getSignature(), function.getParameterNames(), true);
diff --git a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/parser/SqlppParserFactory.java b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/parser/SqlppParserFactory.java
index 5b724f2..3aec49b 100644
--- a/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/parser/SqlppParserFactory.java
+++ b/asterixdb/asterix-lang-sqlpp/src/main/java/org/apache/asterix/lang/sqlpp/parser/SqlppParserFactory.java
@@ -18,10 +18,9 @@
*/
package org.apache.asterix.lang.sqlpp.parser;
-import java.io.Reader;
-
import org.apache.asterix.lang.common.base.IParser;
import org.apache.asterix.lang.common.base.IParserFactory;
+import org.apache.asterix.lang.common.parser.ScopeChecker;
public class SqlppParserFactory implements IParserFactory {
@@ -30,12 +29,7 @@
@Override
public IParser createParser(String query) {
- return new SQLPPParser(query);
- }
-
- @Override
- public IParser createParser(Reader reader) {
- return new SQLPPParser(reader);
+ return new SQLPPParser(query, new ScopeChecker(query));
}
@Override
diff --git a/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj b/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj
index c4c86f4..a895bc9 100644
--- a/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj
+++ b/asterixdb/asterix-lang-sqlpp/src/main/javacc/SQLPP.jj
@@ -216,7 +216,7 @@
import org.apache.hyracks.util.LogRedactionUtil;
import org.apache.hyracks.util.StringUtil;
-class SQLPPParser extends ScopeChecker implements IParser {
+class SQLPPParser implements IParser {
// tokens parsed as identifiers
private static final String CUBE = "CUBE";
@@ -246,6 +246,8 @@
private static final String RETURNS = "RETURNS";
private static final String CONFIG = "CONFIG";
+ private static final String QUOT = "\"";
+ private static final String EOL = System.getProperty("line.separator", "\n");
private static final String INT_TYPE_NAME = "int";
private static final String UDF_VARARGS_PARAM_NAME = "args"; // Note: this value is stored in the function metadata
@@ -263,6 +265,9 @@
private final Map<SourceLocation, String> hintCollector = new HashMap<SourceLocation, String>();
+ // This cannot be final and unassigned since generated constructors do not assign it.
+ private ScopeChecker scopeChecker = null;
+
private static class IndexParams {
public IndexType type;
public int gramLength;
@@ -333,9 +338,9 @@
}
}
- public SQLPPParser(String s) {
+ public SQLPPParser(String s, ScopeChecker scopeChecker) {
this(new StringReader(s));
- super.setInput(s);
+ this.scopeChecker = scopeChecker;
}
public static void main(String args[]) throws ParseException, TokenMgrError, IOException, FileNotFoundException, CompilationException {
@@ -367,7 +372,7 @@
}
private static Expression parseExpression(String text) throws CompilationException {
- return new SQLPPParser(text).parseExpression();
+ return new SQLPPParser(text, new ScopeChecker(text)).parseExpression();
}
@Override
@@ -390,7 +395,7 @@
}
private static List<String> parseParenthesizedIdentifierList(String text) throws CompilationException {
- return new SQLPPParser(text).parseParenthesizedIdentifierList();
+ return new SQLPPParser(text, new ScopeChecker(text)).parseParenthesizedIdentifierList();
}
@Override
@@ -401,13 +406,13 @@
public FunctionDecl parse() throws ParseException {
DataverseName dataverse = defaultDataverse;
defaultDataverse = signature.getDataverseName();
- createNewScope();
+ scopeChecker.createNewScope();
List<VarIdentifier> paramVars = new ArrayList<VarIdentifier>(paramNames.size());
for (String paramName : paramNames) {
paramVars.add(SqlppVariableUtil.toInternalVariableIdentifier(paramName));
}
Expression functionBodyExpr = SQLPPParser.this.FunctionBody();
- removeCurrentScope();
+ scopeChecker.removeCurrentScope();
defaultDataverse = dataverse;
return new FunctionDecl(signature, paramVars, functionBodyExpr, isStored);
}
@@ -469,26 +474,26 @@
}
int[][] expectedTokenSequences = pe.expectedTokenSequences;
String[] tokenImage = pe.tokenImage;
- String sep = REPORT_EXPECTED_TOKENS ? eol : " ";
+ String sep = REPORT_EXPECTED_TOKENS ? EOL : " ";
StringBuilder expected = REPORT_EXPECTED_TOKENS ? new StringBuilder() : null;
- int maxSize = appendExpected(expected, expectedTokenSequences, tokenImage);
+ int maxSize = scopeChecker.appendExpected(expected, expectedTokenSequences, tokenImage);
Token tok = currentToken.next;
int line = tok.beginLine;
StringBuilder message = new StringBuilder(128);
- message.append("In line ").append(line).append(" >>").append(getLine(line)).append("<<").append(sep).append("Encountered ");
+ message.append("In line ").append(line).append(" >>").append(scopeChecker.getLine(line)).append("<<").append(sep).append("Encountered ");
for (int i = 0; i < maxSize; i++) {
if (i != 0) {
message.append(' ');
}
if (tok.kind == 0) {
- message.append(fixQuotes(tokenImage[0]));
+ message.append(scopeChecker.fixQuotes(tokenImage[0]));
break;
}
final String fixedTokenImage = tokenImage[tok.kind];
- if (! tok.image.equalsIgnoreCase(stripQuotes(fixedTokenImage))) {
- message.append(fixQuotes(fixedTokenImage)).append(' ');
+ if (! tok.image.equalsIgnoreCase(scopeChecker.stripQuotes(fixedTokenImage))) {
+ message.append(scopeChecker.fixQuotes(fixedTokenImage)).append(' ');
}
- message.append(quot).append(addEscapes(tok.image)).append(quot);
+ message.append(QUOT).append(scopeChecker.addEscapes(tok.image)).append(QUOT);
tok = tok.next;
}
message.append(" at column ").append(currentToken.next.beginColumn).append('.').append(sep);
@@ -695,7 +700,6 @@
List<Statement> Statement() throws ParseException:
{
- scopeStack.push(RootScopeFactory.createRootScope(this));
List<Statement> decls = new ArrayList<Statement>();
Statement stmt = null;
}
@@ -1427,16 +1431,16 @@
<LEFTBRACE>
{
beginPos = token;
- createNewScope();
+ scopeChecker.createNewScope();
}
functionBodyExpr = FunctionBody()
<RIGHTBRACE>
{
endPos = token;
- String functionBody = extractFragment(beginPos.beginLine, beginPos.beginColumn, endPos.beginLine,
+ String functionBody = scopeChecker.extractFragment(beginPos.beginLine, beginPos.beginColumn, endPos.beginLine,
endPos.beginColumn);
- getCurrentScope().addFunctionDescriptor(signature, false);
- removeCurrentScope();
+ scopeChecker.getCurrentScope().addFunctionDescriptor(signature, false);
+ scopeChecker.removeCurrentScope();
ensureNoTypeDeclsInFunction(fctName.function, params, returnType, startStmtToken);
stmt = new CreateFunctionStatement(signature, params, functionBody, functionBodyExpr, orReplace, ifNotExists);
}
@@ -2175,7 +2179,7 @@
}
query.setTopLevel(true);
InsertStatement stmt = new InsertStatement(nameComponents.first, nameComponents.second.getValue(), query,
- getVarCounter(), var, returnExpression);
+ scopeChecker.getVarCounter(), var, returnExpression);
return addSourceLocation(stmt, startToken);
}
}
@@ -2199,7 +2203,7 @@
}
query.setTopLevel(true);
UpsertStatement stmt = new UpsertStatement(nameComponents.first, nameComponents.second.getValue(), query,
- getVarCounter(), var, returnExpression);
+ scopeChecker.getVarCounter(), var, returnExpression);
return addSourceLocation(stmt, startToken);
}
}
@@ -2221,7 +2225,7 @@
addSourceLocation(var, startToken);
}
DeleteStatement stmt = new DeleteStatement(var, nameComponents.first, nameComponents.second.getValue(),
- condition, getVarCounter());
+ condition, scopeChecker.getVarCounter());
return addSourceLocation(stmt, startToken);
}
}
@@ -2460,12 +2464,12 @@
{
if (whereClause != null) {
endPos = token;
- whereClauseBody = extractFragment(beginPos.endLine, beginPos.endColumn, endPos.endLine, endPos.endColumn + 1);
+ whereClauseBody = scopeChecker.extractFragment(beginPos.endLine, beginPos.endColumn, endPos.endLine, endPos.endColumn + 1);
}
}
{
stmt = new ConnectFeedStatement(feedNameComponents, datasetNameComponents, appliedFunctions,
- policy, whereClauseBody, getVarCounter());
+ policy, whereClauseBody, scopeChecker.getVarCounter());
}
)
{
@@ -2868,7 +2872,7 @@
{
<QUOTED_STRING>
{
- return removeQuotesAndEscapes(token.image);
+ return scopeChecker.removeQuotesAndEscapes(token.image);
}
}
@@ -2878,7 +2882,7 @@
{
<STRING_LITERAL>
{
- return removeQuotesAndEscapes(token.image);
+ return scopeChecker.removeQuotesAndEscapes(token.image);
}
}
@@ -2969,7 +2973,7 @@
String functionName;
Pair<Integer, List<Pair<VarIdentifier,TypeExpression>>> paramsWithArity = null;
Expression funcBody;
- createNewScope();
+ scopeChecker.createNewScope();
}
{
<DECLARE> { startToken = token; } <FUNCTION>
@@ -2982,14 +2986,14 @@
int arity = paramsWithArity.first;
List<Pair<VarIdentifier,TypeExpression>> paramList = paramsWithArity.second;
FunctionSignature signature = new FunctionSignature(defaultDataverse, functionName, arity);
- getCurrentScope().addFunctionDescriptor(signature, false);
+ scopeChecker.getCurrentScope().addFunctionDescriptor(signature, false);
ensureNoTypeDeclsInFunction(functionName, paramList, null, startToken);
List<VarIdentifier> params = new ArrayList<VarIdentifier>(paramList.size());
for (Pair<VarIdentifier,TypeExpression> p: paramList) {
params.add(p.getFirst());
}
FunctionDecl stmt = new FunctionDecl(signature, params, funcBody, false);
- removeCurrentScope();
+ scopeChecker.removeCurrentScope();
return addSourceLocation(stmt, startToken);
}
}
@@ -3607,8 +3611,8 @@
{
id = VariableIdentifier()
{
- Identifier ident = lookupSymbol(id);
- if (isInForbiddenScopes(id)) {
+ Identifier ident = scopeChecker.lookupSymbol(id);
+ if (scopeChecker.isInForbiddenScopes(id)) {
throw new SqlppParseException(getSourceLocation(token),
"Inside limit clauses, it is disallowed to reference a variable having the same name as any variable bound in the same scope as the limit clause.");
}
@@ -3630,7 +3634,7 @@
{
id = VariableIdentifier()
{
- Identifier ident = lookupSymbol(id);
+ Identifier ident = scopeChecker.lookupSymbol(id);
VariableExpr varExp = new VariableExpr(new VarIdentifier(id));
if (ident != null) { // exist such ident
varExp.setIsNewVar(false);
@@ -3688,7 +3692,7 @@
(
<DOLLAR_IDENTIFIER> { name = token.image.substring(1); }
| <DOLLAR_INTEGER_LITERAL> { name = token.image.substring(1); }
- | <DOLLAR_QUOTED_STRING> { name = removeQuotesAndEscapes(token.image.substring(1)); }
+ | <DOLLAR_QUOTED_STRING> { name = scopeChecker.removeQuotesAndEscapes(token.image.substring(1)); }
| <QUES> { name = String.valueOf(++externalVarCounter); }
)
{
@@ -3845,7 +3849,7 @@
}
String fqFunctionName = funcName.library == null ? name : funcName.library + "#" + name;
int arity = argList.size();
- FunctionSignature signature = lookupFunctionSignature(funcName.dataverse, fqFunctionName, arity);
+ FunctionSignature signature = scopeChecker.lookupFunctionSignature(funcName.dataverse, fqFunctionName, arity);
if (signature == null) {
signature = new FunctionSignature(funcName.dataverse, fqFunctionName, arity);
}
@@ -4134,7 +4138,7 @@
SelectSetOperation selectSetOperation;
OrderbyClause orderbyClause = null;
LimitClause limitClause = null;
- createNewScope();
+ scopeChecker.createNewScope();
}
{
( letClauses = LetClause() )?
@@ -4383,7 +4387,7 @@
{
Token startToken = null;
List<FromTerm> fromTerms = new ArrayList<FromTerm>();
- extendCurrentScope();
+ scopeChecker.extendCurrentScope();
}
{
{
@@ -4582,8 +4586,7 @@
}
{
{
- Scope newScope = extendCurrentScopeNoPush(true);
- // extendCurrentScope(true);
+ Scope newScope = scopeChecker.extendCurrentScopeNoPush(true);
}
<GROUP>
{
@@ -4616,7 +4619,7 @@
gbc.setWithVarMap(new HashMap<Expression, VariableExpr>());
gbc.setGroupVar(groupVar);
gbc.setGroupFieldList(groupFieldList);
- replaceCurrentScope(newScope);
+ scopeChecker.replaceCurrentScope(newScope);
return addSourceLocation(gbc, startToken);
}
}
@@ -4750,14 +4753,14 @@
{
(
(
- <LIMIT> { startToken = token; pushForbiddenScope(getCurrentScope()); } limitExpr = Expression()
+ <LIMIT> { startToken = token; scopeChecker.pushForbiddenScope(scopeChecker.getCurrentScope()); } limitExpr = Expression()
( <OFFSET> offsetExpr = Expression() )?
- { popForbiddenScope(); }
+ { scopeChecker.popForbiddenScope(); }
)
|
(
- <OFFSET> { startToken = token; pushForbiddenScope(getCurrentScope()); } offsetExpr = Expression()
- { popForbiddenScope(); }
+ <OFFSET> { startToken = token; scopeChecker.pushForbiddenScope(scopeChecker.getCurrentScope()); } offsetExpr = Expression()
+ { scopeChecker.popForbiddenScope(); }
)
)
{
@@ -4778,7 +4781,7 @@
}
{
{
- createNewScope();
+ scopeChecker.createNewScope();
}
( ((<ANY>|<SOME>) { startToken = token; qc.setQuantifier(QuantifiedExpression.Quantifier.SOME); })
@@ -4799,7 +4802,7 @@
{
qc.setSatisfiesExpr(satisfiesExpr);
qc.setQuantifiedList(quantifiedList);
- removeCurrentScope();
+ scopeChecker.removeCurrentScope();
return addSourceLocation(qc, startToken);
}
}
@@ -4808,7 +4811,7 @@
{
VariableExpr varExp;
Expression beExp;
- extendCurrentScope();
+ scopeChecker.extendCurrentScope();
}
{
varExp = Variable() <EQ> beExp = Expression()
@@ -4823,7 +4826,7 @@
{
VariableExpr varExp;
Expression beExp;
- extendCurrentScope();
+ scopeChecker.extendCurrentScope();
}
{
varExp = Variable() <AS> beExp = Expression()
diff --git a/asterixdb/asterix-tools/src/test/java/org/apache/asterix/tools/datagen/AdgClientDriver.java b/asterixdb/asterix-tools/src/test/java/org/apache/asterix/tools/datagen/AdgClientDriver.java
index 8a1adb0..296d632 100644
--- a/asterixdb/asterix-tools/src/test/java/org/apache/asterix/tools/datagen/AdgClientDriver.java
+++ b/asterixdb/asterix-tools/src/test/java/org/apache/asterix/tools/datagen/AdgClientDriver.java
@@ -19,6 +19,7 @@
package org.apache.asterix.tools.datagen;
import java.io.File;
+import java.nio.file.Path;
import org.kohsuke.args4j.Argument;
import org.kohsuke.args4j.CmdLineParser;
@@ -30,7 +31,7 @@
public static class AdgClientConfig {
@Argument(index = 0, required = true, metaVar = "ARG1", usage = "The file containing the annotated schema.")
- private File schemaFile;
+ private Path schemaFile;
@Argument(index = 1, required = true, metaVar = "ARG2", usage = "The output directory path.")
private File outputDir;
diff --git a/asterixdb/asterix-tools/src/test/java/org/apache/asterix/tools/datagen/AdmDataGen.java b/asterixdb/asterix-tools/src/test/java/org/apache/asterix/tools/datagen/AdmDataGen.java
index aa86916..fb02243 100644
--- a/asterixdb/asterix-tools/src/test/java/org/apache/asterix/tools/datagen/AdmDataGen.java
+++ b/asterixdb/asterix-tools/src/test/java/org/apache/asterix/tools/datagen/AdmDataGen.java
@@ -21,9 +21,11 @@
import java.io.BufferedOutputStream;
import java.io.File;
import java.io.FileOutputStream;
-import java.io.FileReader;
import java.io.IOException;
import java.io.PrintStream;
+import java.nio.charset.StandardCharsets;
+import java.nio.file.Files;
+import java.nio.file.Path;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
@@ -922,23 +924,21 @@
}
- private final File schemaFile;
+ private final Path schemaFile;
private final File outputDir;
private Map<TypeSignature, IAType> typeMap;
private Map<TypeSignature, TypeDataGen> typeAnnotMap;
private DataGeneratorContext dgCtx;
private final IParserFactory parserFactory = new SqlppParserFactory();
- public AdmDataGen(File schemaFile, File outputDir) {
+ public AdmDataGen(Path schemaFile, File outputDir) {
this.schemaFile = schemaFile;
this.outputDir = outputDir;
}
public void init() throws IOException, ACIDException, AlgebricksException {
- FileReader aql = new FileReader(schemaFile);
- IParser parser = parserFactory.createParser(aql);
+ IParser parser = parserFactory.createParser(Files.readString(schemaFile, StandardCharsets.UTF_8));
List<Statement> statements = parser.parse();
- aql.close();
// TODO: Need to fix how to use transactions here.
MetadataTransactionContext mdTxnCtx = new MetadataTransactionContext(new TxnId(-1));
ADGenDmlTranslator dmlt = new ADGenDmlTranslator(mdTxnCtx, statements);
--
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/11943
To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Change-Id: Iea8b66d064386124725de450e0faf7cb6dc0fbcc
Gerrit-Change-Number: 11943
Gerrit-PatchSet: 1
Gerrit-Owner: Cameron Samak <cs...@apache.org>
Gerrit-MessageType: newchange
Change in asterixdb[master]: [ASTERIXDB-2915][SQL] Fix UDF parsing issues
Posted by AsterixDB Code Review <do...@asterix-gerrit.ics.uci.edu>.
From Cameron Samak <cs...@apache.org>:
Cameron Samak has posted comments on this change. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/11943 )
Change subject: [ASTERIXDB-2915][SQL] Fix UDF parsing issues
......................................................................
Patch Set 1: Code-Review+1
--
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/11943
To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Change-Id: Iea8b66d064386124725de450e0faf7cb6dc0fbcc
Gerrit-Change-Number: 11943
Gerrit-PatchSet: 1
Gerrit-Owner: Cameron Samak <cs...@apache.org>
Gerrit-Reviewer: Anon. E. Moose #1000171
Gerrit-Reviewer: Cameron Samak <cs...@apache.org>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-Comment-Date: Mon, 14 Jun 2021 22:04:01 +0000
Gerrit-HasComments: No
Gerrit-Has-Labels: Yes
Gerrit-MessageType: comment