You are viewing a plain text version of this content. The canonical link for it is here.

Posted to notifications@asterixdb.apache.org by AsterixDB Code Review <do...@vitalstatistix.ics.uci.edu> on 2019/11/26 16:51:10 UTC

Change in asterixdb[master]: [ASTERIXDB-2683][OTH] Add tests for filters using large (>50K) in-lists.

From Hussain Towaileb <hu...@gmail.com>:

Hussain Towaileb has uploaded this change for review. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/4363 )


Change subject: [ASTERIXDB-2683][OTH] Add tests for filters using large (>50K) in-lists.
......................................................................

[ASTERIXDB-2683][OTH] Add tests for filters using large (>50K) in-lists.

- user model changes: no
- storage format changes: no
- interface changes: no

Details:
- Added a methodology for generating random number
  of items that can be inserted in queries for
  convenience.
- Added test cases for filters using large number
  of items for in-lists.

Change-Id: I39f88d6faead10b106ebdc147d0f9137090d061c
---
M asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
M asterixdb/asterix-app/src/test/resources/runtimets/only_sqlpp.xml
A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/list/big_in_list/001/big_in_list.000.ddl.sqlpp
A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/list/big_in_list/001/big_in_list.001.update.sqlpp
A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/list/big_in_list/001/big_in_list.002.query.sqlpp
A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/list/big_in_list/001/big_in_list.003.ddl.sqlpp
A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/list/big_in_list/002/big_in_list.000.query.sqlpp
A asterixdb/asterix-app/src/test/resources/runtimets/results/list/big_in_list/001/big_in_list.000.adm
A asterixdb/asterix-app/src/test/resources/runtimets/results/list/big_in_list/002/big_in_list.000.adm
9 files changed, 186 insertions(+), 1 deletion(-)



  git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/63/4363/1

diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
index 7c1b1f3..259c461 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/common/TestExecutor.java
@@ -64,6 +64,8 @@
 import java.util.function.Predicate;
 import java.util.regex.Matcher;
 import java.util.regex.Pattern;
+import java.util.stream.Collectors;
+import java.util.stream.IntStream;
 import java.util.stream.Stream;
 
 import org.apache.asterix.api.http.server.QueryServiceRequestParameters;
@@ -191,6 +193,11 @@
     private List<Charset> allCharsets;
     private final Queue<Charset> charsetsRemaining = new ArrayDeque<>();
 
+    // Used when substitution of value is needed
+    private static final String START_SUBSTITUTE_TOKEN = "%@";
+    private static final String END_SUBSTITUTE_TOKEN = "@%";
+    private static final String SUBSTITUTION_COMMAND_SEPARATOR = "-";
+
     /*
      * Instance members
      */
@@ -1836,7 +1843,11 @@
                 savedQueryCounts[numOfFiles] = queryCount.getValue();
                 numOfFiles++;
                 final File testFile = ctx.getFile();
-                final String statement = readTestFile(testFile);
+                String statement = readTestFile(testFile);
+
+                // Perform any replacements if needed
+                statement = applySubstitution(statement);
+
                 try {
                     if (!testFile.getName().startsWith(DIAGNOSE)) {
                         executeTestFile(testCaseCtx, ctx, variableCtx, statement, isDmlRecoveryTest, pb, cUnit,
@@ -1883,6 +1894,49 @@
         }
     }
 
+    private String applySubstitution(String statement) {
+        while (statement.contains(START_SUBSTITUTE_TOKEN) && statement.contains(END_SUBSTITUTE_TOKEN)) {
+            int start = statement.indexOf(START_SUBSTITUTE_TOKEN);
+            int end = statement.indexOf(END_SUBSTITUTE_TOKEN) + END_SUBSTITUTE_TOKEN.length();
+
+            // The string token to be replaced
+            String token = statement.substring(start, end);
+            statement = statement.replace(token, doApplySubstitution(token));
+        }
+
+        return statement;
+    }
+
+    private String doApplySubstitution(String token) {
+        // Remove start and end markers
+        token = token.substring(START_SUBSTITUTE_TOKEN.length(), token.length() - END_SUBSTITUTE_TOKEN.length());
+        String[] tokenSplits = token.split(SUBSTITUTION_COMMAND_SEPARATOR);
+        String substitute = "";
+
+        switch (tokenSplits[0].toLowerCase()) {
+            // gen command is of format gen-type-count-separator (e.g gen-int-5000-,)
+            case "gen":
+                try {
+                    String type = tokenSplits[1];
+                    int count = Integer.valueOf(tokenSplits[2]);
+                    String separator = tokenSplits[3];
+
+                    // Generate the substitute based on the provided type
+                    switch (type.toLowerCase()) {
+                        case "number":
+                            substitute = IntStream.range(0, count).mapToObj(Integer::toString)
+                                    .collect(Collectors.joining(separator));
+                            break;
+                        default:
+                            throw new IllegalStateException("gen command - unknown type: " + type);
+                    }
+                } catch (Exception e) {
+                    throw new IllegalStateException("gen command is not formatted properly");
+                }
+        }
+        return substitute;
+    }
+
     protected void fail(boolean runDiagnostics, TestCaseContext testCaseCtx, CompilationUnit cUnit,
             List<TestFileContext> testFileCtxs, ProcessBuilder pb, File testFile, Exception e) throws Exception {
         if (runDiagnostics) {
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/only_sqlpp.xml b/asterixdb/asterix-app/src/test/resources/runtimets/only_sqlpp.xml
index 334dd52..8404001 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/only_sqlpp.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/only_sqlpp.xml
@@ -19,5 +19,15 @@
  !-->
 <test-suite xmlns="urn:xml.testframework.asterix.apache.org" ResultOffsetPath="results" QueryOffsetPath="queries_sqlpp" QueryFileExtension=".sqlpp">
   <test-group name="failed">
+    <test-case FilePath="list">
+      <compilation-unit name="big_in_list/001">
+        <output-dir compare="Text">big_in_list/001</output-dir>
+      </compilation-unit>
+    </test-case>
+    <test-case FilePath="list">
+      <compilation-unit name="big_in_list/002">
+        <output-dir compare="Text">big_in_list/002</output-dir>
+      </compilation-unit>
+    </test-case>
   </test-group>
 </test-suite>
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/list/big_in_list/001/big_in_list.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/list/big_in_list/001/big_in_list.000.ddl.sqlpp
new file mode 100644
index 0000000..ca9b405
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/list/big_in_list/001/big_in_list.000.ddl.sqlpp
@@ -0,0 +1,30 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop dataverse test if exists;
+create dataverse test;
+use test;
+
+drop type test if exists;
+create type test as open {
+id: int64
+};
+
+drop dataset test if exists;
+create dataset test(test) primary key id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/list/big_in_list/001/big_in_list.001.update.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/list/big_in_list/001/big_in_list.001.update.sqlpp
new file mode 100644
index 0000000..8c45e13
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/list/big_in_list/001/big_in_list.001.update.sqlpp
@@ -0,0 +1,24 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+insert into test([
+{"id": 0, "f1": [%@gen-number-100000-,@%]}
+]);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/list/big_in_list/001/big_in_list.002.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/list/big_in_list/001/big_in_list.002.query.sqlpp
new file mode 100644
index 0000000..e2815ba
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/list/big_in_list/001/big_in_list.002.query.sqlpp
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+use test;
+
+select value f1 = [%@gen-number-100000-,@%]
+from test;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/list/big_in_list/001/big_in_list.003.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/list/big_in_list/001/big_in_list.003.ddl.sqlpp
new file mode 100644
index 0000000..548e632
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/list/big_in_list/001/big_in_list.003.ddl.sqlpp
@@ -0,0 +1,20 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+drop dataverse test if exists;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/list/big_in_list/002/big_in_list.000.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/list/big_in_list/002/big_in_list.000.query.sqlpp
new file mode 100644
index 0000000..bdbe2a9
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/list/big_in_list/002/big_in_list.000.query.sqlpp
@@ -0,0 +1,22 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+select value count(mydata)
+from [%@gen-number-1500000-,@%] as mydata
+where mydata IN [%@gen-number-1000000-,@%];
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/list/big_in_list/001/big_in_list.000.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/list/big_in_list/001/big_in_list.000.adm
new file mode 100644
index 0000000..f32a580
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/list/big_in_list/001/big_in_list.000.adm
@@ -0,0 +1 @@
+true
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/list/big_in_list/002/big_in_list.000.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/list/big_in_list/002/big_in_list.000.adm
new file mode 100644
index 0000000..c2473d6
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/list/big_in_list/002/big_in_list.000.adm
@@ -0,0 +1 @@
+1000000
\ No newline at end of file

-- 
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/4363
To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Change-Id: I39f88d6faead10b106ebdc147d0f9137090d061c
Gerrit-Change-Number: 4363
Gerrit-PatchSet: 1
Gerrit-Owner: Hussain Towaileb <hu...@gmail.com>
Gerrit-MessageType: newchange