You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@asterixdb.apache.org by AsterixDB Code Review <do...@asterix-gerrit.ics.uci.edu> on 2023/08/08 21:37:19 UTC
Change in asterixdb[master]: [NO ISSUE][OTH] Add external filter test against one computed field
From Wail Alkowaileet <wa...@gmail.com>:
Wail Alkowaileet has uploaded this change for review. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17708 )
Change subject: [NO ISSUE][OTH] Add external filter test against one computed field
......................................................................
[NO ISSUE][OTH] Add external filter test against one computed field
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Fix: return all files if the filter is NoOp
- Add tests against one computed field
Change-Id: Idaf2bd4cf467b8ceea99a21fb1ed1892485c6715
---
A asterixdb/asterix-app/data/json/external-filter/last-name/miller/0.json
A asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/s3/filter/one-field/one-field.040.adm
A asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/s3/filter/one-field/one-field.021.plan
A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/filter/one-field/one-field.040.query.sqlpp
A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/filter/one-field/one-field.111.query.sqlpp
A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/filter/one-field/one-field.041.query.sqlpp
A asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/s3/filter/one-field/one-field.121.plan
A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/filter/one-field/one-field.030.query.sqlpp
A asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/s3/filter/one-field/one-field.130.adm
M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataPrefix.java
A asterixdb/asterix-app/data/json/external-filter/department/accounting/0.json
A asterixdb/asterix-app/data/json/external-filter/last-name/smith/0.json
A asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/s3/filter/one-field/one-field.120.adm
A asterixdb/asterix-app/data/json/external-filter/department/hr/0.json
A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/filter/one-field/one-field.000.ddl.sqlpp
A asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/s3/filter/one-field/one-field.010.adm
A asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/s3/filter/one-field/one-field.041.plan
A asterixdb/asterix-app/data/json/external-filter/department/engineering/0.json
A asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/s3/filter/one-field/one-field.110.adm
M asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/ExternalDatasetTestUtils.java
M asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
A asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/s3/filter/one-field/one-field.011.plan
A asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/s3/filter/one-field/one-field.111.plan
A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/filter/one-field/one-field.021.query.sqlpp
A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/filter/one-field/one-field.010.query.sqlpp
A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/filter/one-field/one-field.020.query.sqlpp
A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/filter/one-field/one-field.110.query.sqlpp
A asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/s3/filter/one-field/one-field.020.adm
A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/filter/one-field/one-field.031.query.sqlpp
A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/filter/one-field/one-field.121.query.sqlpp
M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStreamFactory.java
A asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/s3/filter/one-field/one-field.031.plan
A asterixdb/asterix-app/data/json/external-filter/last-name/Jones/0.json
A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/filter/one-field/one-field.120.query.sqlpp
A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/filter/one-field/one-field.131.query.sqlpp
A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/filter/one-field/one-field.011.query.sqlpp
A asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/s3/filter/one-field/one-field.131.plan
A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/filter/one-field/one-field.130.query.sqlpp
A asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/s3/filter/one-field/one-field.030.adm
M asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java
40 files changed, 673 insertions(+), 2 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/08/17708/1
diff --git a/asterixdb/asterix-app/data/json/external-filter/department/accounting/0.json b/asterixdb/asterix-app/data/json/external-filter/department/accounting/0.json
new file mode 100644
index 0000000..880162e
--- /dev/null
+++ b/asterixdb/asterix-app/data/json/external-filter/department/accounting/0.json
@@ -0,0 +1,3 @@
+{"id": 1, "department": "accounting", "name": {"first": "John", "last": "Smith"}}
+{"id": 2, "department": "accounting", "name": {"first": "Mike", "last": "Jones"}}
+{"id": 3, "department": "accounting", "name": {"first": "Alex", "last": "Miller"}}
\ No newline at end of file
diff --git a/asterixdb/asterix-app/data/json/external-filter/department/engineering/0.json b/asterixdb/asterix-app/data/json/external-filter/department/engineering/0.json
new file mode 100644
index 0000000..610aaed
--- /dev/null
+++ b/asterixdb/asterix-app/data/json/external-filter/department/engineering/0.json
@@ -0,0 +1,3 @@
+{"id": 4, "department": "engineering", "name": {"first": "Tom", "last": "Smith"}}
+{"id": 5, "department": "engineering", "name": {"first": "Alice", "last": "Jones"}}
+{"id": 6, "department": "engineering", "name": {"first": "Sheila", "last": "Miller"}}
\ No newline at end of file
diff --git a/asterixdb/asterix-app/data/json/external-filter/department/hr/0.json b/asterixdb/asterix-app/data/json/external-filter/department/hr/0.json
new file mode 100644
index 0000000..3d09fa0
--- /dev/null
+++ b/asterixdb/asterix-app/data/json/external-filter/department/hr/0.json
@@ -0,0 +1,3 @@
+{"id": 7, "department": "hr", "name": {"first": "James", "last": "Smith"}}
+{"id": 8, "department": "hr", "name": {"first": "David", "last": "Jones"}}
+{"id": 9, "department": "hr", "name": {"first": "Noah", "last": "Miller"}}
\ No newline at end of file
diff --git a/asterixdb/asterix-app/data/json/external-filter/last-name/Jones/0.json b/asterixdb/asterix-app/data/json/external-filter/last-name/Jones/0.json
new file mode 100644
index 0000000..d618df0
--- /dev/null
+++ b/asterixdb/asterix-app/data/json/external-filter/last-name/Jones/0.json
@@ -0,0 +1,3 @@
+{"id": 2, "department": "accounting", "name": {"first": "Mike", "last": "Jones"}}
+{"id": 5, "department": "engineering", "name": {"first": "Alice", "last": "Jones"}}
+{"id": 8, "department": "hr", "name": {"first": "David", "last": "Jones"}}
\ No newline at end of file
diff --git a/asterixdb/asterix-app/data/json/external-filter/last-name/miller/0.json b/asterixdb/asterix-app/data/json/external-filter/last-name/miller/0.json
new file mode 100644
index 0000000..6d7542d
--- /dev/null
+++ b/asterixdb/asterix-app/data/json/external-filter/last-name/miller/0.json
@@ -0,0 +1,3 @@
+{"id": 3, "department": "accounting", "name": {"first": "Alex", "last": "Miller"}}
+{"id": 6, "department": "engineering", "name": {"first": "Sheila", "last": "Miller"}}
+{"id": 9, "department": "hr", "name": {"first": "Noah", "last": "Miller"}}
\ No newline at end of file
diff --git a/asterixdb/asterix-app/data/json/external-filter/last-name/smith/0.json b/asterixdb/asterix-app/data/json/external-filter/last-name/smith/0.json
new file mode 100644
index 0000000..972c531
--- /dev/null
+++ b/asterixdb/asterix-app/data/json/external-filter/last-name/smith/0.json
@@ -0,0 +1,3 @@
+{"id": 1, "department": "accounting", "name": {"first": "John", "last": "Smith"}}
+{"id": 4, "department": "engineering", "name": {"first": "Tom", "last": "Smith"}}
+{"id": 7, "department": "hr", "name": {"first": "James", "last": "Smith"}}
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/ExternalDatasetTestUtils.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/ExternalDatasetTestUtils.java
index 316d261..b50c353 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/ExternalDatasetTestUtils.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/ExternalDatasetTestUtils.java
@@ -25,14 +25,17 @@
import java.io.BufferedWriter;
import java.io.File;
import java.io.FileWriter;
+import java.io.FilenameFilter;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
+import java.util.Collection;
import org.apache.asterix.test.external_dataset.parquet.BinaryFileConverterUtil;
import org.apache.asterix.testframework.context.TestCaseContext;
import org.apache.commons.io.FilenameUtils;
+import org.apache.hyracks.api.util.IoUtil;
import org.apache.logging.log4j.LogManager;
import org.apache.logging.log4j.Logger;
import org.junit.Assert;
@@ -43,6 +46,8 @@
public class ExternalDatasetTestUtils {
protected static final Logger LOGGER = LogManager.getLogger();
+ // Extension filters
+ private static final FilenameFilter JSON_FILTER = ((dir, name) -> name.endsWith(".json"));
// Base directory paths for data files
private static String JSON_DATA_PATH;
@@ -212,6 +217,9 @@
loadGzData(dataBasePath, "multi-lines-with-arrays", "5-records.json", definition, definitionSegment, false);
loadGzData(dataBasePath, "multi-lines-with-nested-objects", "5-records.json", definition, definitionSegment,
false);
+
+ // Load external filter directories and files
+ loadDirectory(dataBasePath, "external-filter", JSON_FILTER);
}
private static void loadCsvFiles() {
@@ -274,6 +282,21 @@
loadData(generatedDataBasePath, "", "parquetTypes.parquet", definition, definitionSegment, false, false);
}
+ private static void loadDirectory(String dataBasePath, String rootPath, FilenameFilter filter) {
+ Collection<File> files = IoUtil.getMatchingFiles(Path.of(dataBasePath, rootPath), filter);
+ int size = 0;
+ for (File file : files) {
+ String path = file.getPath();
+ // +1 to remove the leading '/'
+ int startIndex = path.indexOf(rootPath) + rootPath.length() + 1;
+ int endIndex = path.lastIndexOf(File.separatorChar);
+ String definitionSegment = rootPath + File.separator + path.substring(startIndex, endIndex);
+ loadData(path.substring(0, endIndex), "", file.getName(), "", definitionSegment, false, false);
+ size++;
+ }
+ LOGGER.info("Loaded {} files from {}", size, dataBasePath + File.separator + rootPath);
+ }
+
private static void loadData(String fileBasePath, String filePathSegment, String filename, String definition,
String definitionSegment, boolean removeExtension) {
loadData(fileBasePath, filePathSegment, filename, definition, definitionSegment, removeExtension, true);
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java
index 0121b58..ee00b50 100644
--- a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/external_dataset/aws/AwsS3ExternalDatasetTest.java
@@ -375,7 +375,7 @@
private static void startAwsS3MockServer() {
// Starting S3 mock server to be used instead of real S3 server
LOGGER.info("Starting S3 mock server");
- s3MockServer = new S3Mock.Builder().withPort(MOCK_SERVER_PORT).withInMemoryBackend().build();
+ s3MockServer = new S3Mock.Builder().withPort(MOCK_SERVER_PORT).build();
try {
s3MockServer.start();
} catch (Exception ex) {
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/filter/one-field/one-field.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/filter/one-field/one-field.000.ddl.sqlpp
new file mode 100644
index 0000000..158cea1
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/filter/one-field/one-field.000.ddl.sqlpp
@@ -0,0 +1,47 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+DROP DATAVERSE test IF EXISTS;
+CREATE DATAVERSE test;
+USE test;
+
+CREATE TYPE TestType AS {
+};
+
+
+CREATE EXTERNAL DATASET Department(TestType) USING S3 (
+ ("accessKeyId"="dummyAccessKey"),
+ ("secretAccessKey"="dummySecretKey"),
+ ("region"="us-west-2"),
+ ("serviceEndpoint"="http://127.0.0.1:8001"),
+ ("container"="playground"),
+ ("definition"="external-filter/department/{department:string}"),
+ ("format"="json")
+);
+
+CREATE EXTERNAL DATASET LastName(TestType) USING S3 (
+ ("accessKeyId"="dummyAccessKey"),
+ ("secretAccessKey"="dummySecretKey"),
+ ("region"="us-west-2"),
+ ("serviceEndpoint"="http://127.0.0.1:8001"),
+ ("container"="playground"),
+ ("container"="playground"),
+ ("definition"="external-filter/last-name/{name.last:string}"),
+ ("format"="json")
+);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/filter/one-field/one-field.010.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/filter/one-field/one-field.010.query.sqlpp
new file mode 100644
index 0000000..0034525
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/filter/one-field/one-field.010.query.sqlpp
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+
+SELECT VALUE d
+FROM Department d
+WHERE d.department = "accounting"
+ORDER BY d.id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/filter/one-field/one-field.011.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/filter/one-field/one-field.011.query.sqlpp
new file mode 100644
index 0000000..b5856ac
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/filter/one-field/one-field.011.query.sqlpp
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+EXPLAIN
+SELECT VALUE d
+FROM Department d
+WHERE d.department = "accounting"
+ORDER BY d.id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/filter/one-field/one-field.020.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/filter/one-field/one-field.020.query.sqlpp
new file mode 100644
index 0000000..a0919a7
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/filter/one-field/one-field.020.query.sqlpp
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+
+SELECT VALUE d
+FROM Department d
+WHERE d.department = "accounting"
+ AND d.name.last = "Smith"
+ORDER BY d.id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/filter/one-field/one-field.021.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/filter/one-field/one-field.021.query.sqlpp
new file mode 100644
index 0000000..3eb867c
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/filter/one-field/one-field.021.query.sqlpp
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+EXPLAIN
+SELECT VALUE d
+FROM Department d
+WHERE d.department = "accounting"
+ AND d.name.last = "Smith"
+ORDER BY d.id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/filter/one-field/one-field.030.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/filter/one-field/one-field.030.query.sqlpp
new file mode 100644
index 0000000..c24bf2c
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/filter/one-field/one-field.030.query.sqlpp
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+
+SELECT VALUE d
+FROM Department d
+WHERE d.department = "accounting"
+ OR d.name.last = "Smith"
+ORDER BY d.id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/filter/one-field/one-field.031.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/filter/one-field/one-field.031.query.sqlpp
new file mode 100644
index 0000000..e1c0b02
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/filter/one-field/one-field.031.query.sqlpp
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+EXPLAIN
+SELECT VALUE d
+FROM Department d
+WHERE d.department = "accounting"
+ OR d.name.last = "Smith"
+ORDER BY d.id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/filter/one-field/one-field.040.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/filter/one-field/one-field.040.query.sqlpp
new file mode 100644
index 0000000..e3e6ee5
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/filter/one-field/one-field.040.query.sqlpp
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+
+SELECT VALUE d
+FROM Department d
+WHERE uppercase(d.department) = "H" || "R"
+ORDER BY d.id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/filter/one-field/one-field.041.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/filter/one-field/one-field.041.query.sqlpp
new file mode 100644
index 0000000..bfa65b5
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/filter/one-field/one-field.041.query.sqlpp
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+EXPLAIN
+SELECT VALUE d
+FROM Department d
+WHERE uppercase(d.department) = "H" || "R"
+ORDER BY d.id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/filter/one-field/one-field.110.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/filter/one-field/one-field.110.query.sqlpp
new file mode 100644
index 0000000..349ba83
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/filter/one-field/one-field.110.query.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+
+SELECT VALUE d
+FROM LastName d
+ORDER BY d.id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/filter/one-field/one-field.111.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/filter/one-field/one-field.111.query.sqlpp
new file mode 100644
index 0000000..1beac09
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/filter/one-field/one-field.111.query.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+EXPLAIN
+SELECT VALUE d
+FROM LastName d
+ORDER BY d.id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/filter/one-field/one-field.120.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/filter/one-field/one-field.120.query.sqlpp
new file mode 100644
index 0000000..7ae11e6
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/filter/one-field/one-field.120.query.sqlpp
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+
+SELECT VALUE d
+FROM LastName d
+WHERE d.name.last = "Jones"
+ORDER BY d.id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/filter/one-field/one-field.121.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/filter/one-field/one-field.121.query.sqlpp
new file mode 100644
index 0000000..e26b35a
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/filter/one-field/one-field.121.query.sqlpp
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+EXPLAIN
+SELECT VALUE d
+FROM LastName d
+WHERE d.name.last = "Jones"
+ORDER BY d.id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/filter/one-field/one-field.130.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/filter/one-field/one-field.130.query.sqlpp
new file mode 100644
index 0000000..c4d575d
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/filter/one-field/one-field.130.query.sqlpp
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+
+SELECT VALUE d
+FROM LastName d
+WHERE lowercase(d.name.first) = "john"
+ AND "john " || lowercase(d.name.last) = "john smith"
+ORDER BY d.id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/filter/one-field/one-field.131.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/filter/one-field/one-field.131.query.sqlpp
new file mode 100644
index 0000000..94b3be1
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/s3/filter/one-field/one-field.131.query.sqlpp
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+EXPLAIN
+SELECT VALUE d
+FROM LastName d
+WHERE "john " || lowercase(d.name.last) = "john smith"
+ORDER BY d.id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/s3/filter/one-field/one-field.010.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/s3/filter/one-field/one-field.010.adm
new file mode 100644
index 0000000..cf43451
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/s3/filter/one-field/one-field.010.adm
@@ -0,0 +1,3 @@
+{ "id": 1, "department": "accounting", "name": { "first": "John", "last": "Smith" } }
+{ "id": 2, "department": "accounting", "name": { "first": "Mike", "last": "Jones" } }
+{ "id": 3, "department": "accounting", "name": { "first": "Alex", "last": "Miller" } }
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/s3/filter/one-field/one-field.011.plan b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/s3/filter/one-field/one-field.011.plan
new file mode 100644
index 0000000..8dcaa95
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/s3/filter/one-field/one-field.011.plan
@@ -0,0 +1,24 @@
+distribute result [$$d] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+-- DISTRIBUTE_RESULT |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ project ([$$d]) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- STREAM_PROJECT |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- SORT_MERGE_EXCHANGE [$$17(ASC) ] |PARTITIONED|
+ order (ASC, $$17) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- STABLE_SORT [$$17(ASC)] |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ select (eq($$d.getField("department"), "accounting")) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- STREAM_SELECT |PARTITIONED|
+ assign [$$17] <- [$$d.getField("id")] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ASSIGN |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ data-scan []<-[$$d] <- test.Department prefix-filter on: eq($$d.getField("department"), "accounting") [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- DATASOURCE_SCAN |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ empty-tuple-source [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/s3/filter/one-field/one-field.020.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/s3/filter/one-field/one-field.020.adm
new file mode 100644
index 0000000..dde5d2a
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/s3/filter/one-field/one-field.020.adm
@@ -0,0 +1 @@
+{ "id": 1, "department": "accounting", "name": { "first": "John", "last": "Smith" } }
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/s3/filter/one-field/one-field.021.plan b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/s3/filter/one-field/one-field.021.plan
new file mode 100644
index 0000000..a99cc0c
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/s3/filter/one-field/one-field.021.plan
@@ -0,0 +1,24 @@
+distribute result [$$d] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+-- DISTRIBUTE_RESULT |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ project ([$$d]) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- STREAM_PROJECT |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- SORT_MERGE_EXCHANGE [$$22(ASC) ] |PARTITIONED|
+ order (ASC, $$22) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- STABLE_SORT [$$22(ASC)] |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ select (and(eq($$d.getField("department"), "accounting"), eq($$d.getField("name").getField("last"), "Smith"))) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- STREAM_SELECT |PARTITIONED|
+ assign [$$22] <- [$$d.getField("id")] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ASSIGN |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ data-scan []<-[$$d] <- test.Department prefix-filter on: eq($$d.getField("department"), "accounting") [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- DATASOURCE_SCAN |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ empty-tuple-source [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/s3/filter/one-field/one-field.030.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/s3/filter/one-field/one-field.030.adm
new file mode 100644
index 0000000..aaff1e3
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/s3/filter/one-field/one-field.030.adm
@@ -0,0 +1,5 @@
+{ "id": 1, "department": "accounting", "name": { "first": "John", "last": "Smith" } }
+{ "id": 2, "department": "accounting", "name": { "first": "Mike", "last": "Jones" } }
+{ "id": 3, "department": "accounting", "name": { "first": "Alex", "last": "Miller" } }
+{ "id": 4, "department": "engineering", "name": { "first": "Tom", "last": "Smith" } }
+{ "id": 7, "department": "hr", "name": { "first": "James", "last": "Smith" } }
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/s3/filter/one-field/one-field.031.plan b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/s3/filter/one-field/one-field.031.plan
new file mode 100644
index 0000000..a7c5727
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/s3/filter/one-field/one-field.031.plan
@@ -0,0 +1,24 @@
+distribute result [$$d] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+-- DISTRIBUTE_RESULT |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ project ([$$d]) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- STREAM_PROJECT |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- SORT_MERGE_EXCHANGE [$$22(ASC) ] |PARTITIONED|
+ order (ASC, $$22) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- STABLE_SORT [$$22(ASC)] |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ select (or(eq($$d.getField("department"), "accounting"), eq($$d.getField("name").getField("last"), "Smith"))) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- STREAM_SELECT |PARTITIONED|
+ assign [$$22] <- [$$d.getField("id")] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ASSIGN |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ data-scan []<-[$$d] <- test.Department [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- DATASOURCE_SCAN |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ empty-tuple-source [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/s3/filter/one-field/one-field.040.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/s3/filter/one-field/one-field.040.adm
new file mode 100644
index 0000000..296a908
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/s3/filter/one-field/one-field.040.adm
@@ -0,0 +1,3 @@
+{ "id": 7, "department": "hr", "name": { "first": "James", "last": "Smith" } }
+{ "id": 8, "department": "hr", "name": { "first": "David", "last": "Jones" } }
+{ "id": 9, "department": "hr", "name": { "first": "Noah", "last": "Miller" } }
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/s3/filter/one-field/one-field.041.plan b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/s3/filter/one-field/one-field.041.plan
new file mode 100644
index 0000000..a29116a
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/s3/filter/one-field/one-field.041.plan
@@ -0,0 +1,24 @@
+distribute result [$$d] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+-- DISTRIBUTE_RESULT |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ project ([$$d]) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- STREAM_PROJECT |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- SORT_MERGE_EXCHANGE [$$20(ASC) ] |PARTITIONED|
+ order (ASC, $$20) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- STABLE_SORT [$$20(ASC)] |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ select (eq(uppercase($$d.getField("department")), "HR")) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- STREAM_SELECT |PARTITIONED|
+ assign [$$20] <- [$$d.getField("id")] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ASSIGN |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ data-scan []<-[$$d] <- test.Department prefix-filter on: eq(uppercase($$d.getField("department")), "HR") [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- DATASOURCE_SCAN |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ empty-tuple-source [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/s3/filter/one-field/one-field.110.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/s3/filter/one-field/one-field.110.adm
new file mode 100644
index 0000000..0184b2a8
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/s3/filter/one-field/one-field.110.adm
@@ -0,0 +1,9 @@
+{ "id": 1, "department": "accounting", "name": { "first": "John", "last": "Smith" } }
+{ "id": 2, "department": "accounting", "name": { "first": "Mike", "last": "Jones" } }
+{ "id": 3, "department": "accounting", "name": { "first": "Alex", "last": "Miller" } }
+{ "id": 4, "department": "engineering", "name": { "first": "Tom", "last": "Smith" } }
+{ "id": 5, "department": "engineering", "name": { "first": "Alice", "last": "Jones" } }
+{ "id": 6, "department": "engineering", "name": { "first": "Sheila", "last": "Miller" } }
+{ "id": 7, "department": "hr", "name": { "first": "James", "last": "Smith" } }
+{ "id": 8, "department": "hr", "name": { "first": "David", "last": "Jones" } }
+{ "id": 9, "department": "hr", "name": { "first": "Noah", "last": "Miller" } }
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/s3/filter/one-field/one-field.111.plan b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/s3/filter/one-field/one-field.111.plan
new file mode 100644
index 0000000..8a6ad1c
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/s3/filter/one-field/one-field.111.plan
@@ -0,0 +1,22 @@
+distribute result [$$d] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+-- DISTRIBUTE_RESULT |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ project ([$$d]) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- STREAM_PROJECT |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- SORT_MERGE_EXCHANGE [$$14(ASC) ] |PARTITIONED|
+ order (ASC, $$14) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- STABLE_SORT [$$14(ASC)] |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ assign [$$14] <- [$$d.getField("id")] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ASSIGN |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ data-scan []<-[$$d] <- test.LastName [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- DATASOURCE_SCAN |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ empty-tuple-source [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/s3/filter/one-field/one-field.120.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/s3/filter/one-field/one-field.120.adm
new file mode 100644
index 0000000..1b21e89
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/s3/filter/one-field/one-field.120.adm
@@ -0,0 +1,3 @@
+{ "id": 2, "department": "accounting", "name": { "first": "Mike", "last": "Jones" } }
+{ "id": 5, "department": "engineering", "name": { "first": "Alice", "last": "Jones" } }
+{ "id": 8, "department": "hr", "name": { "first": "David", "last": "Jones" } }
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/s3/filter/one-field/one-field.121.plan b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/s3/filter/one-field/one-field.121.plan
new file mode 100644
index 0000000..b48def6
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/s3/filter/one-field/one-field.121.plan
@@ -0,0 +1,24 @@
+distribute result [$$d] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+-- DISTRIBUTE_RESULT |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ project ([$$d]) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- STREAM_PROJECT |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- SORT_MERGE_EXCHANGE [$$18(ASC) ] |PARTITIONED|
+ order (ASC, $$18) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- STABLE_SORT [$$18(ASC)] |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ select (eq($$d.getField("name").getField("last"), "Jones")) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- STREAM_SELECT |PARTITIONED|
+ assign [$$18] <- [$$d.getField("id")] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ASSIGN |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ data-scan []<-[$$d] <- test.LastName prefix-filter on: eq($$d.getField("name").getField("last"), "Jones") [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- DATASOURCE_SCAN |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ empty-tuple-source [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/s3/filter/one-field/one-field.130.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/s3/filter/one-field/one-field.130.adm
new file mode 100644
index 0000000..dde5d2a
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/s3/filter/one-field/one-field.130.adm
@@ -0,0 +1 @@
+{ "id": 1, "department": "accounting", "name": { "first": "John", "last": "Smith" } }
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/s3/filter/one-field/one-field.131.plan b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/s3/filter/one-field/one-field.131.plan
new file mode 100644
index 0000000..eca6df3
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/s3/filter/one-field/one-field.131.plan
@@ -0,0 +1,24 @@
+distribute result [$$d] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+-- DISTRIBUTE_RESULT |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ project ([$$d]) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- STREAM_PROJECT |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- SORT_MERGE_EXCHANGE [$$21(ASC) ] |PARTITIONED|
+ order (ASC, $$21) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- STABLE_SORT [$$21(ASC)] |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ select (eq(string-concat(ordered-list-constructor("john ", lowercase($$d.getField("name").getField("last")))), "john smith")) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- STREAM_SELECT |PARTITIONED|
+ assign [$$21] <- [$$d.getField("id")] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ASSIGN |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ data-scan []<-[$$d] <- test.LastName prefix-filter on: eq(string-concat(ordered-list-constructor("john ", lowercase($$d.getField("name").getField("last")))), "john smith") [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- DATASOURCE_SCAN |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ empty-tuple-source [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
index 5950211..4414ebf 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
@@ -221,6 +221,11 @@
</compilation-unit>
</test-case>
<!-- Parquet Tests End -->
+ <test-case FilePath="external-dataset/s3/filter">
+ <compilation-unit name="one-field">
+ <output-dir compare="Text">one-field</output-dir>
+ </compilation-unit>
+ </test-case>
<test-case FilePath="external-dataset">
<compilation-unit name="common/empty-string-definition">
<placeholder name="adapter" value="S3" />
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStreamFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStreamFactory.java
index 881c5a3..69393a1 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStreamFactory.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/record/reader/aws/AwsS3InputStreamFactory.java
@@ -75,7 +75,7 @@
IExternalFilterEvaluator evaluator) throws HyracksDataException {
// if no computed fields or empty files list, return the original list
- if (filesOnly.isEmpty() || !prefix.hasComputedFields()) {
+ if (filesOnly.isEmpty() || !prefix.hasComputedFields() || evaluator.isEmpty()) {
return filesOnly;
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataPrefix.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataPrefix.java
index ec56009..18dbf43 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataPrefix.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/util/ExternalDataPrefix.java
@@ -209,12 +209,14 @@
/**
* Evaluates whether the provided key satisfies the conditions of the evaluator or not
+ * TODO Check if {@link IExternalFilterEvaluator#isComputedFieldUsed(int)} is useful once we have regex extractor
*
* @param key ke
* @param evaluator evaluator
* @return true if key satisfies the evaluator conditions, false otherwise
*/
public boolean evaluate(String key, IExternalFilterEvaluator evaluator) throws HyracksDataException {
+ // TODO provide the List to avoid array creation
List<String> keySegments = extractPrefixSegments(key);
// segments of object key have to be larger than segments of the prefix
@@ -223,6 +225,7 @@
}
// extract values for all compute fields and set them in the evaluator
+ // TODO provide the List to avoid array creation
List<String> values = extractValues(keySegments);
for (int i = 0; i < computedFieldNames.size(); i++) {
evaluator.setValue(i, values.get(i));
--
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17708
To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Change-Id: Idaf2bd4cf467b8ceea99a21fb1ed1892485c6715
Gerrit-Change-Number: 17708
Gerrit-PatchSet: 1
Gerrit-Owner: Wail Alkowaileet <wa...@gmail.com>
Gerrit-MessageType: newchange