You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@asterixdb.apache.org by AsterixDB Code Review <do...@asterix-gerrit.ics.uci.edu> on 2023/10/02 19:56:31 UTC
Change in asterixdb[master]: [ASTERIXDB-3273][ASTERIXDB-3274][RT] Value embedder fixes
From Wail Alkowaileet <wa...@gmail.com>:
Wail Alkowaileet has uploaded this change for review. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17830 )
Change subject: [ASTERIXDB-3273][ASTERIXDB-3274][RT] Value embedder fixes
......................................................................
[ASTERIXDB-3273][ASTERIXDB-3274][RT] Value embedder fixes
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
- Fix class exception issue when going more than 3 level
deep when parsing nested values.
- Fix the issue when replacing nested values with
scalar values.
Change-Id: I7c4f67deaf984132c3dfefc1bb7bbf894ce8a01c
---
A asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/embed-flat/embed-flat.030.adm
A asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/parquet/embed-flat/embed-flat.041.plan
A asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/embed-flat/embed-flat.031.plan
A asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/embed-with-nested-values/embed-with-nested-values.010.adm
A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/embed-flat/embed-flat.030.query.sqlpp
A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/embed-flat/embed-flat.020.query.sqlpp
A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/embed-flat/embed-flat.041.query.sqlpp
A asterixdb/asterix-app/data/json/external-filter/embed/nested-value/Jones/0.json
A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/embed-flat/embed-flat.031.query.sqlpp
A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/embed-with-nested-values/embed-flat.010.query.sqlpp
A asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/embed-flat/embed-flat.020.adm
A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/embed-flat/embed-flat.010.query.sqlpp
A asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/parquet/embed-flat/embed-flat.030.adm
A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/embed-with-nested-values/embed-flat.000.ddl.sqlpp
A asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/parquet/embed-with-nested-values/embed-with-nested-values.010.adm
A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/embed-with-nested-values/embed-flat.000.ddl.sqlpp
M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/AbstractJsonDataParser.java
A asterixdb/asterix-app/data/json/external-filter/embed/name/Jones/0.json
A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/embed-flat/embed-flat.040.query.sqlpp
M asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/filter/ExternalFilterEvaluatorFactory.java
A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/embed-flat/embed-flat.000.ddl.sqlpp
A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/embed-flat/embed-flat.010.query.sqlpp
A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/embed-flat/embed-flat.031.query.sqlpp
A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/embed-flat/embed-flat.030.query.sqlpp
A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/embed-flat/embed-flat.020.query.sqlpp
A asterixdb/asterix-app/data/json/external-filter/embed/name/smith/0.json
A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/embed-flat/embed-flat.000.ddl.sqlpp
A asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/parquet/embed-flat/embed-flat.020.adm
A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/embed-flat/embed-flat.041.query.sqlpp
A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/embed-flat/embed-flat.040.query.sqlpp
A asterixdb/asterix-app/data/json/external-filter/embed/name/miller/0.json
A asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/embed-flat/embed-flat.041.plan
A asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/parquet/embed-flat/embed-flat.031.plan
A asterixdb/asterix-app/data/json/external-filter/embed/nested-value/miller/0.json
A asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/embed-with-nested-values/embed-flat.010.query.sqlpp
M asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/filter/embedder/ExternalFilterValueEmbedder.java
A asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/parquet/embed-flat/embed-flat.040.adm
A asterixdb/asterix-app/data/json/external-filter/embed/nested-value/smith/0.json
A asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/embed-flat/embed-flat.010.adm
A asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/parquet/embed-flat/embed-flat.010.adm
A asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/embed-flat/embed-flat.040.adm
42 files changed, 719 insertions(+), 7 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/30/17830/1
diff --git a/asterixdb/asterix-app/data/json/external-filter/embed/name/Jones/0.json b/asterixdb/asterix-app/data/json/external-filter/embed/name/Jones/0.json
new file mode 100644
index 0000000..fe94583
--- /dev/null
+++ b/asterixdb/asterix-app/data/json/external-filter/embed/name/Jones/0.json
@@ -0,0 +1,3 @@
+{"id": 1, "name": {"first": "Mike", "last": "Jones"}, "department": "accounting"}
+{"id": 2, "name": {"first": "Alice"},"department": "engineering"}
+{"id": 3, "name": null, "department": "hr"}
\ No newline at end of file
diff --git a/asterixdb/asterix-app/data/json/external-filter/embed/name/miller/0.json b/asterixdb/asterix-app/data/json/external-filter/embed/name/miller/0.json
new file mode 100644
index 0000000..478c188
--- /dev/null
+++ b/asterixdb/asterix-app/data/json/external-filter/embed/name/miller/0.json
@@ -0,0 +1,3 @@
+{"id": 4, "department": "accounting", "name": {"first": "Alex", "last": "Miller"}}
+{"id": 5, "department": "engineering", "name": {}}
+{"id": 6, "department": "hr"}
\ No newline at end of file
diff --git a/asterixdb/asterix-app/data/json/external-filter/embed/name/smith/0.json b/asterixdb/asterix-app/data/json/external-filter/embed/name/smith/0.json
new file mode 100644
index 0000000..f25bdff
--- /dev/null
+++ b/asterixdb/asterix-app/data/json/external-filter/embed/name/smith/0.json
@@ -0,0 +1,3 @@
+{"id": 7, "department": "accounting", "name": "Smith"}
+{"id": 8, "department": "engineering", "name": "Smith"}
+{"id": 9, "department": "hr", "name": "Smith"}
\ No newline at end of file
diff --git a/asterixdb/asterix-app/data/json/external-filter/embed/nested-value/Jones/0.json b/asterixdb/asterix-app/data/json/external-filter/embed/nested-value/Jones/0.json
new file mode 100644
index 0000000..308f604
--- /dev/null
+++ b/asterixdb/asterix-app/data/json/external-filter/embed/nested-value/Jones/0.json
@@ -0,0 +1,3 @@
+{"id": 1, "some":{"unrelated": {"nestedValue": 1}}, "name": {"first": "Mike", "last": "Jones"}, "department": "accounting"}
+{"id": 2, "some":{"unrelated": {"nestedValue": 1}}, "name": {"first": "Alice"},"department": "engineering"}
+{"id": 3, "some":{"unrelated": {"nestedValue": 1}}, "name": null, "department": "hr"}
\ No newline at end of file
diff --git a/asterixdb/asterix-app/data/json/external-filter/embed/nested-value/miller/0.json b/asterixdb/asterix-app/data/json/external-filter/embed/nested-value/miller/0.json
new file mode 100644
index 0000000..6585785
--- /dev/null
+++ b/asterixdb/asterix-app/data/json/external-filter/embed/nested-value/miller/0.json
@@ -0,0 +1,3 @@
+{"id": 4, "some":{"unrelated": {"nestedValue": 1}}, "department": "accounting", "name": {"first": "Alex", "last": "Miller"}}
+{"id": 5, "some":{"unrelated": {"nestedValue": 1}}, "department": "engineering", "name": {}}
+{"id": 6, "some":{"unrelated": {"nestedValue": 1}}, "department": "hr"}
\ No newline at end of file
diff --git a/asterixdb/asterix-app/data/json/external-filter/embed/nested-value/smith/0.json b/asterixdb/asterix-app/data/json/external-filter/embed/nested-value/smith/0.json
new file mode 100644
index 0000000..7d9b7a2
--- /dev/null
+++ b/asterixdb/asterix-app/data/json/external-filter/embed/nested-value/smith/0.json
@@ -0,0 +1,3 @@
+{"id": 7, "some":{"unrelated": {"nestedValue": 1}}, "department": "accounting", "name": "Smith"}
+{"id": 8, "some":{"unrelated": {"nestedValue": 1}}, "department": "engineering", "name": "Smith"}
+{"id": 9, "some":{"unrelated": {"nestedValue": 1}}, "department": "hr", "name": "Smith"}
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/embed-flat/embed-flat.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/embed-flat/embed-flat.000.ddl.sqlpp
new file mode 100644
index 0000000..6af29db
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/embed-flat/embed-flat.000.ddl.sqlpp
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Tests embedding flat values while the data has nested values for the same fields
+ */
+
+DROP DATAVERSE test IF EXISTS;
+CREATE DATAVERSE test;
+USE test;
+
+CREATE TYPE TestType AS {
+};
+
+CREATE EXTERNAL DATASET Name(TestType) USING %adapter% (
+ %template%,
+ ("container"="playground"),
+ ("definition"="external-filter/embed/name/{name:string}"),
+ ("embed-filter-values" = "true"),
+ ("format"="json")
+);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/embed-flat/embed-flat.010.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/embed-flat/embed-flat.010.query.sqlpp
new file mode 100644
index 0000000..61799e8
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/embed-flat/embed-flat.010.query.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+
+SELECT VALUE n
+FROM Name n
+ORDER BY n.id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/embed-flat/embed-flat.020.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/embed-flat/embed-flat.020.query.sqlpp
new file mode 100644
index 0000000..79e7f73
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/embed-flat/embed-flat.020.query.sqlpp
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+
+SELECT VALUE n
+FROM Name n
+WHERE lowercase(n.name) IN ["jones", "miller"]
+ORDER BY n.id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/embed-flat/embed-flat.030.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/embed-flat/embed-flat.030.query.sqlpp
new file mode 100644
index 0000000..1764ba8
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/embed-flat/embed-flat.030.query.sqlpp
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+
+SELECT VALUE n
+FROM Name n
+WHERE lowercase(n.name.last) = "smith"
+ORDER BY n.id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/embed-flat/embed-flat.031.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/embed-flat/embed-flat.031.query.sqlpp
new file mode 100644
index 0000000..e0b0936
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/embed-flat/embed-flat.031.query.sqlpp
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+EXPLAIN
+SELECT VALUE n
+FROM Name n
+WHERE lowercase(n.name.last) = "smith"
+ORDER BY n.id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/embed-flat/embed-flat.040.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/embed-flat/embed-flat.040.query.sqlpp
new file mode 100644
index 0000000..437bfb9
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/embed-flat/embed-flat.040.query.sqlpp
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+
+SELECT VALUE n
+FROM Name n
+WHERE lowercase(n.name.first) = "john"
+ORDER BY n.id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/embed-flat/embed-flat.041.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/embed-flat/embed-flat.041.query.sqlpp
new file mode 100644
index 0000000..4e1beb9
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/embed-flat/embed-flat.041.query.sqlpp
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+EXPLAIN
+SELECT VALUE n
+FROM Name n
+WHERE lowercase(n.name.first) = "john"
+ORDER BY n.id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/embed-with-nested-values/embed-flat.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/embed-with-nested-values/embed-flat.000.ddl.sqlpp
new file mode 100644
index 0000000..bb97366
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/embed-with-nested-values/embed-flat.000.ddl.sqlpp
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Tests embedding flat values while the data has nested values for the same fields
+ */
+
+DROP DATAVERSE test IF EXISTS;
+CREATE DATAVERSE test;
+USE test;
+
+CREATE TYPE TestType AS {
+};
+
+CREATE EXTERNAL DATASET Name(TestType) USING %adapter% (
+ %template%,
+ ("container"="playground"),
+ ("definition"="external-filter/embed/nested-value/{name:string}"),
+ ("embed-filter-values" = "true"),
+ ("format"="json")
+);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/embed-with-nested-values/embed-flat.010.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/embed-with-nested-values/embed-flat.010.query.sqlpp
new file mode 100644
index 0000000..61799e8
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/embed-with-nested-values/embed-flat.010.query.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+
+SELECT VALUE n
+FROM Name n
+ORDER BY n.id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/embed-flat/embed-flat.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/embed-flat/embed-flat.000.ddl.sqlpp
new file mode 100644
index 0000000..220afd8
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/embed-flat/embed-flat.000.ddl.sqlpp
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Tests embedding flat values while the data has nested values for the same fields
+ */
+
+DROP DATAVERSE test IF EXISTS;
+CREATE DATAVERSE test;
+USE test;
+
+CREATE TYPE TestType AS {
+};
+
+CREATE EXTERNAL DATASET Name(TestType) USING %adapter% (
+ %template%,
+ ("container"="playground"),
+ ("definition"="parquet-data/external-filter/embed/name/{name:string}"),
+ ("embed-filter-values" = "true"),
+ ("format"="parquet")
+);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/embed-flat/embed-flat.010.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/embed-flat/embed-flat.010.query.sqlpp
new file mode 100644
index 0000000..61799e8
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/embed-flat/embed-flat.010.query.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+
+SELECT VALUE n
+FROM Name n
+ORDER BY n.id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/embed-flat/embed-flat.020.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/embed-flat/embed-flat.020.query.sqlpp
new file mode 100644
index 0000000..79e7f73
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/embed-flat/embed-flat.020.query.sqlpp
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+
+SELECT VALUE n
+FROM Name n
+WHERE lowercase(n.name) IN ["jones", "miller"]
+ORDER BY n.id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/embed-flat/embed-flat.030.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/embed-flat/embed-flat.030.query.sqlpp
new file mode 100644
index 0000000..1764ba8
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/embed-flat/embed-flat.030.query.sqlpp
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+
+SELECT VALUE n
+FROM Name n
+WHERE lowercase(n.name.last) = "smith"
+ORDER BY n.id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/embed-flat/embed-flat.031.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/embed-flat/embed-flat.031.query.sqlpp
new file mode 100644
index 0000000..e0b0936
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/embed-flat/embed-flat.031.query.sqlpp
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+EXPLAIN
+SELECT VALUE n
+FROM Name n
+WHERE lowercase(n.name.last) = "smith"
+ORDER BY n.id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/embed-flat/embed-flat.040.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/embed-flat/embed-flat.040.query.sqlpp
new file mode 100644
index 0000000..437bfb9
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/embed-flat/embed-flat.040.query.sqlpp
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+
+SELECT VALUE n
+FROM Name n
+WHERE lowercase(n.name.first) = "john"
+ORDER BY n.id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/embed-flat/embed-flat.041.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/embed-flat/embed-flat.041.query.sqlpp
new file mode 100644
index 0000000..4e1beb9
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/embed-flat/embed-flat.041.query.sqlpp
@@ -0,0 +1,26 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+EXPLAIN
+SELECT VALUE n
+FROM Name n
+WHERE lowercase(n.name.first) = "john"
+ORDER BY n.id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/embed-with-nested-values/embed-flat.000.ddl.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/embed-with-nested-values/embed-flat.000.ddl.sqlpp
new file mode 100644
index 0000000..b4110ac
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/embed-with-nested-values/embed-flat.000.ddl.sqlpp
@@ -0,0 +1,37 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+/*
+ * Tests embedding flat values while the data has nested values for the same fields
+ */
+
+DROP DATAVERSE test IF EXISTS;
+CREATE DATAVERSE test;
+USE test;
+
+CREATE TYPE TestType AS {
+};
+
+CREATE EXTERNAL DATASET Name(TestType) USING %adapter% (
+ %template%,
+ ("container"="playground"),
+ ("definition"="parquet-data/external-filter/embed/nested-value/{name:string}"),
+ ("embed-filter-values" = "true"),
+ ("format"="parquet")
+);
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/embed-with-nested-values/embed-flat.010.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/embed-with-nested-values/embed-flat.010.query.sqlpp
new file mode 100644
index 0000000..61799e8
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/external-dataset/common/dynamic-prefixes/parquet/embed-with-nested-values/embed-flat.010.query.sqlpp
@@ -0,0 +1,25 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+USE test;
+
+
+SELECT VALUE n
+FROM Name n
+ORDER BY n.id;
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/embed-flat/embed-flat.010.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/embed-flat/embed-flat.010.adm
new file mode 100644
index 0000000..9df7d6d
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/embed-flat/embed-flat.010.adm
@@ -0,0 +1,9 @@
+{ "id": 1, "name": "Jones", "department": "accounting" }
+{ "id": 2, "name": "Jones", "department": "engineering" }
+{ "id": 3, "name": "Jones", "department": "hr" }
+{ "id": 4, "department": "accounting", "name": "miller" }
+{ "id": 5, "department": "engineering", "name": "miller" }
+{ "id": 6, "department": "hr", "name": "miller" }
+{ "id": 7, "department": "accounting", "name": "smith" }
+{ "id": 8, "department": "engineering", "name": "smith" }
+{ "id": 9, "department": "hr", "name": "smith" }
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/embed-flat/embed-flat.020.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/embed-flat/embed-flat.020.adm
new file mode 100644
index 0000000..354cb6e
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/embed-flat/embed-flat.020.adm
@@ -0,0 +1,6 @@
+{ "id": 1, "name": "Jones", "department": "accounting" }
+{ "id": 2, "name": "Jones", "department": "engineering" }
+{ "id": 3, "name": "Jones", "department": "hr" }
+{ "id": 4, "department": "accounting", "name": "miller" }
+{ "id": 5, "department": "engineering", "name": "miller" }
+{ "id": 6, "department": "hr", "name": "miller" }
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/embed-flat/embed-flat.030.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/embed-flat/embed-flat.030.adm
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/embed-flat/embed-flat.030.adm
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/embed-flat/embed-flat.031.plan b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/embed-flat/embed-flat.031.plan
new file mode 100644
index 0000000..ad09779
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/embed-flat/embed-flat.031.plan
@@ -0,0 +1,24 @@
+distribute result [$$n] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+-- DISTRIBUTE_RESULT |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ project ([$$n]) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- STREAM_PROJECT |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- SORT_MERGE_EXCHANGE [$$19(ASC) ] |PARTITIONED|
+ order (ASC, $$19) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- STABLE_SORT [$$19(ASC)] |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ select (eq(lowercase($$n.getField("name").getField("last")), "smith")) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- STREAM_SELECT |PARTITIONED|
+ assign [$$19] <- [$$n.getField("id")] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ASSIGN |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ data-scan []<-[$$n] <- test.Name embed-filter-value: true [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- DATASOURCE_SCAN |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ empty-tuple-source [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/embed-flat/embed-flat.040.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/embed-flat/embed-flat.040.adm
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/embed-flat/embed-flat.040.adm
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/embed-flat/embed-flat.041.plan b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/embed-flat/embed-flat.041.plan
new file mode 100644
index 0000000..f6c3137
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/embed-flat/embed-flat.041.plan
@@ -0,0 +1,24 @@
+distribute result [$$n] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+-- DISTRIBUTE_RESULT |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ project ([$$n]) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- STREAM_PROJECT |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- SORT_MERGE_EXCHANGE [$$19(ASC) ] |PARTITIONED|
+ order (ASC, $$19) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- STABLE_SORT [$$19(ASC)] |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ select (eq(lowercase($$n.getField("name").getField("first")), "john")) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- STREAM_SELECT |PARTITIONED|
+ assign [$$19] <- [$$n.getField("id")] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ASSIGN |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ data-scan []<-[$$n] <- test.Name embed-filter-value: true [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- DATASOURCE_SCAN |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ empty-tuple-source [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/embed-with-nested-values/embed-with-nested-values.010.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/embed-with-nested-values/embed-with-nested-values.010.adm
new file mode 100644
index 0000000..8f930d0
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/embed-with-nested-values/embed-with-nested-values.010.adm
@@ -0,0 +1,9 @@
+{ "id": 1, "some": { "unrelated": { "nestedValue": 1 } }, "name": "Jones", "department": "accounting" }
+{ "id": 2, "some": { "unrelated": { "nestedValue": 1 } }, "name": "Jones", "department": "engineering" }
+{ "id": 3, "some": { "unrelated": { "nestedValue": 1 } }, "name": "Jones", "department": "hr" }
+{ "id": 4, "some": { "unrelated": { "nestedValue": 1 } }, "department": "accounting", "name": "miller" }
+{ "id": 5, "some": { "unrelated": { "nestedValue": 1 } }, "department": "engineering", "name": "miller" }
+{ "id": 6, "some": { "unrelated": { "nestedValue": 1 } }, "department": "hr", "name": "miller" }
+{ "id": 7, "some": { "unrelated": { "nestedValue": 1 } }, "department": "accounting", "name": "smith" }
+{ "id": 8, "some": { "unrelated": { "nestedValue": 1 } }, "department": "engineering", "name": "smith" }
+{ "id": 9, "some": { "unrelated": { "nestedValue": 1 } }, "department": "hr", "name": "smith" }
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/parquet/embed-flat/embed-flat.010.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/parquet/embed-flat/embed-flat.010.adm
new file mode 100644
index 0000000..4cd1fc1
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/parquet/embed-flat/embed-flat.010.adm
@@ -0,0 +1,9 @@
+{ "id": 1, "name": "Jones", "department": "accounting" }
+{ "id": 2, "name": "Jones", "department": "engineering" }
+{ "id": 3, "department": "hr", "name": "Jones" }
+{ "id": 4, "department": "accounting", "name": "miller" }
+{ "id": 5, "department": "engineering", "name": "miller" }
+{ "id": 6, "department": "hr", "name": "miller" }
+{ "id": 7, "department": "accounting", "name": "smith" }
+{ "id": 8, "department": "engineering", "name": "smith" }
+{ "id": 9, "department": "hr", "name": "smith" }
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/parquet/embed-flat/embed-flat.020.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/parquet/embed-flat/embed-flat.020.adm
new file mode 100644
index 0000000..9f43a87
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/parquet/embed-flat/embed-flat.020.adm
@@ -0,0 +1,6 @@
+{ "id": 1, "name": "Jones", "department": "accounting" }
+{ "id": 2, "name": "Jones", "department": "engineering" }
+{ "id": 3, "department": "hr", "name": "Jones" }
+{ "id": 4, "department": "accounting", "name": "miller" }
+{ "id": 5, "department": "engineering", "name": "miller" }
+{ "id": 6, "department": "hr", "name": "miller" }
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/parquet/embed-flat/embed-flat.030.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/parquet/embed-flat/embed-flat.030.adm
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/parquet/embed-flat/embed-flat.030.adm
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/parquet/embed-flat/embed-flat.031.plan b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/parquet/embed-flat/embed-flat.031.plan
new file mode 100644
index 0000000..ad09779
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/parquet/embed-flat/embed-flat.031.plan
@@ -0,0 +1,24 @@
+distribute result [$$n] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+-- DISTRIBUTE_RESULT |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ project ([$$n]) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- STREAM_PROJECT |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- SORT_MERGE_EXCHANGE [$$19(ASC) ] |PARTITIONED|
+ order (ASC, $$19) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- STABLE_SORT [$$19(ASC)] |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ select (eq(lowercase($$n.getField("name").getField("last")), "smith")) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- STREAM_SELECT |PARTITIONED|
+ assign [$$19] <- [$$n.getField("id")] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ASSIGN |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ data-scan []<-[$$n] <- test.Name embed-filter-value: true [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- DATASOURCE_SCAN |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ empty-tuple-source [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/parquet/embed-flat/embed-flat.040.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/parquet/embed-flat/embed-flat.040.adm
new file mode 100644
index 0000000..e69de29
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/parquet/embed-flat/embed-flat.040.adm
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/parquet/embed-flat/embed-flat.041.plan b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/parquet/embed-flat/embed-flat.041.plan
new file mode 100644
index 0000000..f6c3137
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/parquet/embed-flat/embed-flat.041.plan
@@ -0,0 +1,24 @@
+distribute result [$$n] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+-- DISTRIBUTE_RESULT |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ project ([$$n]) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- STREAM_PROJECT |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- SORT_MERGE_EXCHANGE [$$19(ASC) ] |PARTITIONED|
+ order (ASC, $$19) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- STABLE_SORT [$$19(ASC)] |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ select (eq(lowercase($$n.getField("name").getField("first")), "john")) [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- STREAM_SELECT |PARTITIONED|
+ assign [$$19] <- [$$n.getField("id")] [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ASSIGN |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ data-scan []<-[$$n] <- test.Name embed-filter-value: true [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- DATASOURCE_SCAN |PARTITIONED|
+ exchange [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- ONE_TO_ONE_EXCHANGE |PARTITIONED|
+ empty-tuple-source [cardinality: 0.0, op-cost: 0.0, total-cost: 0.0]
+ -- EMPTY_TUPLE_SOURCE |PARTITIONED|
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/parquet/embed-with-nested-values/embed-with-nested-values.010.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/parquet/embed-with-nested-values/embed-with-nested-values.010.adm
new file mode 100644
index 0000000..1f7a5ba
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/external-dataset/common/dynamic-prefixes/parquet/embed-with-nested-values/embed-with-nested-values.010.adm
@@ -0,0 +1,9 @@
+{ "id": 1, "some": { "unrelated": { "nestedValue": 1 } }, "name": "Jones", "department": "accounting" }
+{ "id": 2, "some": { "unrelated": { "nestedValue": 1 } }, "name": "Jones", "department": "engineering" }
+{ "id": 3, "some": { "unrelated": { "nestedValue": 1 } }, "department": "hr", "name": "Jones" }
+{ "id": 4, "some": { "unrelated": { "nestedValue": 1 } }, "department": "accounting", "name": "miller" }
+{ "id": 5, "some": { "unrelated": { "nestedValue": 1 } }, "department": "engineering", "name": "miller" }
+{ "id": 6, "some": { "unrelated": { "nestedValue": 1 } }, "department": "hr", "name": "miller" }
+{ "id": 7, "some": { "unrelated": { "nestedValue": 1 } }, "department": "accounting", "name": "smith" }
+{ "id": 8, "some": { "unrelated": { "nestedValue": 1 } }, "department": "engineering", "name": "smith" }
+{ "id": 9, "some": { "unrelated": { "nestedValue": 1 } }, "department": "hr", "name": "smith" }
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
index 2af4b9f..1d0d038 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_external_dataset_s3.xml
@@ -257,6 +257,18 @@
</compilation-unit>
</test-case>
<test-case FilePath="external-dataset/common/dynamic-prefixes">
+ <compilation-unit name="embed-flat">
+ <placeholder name="adapter" value="S3" />
+ <output-dir compare="Text">embed-flat</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="external-dataset/common/dynamic-prefixes">
+ <compilation-unit name="embed-with-nested-values">
+ <placeholder name="adapter" value="S3" />
+ <output-dir compare="Text">embed-with-nested-values</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="external-dataset/common/dynamic-prefixes">
<compilation-unit name="not-in">
<placeholder name="adapter" value="S3" />
<output-dir compare="Text">not-in</output-dir>
@@ -335,6 +347,18 @@
</compilation-unit>
</test-case>
<test-case FilePath="external-dataset/common/dynamic-prefixes/parquet">
+ <compilation-unit name="embed-flat">
+ <placeholder name="adapter" value="S3" />
+ <output-dir compare="Text">embed-flat</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="external-dataset/common/dynamic-prefixes/parquet">
+ <compilation-unit name="embed-with-nested-values">
+ <placeholder name="adapter" value="S3" />
+ <output-dir compare="Text">embed-with-nested-values</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="external-dataset/common/dynamic-prefixes/parquet">
<compilation-unit name="computed-field-between-static-parts">
<placeholder name="adapter" value="S3" />
<output-dir compare="Text">computed-field-between-static-parts</output-dir>
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/filter/ExternalFilterEvaluatorFactory.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/filter/ExternalFilterEvaluatorFactory.java
index 589b382..c8b47b6 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/filter/ExternalFilterEvaluatorFactory.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/filter/ExternalFilterEvaluatorFactory.java
@@ -69,7 +69,7 @@
@Override
public IExternalFilterValueEmbedder createValueEmbedder(IWarningCollector warningCollector) {
if (embedFilterValues) {
- return new ExternalFilterValueEmbedder(allPaths, leafs, prefix, warningCollector);
+ return new ExternalFilterValueEmbedder(allPaths, leafs, prefix);
}
return NoOpFilterValueEmbedder.INSTANCE;
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/filter/embedder/ExternalFilterValueEmbedder.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/filter/embedder/ExternalFilterValueEmbedder.java
index 2e9864b..c8b6a7e 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/filter/embedder/ExternalFilterValueEmbedder.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/input/filter/embedder/ExternalFilterValueEmbedder.java
@@ -31,12 +31,10 @@
import org.apache.asterix.om.types.IAType;
import org.apache.asterix.om.utils.ProjectionFiltrationTypeUtil;
import org.apache.hyracks.api.exceptions.HyracksDataException;
-import org.apache.hyracks.api.exceptions.IWarningCollector;
import org.apache.hyracks.data.std.api.IValueReference;
public class ExternalFilterValueEmbedder implements IExternalFilterValueEmbedder {
private final ARecordType allPaths;
- private final IWarningCollector warningCollector;
private final Map<IAType, BitSet> setValues;
private final EmbeddedValueBuilder builder;
private final Stack<ARecordType> parents;
@@ -47,9 +45,8 @@
private BitSet currentSetValues;
public ExternalFilterValueEmbedder(ARecordType allPaths, List<ProjectionFiltrationTypeUtil.RenamedType> leafs,
- ExternalDataPrefix prefix, IWarningCollector warningCollector) {
+ ExternalDataPrefix prefix) {
this.allPaths = allPaths;
- this.warningCollector = warningCollector;
setValues = new HashMap<>();
builder = new EmbeddedValueBuilder(allPaths, prefix, setValues);
parents = new Stack<>();
@@ -114,7 +111,7 @@
@Override
public void enterObject() {
level++;
- if (level != parents.size()) {
+ if (level != parents.size() && currentType != BuiltinType.AMISSING) {
parents.push((ARecordType) currentType);
currentType = BuiltinType.AMISSING;
}
@@ -122,7 +119,7 @@
@Override
public void exitObject() {
- if (currentType != allPaths) {
+ if (currentType != allPaths && level == parents.size()) {
currentType = parents.pop();
currentSetValues = setValues.get(currentType);
}
diff --git a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/AbstractJsonDataParser.java b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/AbstractJsonDataParser.java
index de9962c..412fcbf 100644
--- a/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/AbstractJsonDataParser.java
+++ b/asterixdb/asterix-external-data/src/main/java/org/apache/asterix/external/parser/AbstractJsonDataParser.java
@@ -217,6 +217,8 @@
if (valueEmbedder.shouldEmbed(fieldName, currentToken().getTypeTag())) {
// It is an embedded value, set it
fieldValue = valueEmbedder.getEmbeddedValue();
+ // This would skip the children of a nested value (if the value is nested)
+ jsonParser.skipChildren();
} else {
fieldValue = valueBuffer;
parseValue(BuiltinType.ANY, valueBuffer.getDataOutput());
--
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/17830
To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Change-Id: I7c4f67deaf984132c3dfefc1bb7bbf894ce8a01c
Gerrit-Change-Number: 17830
Gerrit-PatchSet: 1
Gerrit-Owner: Wail Alkowaileet <wa...@gmail.com>
Gerrit-MessageType: newchange