You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@asterixdb.apache.org by AsterixDB Code Review <do...@asterix-gerrit.ics.uci.edu> on 2021/08/17 02:36:55 UTC

Change in asterixdb[master]: [WIP] Introduce LazyVisitablePointable

From Wael Alkowaileet <wa...@gmail.com>:

Wael Alkowaileet has uploaded this change for review. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/12865 )


Change subject: [WIP] Introduce LazyVisitablePointable
......................................................................

[WIP] Introduce LazyVisitablePointable

- user mode changes: no
- storage format changes: no
- interface changes: no

Details:
- The lazy visitable pointable allows for traversing a record
  and process it in a single pass compared to the
  IVisitablePointable, which requires two passes - one for
  calling set() and another when calling accept().
- Does not require to write the nested values into
  temporary buffers.
- However, it is suitable when traversing the records in DFS.

Change-Id: Ib0877e6cacc2cb1a54c9c8c820f24b239c73f58a
---
A asterixdb/asterix-app/src/test/java/org/apache/asterix/test/om/pointable/lazy/LazyVisitablePointableTest.java
A asterixdb/asterix-app/src/test/java/org/apache/asterix/test/om/pointable/lazy/RecordTypeInference.java
A asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/lazy/AbstractLazyNestedVisitablePointable.java
A asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/lazy/AbstractLazyVisitablePointable.java
A asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/lazy/AbstractListLazyVisitablePointable.java
A asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/lazy/FixedListLazyVisitablePointable.java
A asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/lazy/FlatLazyVisitablePointable.java
A asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/lazy/GenericLazyVisitablePointable.java
A asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/lazy/ILazyVisitablePointableVisitor.java
A asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/lazy/RecordLazyVisitablePointable.java
A asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/lazy/TypedRecordLazyVisitablePointable.java
A asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/lazy/VariableListLazyVisitablePointable.java
M asterixdb/asterix-om/src/main/java/org/apache/asterix/om/types/BuiltinType.java
13 files changed, 1,233 insertions(+), 1 deletion(-)



  git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/65/12865/1

diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/om/pointable/lazy/LazyVisitablePointableTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/om/pointable/lazy/LazyVisitablePointableTest.java
new file mode 100644
index 0000000..bfa771a
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/om/pointable/lazy/LazyVisitablePointableTest.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.test.om.pointable.lazy;
+
+import java.io.FileInputStream;
+import java.io.IOException;
+
+import org.apache.asterix.external.parser.JSONDataParser;
+import org.apache.asterix.om.pointables.ARecordVisitablePointable;
+import org.apache.asterix.om.pointables.base.DefaultOpenFieldType;
+import org.apache.asterix.om.pointables.base.IVisitablePointable;
+import org.apache.asterix.om.pointables.cast.ACastVisitor;
+import org.apache.asterix.om.pointables.lazy.RecordLazyVisitablePointable;
+import org.apache.asterix.om.pointables.lazy.TypedRecordLazyVisitablePointable;
+import org.apache.asterix.om.types.ARecordType;
+import org.apache.asterix.om.types.IAType;
+import org.apache.asterix.runtime.evaluators.comparisons.DeepEqualAssessor;
+import org.apache.hyracks.algebricks.common.utils.Triple;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
+import org.junit.Assert;
+import org.junit.Test;
+
+import com.fasterxml.jackson.core.JsonFactory;
+
+public class LazyVisitablePointableTest {
+    private static final String BASE_DIR = "data/";
+    private static final String[] FILE_PATHS;
+    private final JSONDataParser parser;
+    private final ACastVisitor castVisitor;
+    private final RecordTypeInference schemaInference;
+    private final DeepEqualAssessor deepEqualAssessor;
+    private final RecordLazyVisitablePointable openLazyPointable;
+    private final ARecordVisitablePointable openPointable;
+    private final ArrayBackedValueStorage recordStorage;
+    private final Triple<IVisitablePointable, IAType, Boolean> arg;
+
+    static {
+        FILE_PATHS = new String[] { BASE_DIR + "hdfs/parquet/dummy_tweet.json", BASE_DIR + "nested01/person2.adm",
+                BASE_DIR + "yelp-checkin/use-case-1.json", BASE_DIR + "yelp-checkin/use-case-2.json",
+                BASE_DIR + "yelp-checkin/use-case-3.json", BASE_DIR + "yelp-checkin/use-case-4.json" };
+    }
+
+    public LazyVisitablePointableTest() {
+        parser = new JSONDataParser(DefaultOpenFieldType.NESTED_OPEN_RECORD_TYPE, new JsonFactory());
+        castVisitor = new ACastVisitor();
+        schemaInference = new RecordTypeInference();
+        deepEqualAssessor = new DeepEqualAssessor();
+        openLazyPointable = new RecordLazyVisitablePointable(true);
+        openPointable = new ARecordVisitablePointable(DefaultOpenFieldType.NESTED_OPEN_RECORD_TYPE);
+        recordStorage = new ArrayBackedValueStorage();
+        arg = new Triple<>(null, null, null);
+        arg.third = Boolean.FALSE;
+    }
+
+    private void prepareParser(String path) throws IOException {
+        FileInputStream inputStream = new FileInputStream(path);
+        parser.setInputStream(inputStream);
+    }
+
+    private void inferAndCompare() throws HyracksDataException {
+        recordStorage.reset();
+        while (parser.parse(recordStorage.getDataOutput())) {
+            openLazyPointable.set(recordStorage);
+
+            //Infer the schema
+            ARecordType inferredFromOpen = (ARecordType) openLazyPointable.accept(schemaInference, "fromOpen");
+            ARecordVisitablePointable closedPointable = new ARecordVisitablePointable(inferredFromOpen);
+            arg.first = closedPointable;
+            arg.second = inferredFromOpen;
+
+            //Cast to closed using the inferred type
+            openPointable.set(recordStorage);
+            openPointable.accept(castVisitor, arg);
+            //Ensure both closed and open records are the same
+            Assert.assertTrue(deepEqualAssessor.isEqual(openPointable, closedPointable));
+
+            //Ensure lazy pointable can handle closed types
+            TypedRecordLazyVisitablePointable closedLazyPointable =
+                    new TypedRecordLazyVisitablePointable(inferredFromOpen);
+            closedLazyPointable.set(closedPointable);
+            //Infer the type (again) but from a closed type
+            ARecordType inferredFromClosed = (ARecordType) closedLazyPointable.accept(schemaInference, "fromClosed");
+            //Ensure both inferred types are the same
+            Assert.assertTrue(inferredFromOpen.deepEqual(inferredFromClosed));
+            recordStorage.reset();
+        }
+    }
+
+    @Test
+    public void runTest() throws IOException {
+        for (String path : FILE_PATHS) {
+            prepareParser(path);
+            inferAndCompare();
+        }
+    }
+
+}
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/om/pointable/lazy/RecordTypeInference.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/om/pointable/lazy/RecordTypeInference.java
new file mode 100644
index 0000000..3323a6d
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/om/pointable/lazy/RecordTypeInference.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.test.om.pointable.lazy;
+
+import java.io.DataInputStream;
+import java.io.IOException;
+
+import org.apache.asterix.om.pointables.lazy.AbstractListLazyVisitablePointable;
+import org.apache.asterix.om.pointables.lazy.FlatLazyVisitablePointable;
+import org.apache.asterix.om.pointables.lazy.ILazyVisitablePointableVisitor;
+import org.apache.asterix.om.pointables.lazy.RecordLazyVisitablePointable;
+import org.apache.asterix.om.types.AOrderedListType;
+import org.apache.asterix.om.types.ARecordType;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.asterix.om.types.AUnorderedListType;
+import org.apache.asterix.om.types.BuiltinType;
+import org.apache.asterix.om.types.IAType;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.api.IValueReference;
+import org.apache.hyracks.data.std.util.ByteArrayAccessibleInputStream;
+import org.apache.hyracks.util.string.UTF8StringReader;
+import org.apache.hyracks.util.string.UTF8StringUtil;
+
+/**
+ * Infers the type of a single record using lazy visitable pointable
+ */
+public class RecordTypeInference implements ILazyVisitablePointableVisitor<IAType, String> {
+    private final ByteArrayAccessibleInputStream in;
+    private final DataInputStream dataIn;
+    private final UTF8StringReader utf8Reader;
+
+    public RecordTypeInference() {
+        in = new ByteArrayAccessibleInputStream(new byte[] {}, 0, 0);
+        dataIn = new DataInputStream(in);
+        utf8Reader = new UTF8StringReader();
+    }
+
+    @Override
+    public IAType visit(RecordLazyVisitablePointable pointable, String arg) throws HyracksDataException {
+        String[] fieldNames = new String[pointable.getNumberOfChildren()];
+        IAType[] fieldTypes = new IAType[pointable.getNumberOfChildren()];
+        for (int i = 0; i < pointable.getNumberOfChildren(); i++) {
+            fieldNames[i] = deserializeString(pointable.getFieldName(i));
+            fieldTypes[i] = pointable.getChildVisitablePointable(i).accept(this, fieldNames[i]);
+        }
+        return new ARecordType(arg, fieldNames, fieldTypes, false);
+    }
+
+    @Override
+    public IAType visit(AbstractListLazyVisitablePointable pointable, String arg) throws HyracksDataException {
+        IAType itemType = BuiltinType.ANY;
+        String itemTypeName = arg + "Item";
+        for (int i = 0; i < pointable.getNumberOfChildren(); i++) {
+            IAType ithItemType = pointable.getChildVisitablePointable(i).accept(this, itemTypeName);
+            if (itemType.getTypeTag() != ATypeTag.ANY && itemType.getTypeTag() != ithItemType.getTypeTag()) {
+                throw new UnsupportedOperationException("Union types are not supported");
+            }
+            itemType = ithItemType;
+        }
+        return pointable.getTypeTag() == ATypeTag.ARRAY ? new AOrderedListType(itemType, arg)
+                : new AUnorderedListType(itemType, arg);
+    }
+
+    @Override
+    public IAType visit(FlatLazyVisitablePointable pointable, String arg) throws HyracksDataException {
+        return BuiltinType.getBuiltinType(pointable.getTypeTag());
+    }
+
+    private String deserializeString(IValueReference stringValue) throws HyracksDataException {
+        in.setContent(stringValue.getByteArray(), stringValue.getStartOffset(), stringValue.getLength());
+        try {
+            return UTF8StringUtil.readUTF8(dataIn, utf8Reader);
+        } catch (IOException e) {
+            throw HyracksDataException.create(e);
+        }
+    }
+}
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/lazy/AbstractLazyNestedVisitablePointable.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/lazy/AbstractLazyNestedVisitablePointable.java
new file mode 100644
index 0000000..c1e97c1
--- /dev/null
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/lazy/AbstractLazyNestedVisitablePointable.java
@@ -0,0 +1,163 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.om.pointables.lazy;
+
+import org.apache.asterix.om.types.ARecordType;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.asterix.om.types.AbstractCollectionType;
+import org.apache.asterix.om.types.IAType;
+import org.apache.asterix.om.utils.NonTaggedFormatUtil;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.api.IValueReference;
+import org.apache.hyracks.data.std.primitive.VoidPointable;
+
+/**
+ * A common implementation for nested values (i.e., {@link ATypeTag#OBJECT}, {@link ATypeTag#ARRAY}, and
+ * {@link ATypeTag#MULTISET}).
+ * <p>
+ * Contract:
+ * <p>
+ * 1- A child's value may or may not contain a type tag. Thus, it is the responsibility of the caller to check if the
+ * child's value contains a type tag by calling {@link #isTaggedChild(int)}.
+ * 2- The returned objects from {@link #getChildVisitablePointable(int)} and {@link #getChildValue(int)}, are reused
+ * when possible. Thus, when the caller does the following for example:
+ * <p>
+ * AbstractLazyVisitablePointable child1 = visitablePointable.getChildVisitablePointable(1);
+ * AbstractLazyVisitablePointable child2 = visitablePointable.getChildVisitablePointable(2);
+ * <p>
+ * both child1 and child2 may have the same value, which is the value of the child with index 2.
+ */
+public abstract class AbstractLazyNestedVisitablePointable extends AbstractLazyVisitablePointable {
+    private final ATypeTag typeTag;
+    protected int currentIndex;
+    protected byte currentChildTypeTag;
+    protected final VoidPointable currentValue;
+
+    AbstractLazyNestedVisitablePointable(boolean tagged, ATypeTag typeTag) {
+        super(tagged);
+        this.typeTag = typeTag;
+        currentValue = new VoidPointable();
+        currentIndex = -1;
+    }
+
+    /**
+     * If the child contains a tag
+     *
+     * @param index of the child
+     * @return true if the child is tagged (open value), false otherwise
+     */
+    public abstract boolean isTaggedChild(int index);
+
+    /**
+     * @return number of children
+     */
+    public abstract int getNumberOfChildren();
+
+    /**
+     * Gets a child visitable-pointable.
+     *
+     * @param index of the child
+     */
+    public abstract AbstractLazyVisitablePointable getChildVisitablePointable(int index) throws HyracksDataException;
+
+    /**
+     * Set all the information of a nested value. We do so as usually request every information about a value. For
+     * example, we usually need the field name, the value, and its type tag from an {@link ATypeTag#OBJECT}.
+     * <p>
+     * {@link TypedRecordLazyVisitablePointable} will set fieldName, the field value, and its type tag
+     * {@link AbstractListLazyVisitablePointable} will set the item value and its type tag
+     *
+     * @param index of the child
+     */
+    abstract void setInfoIfNeeded(int index) throws HyracksDataException;
+
+    /**
+     * Returns a value reference of the child. Note that this is not a visitable-pointable reference.
+     *
+     * @param index of the child
+     */
+    public final IValueReference getChildValue(int index) throws HyracksDataException {
+        setInfoIfNeeded(index);
+        return currentValue;
+    }
+
+    /**
+     * The serialized type tag of a child
+     *
+     * @param index of the child
+     */
+    public final byte getChildSerializedTypeTag(int index) throws HyracksDataException {
+        setInfoIfNeeded(index);
+        return currentChildTypeTag;
+    }
+
+    /**
+     * The type tag of a child
+     *
+     * @param index of the child
+     */
+    public final ATypeTag getChildTypeTag(int index) throws HyracksDataException {
+        setInfoIfNeeded(index);
+        return ATypeTag.VALUE_TYPE_MAPPING[currentChildTypeTag];
+    }
+
+    /**
+     * @return The type tag that corresponds to {@code this} visitable-pointable
+     */
+    @Override
+    public final ATypeTag getTypeTag() {
+        return ATypeTag.VALUE_TYPE_MAPPING[getSerializedTypeTag()];
+    }
+
+    /**
+     * @return The serialized type tag that corresponds to {@code this} visitable-pointable
+     */
+    @Override
+    public final byte getSerializedTypeTag() {
+        return typeTag.serialize();
+    }
+
+    @Override
+    final void reset() {
+        currentIndex = -1;
+    }
+
+    /**
+     * Helper method to create a typed (i.e., non-tagged) visitable-pointable
+     *
+     * @param type the required type
+     * @return a visitable pointable that corresponds to {@code type}
+     */
+    static AbstractLazyVisitablePointable createVisitable(IAType type) {
+        ATypeTag typeTag = type.getTypeTag();
+        switch (typeTag) {
+            case OBJECT:
+                return new TypedRecordLazyVisitablePointable(false, (ARecordType) type);
+            case ARRAY:
+            case MULTISET:
+                AbstractCollectionType listType = (AbstractCollectionType) type;
+                return NonTaggedFormatUtil.isFixedSizedCollection(listType.getItemType())
+                        ? new FixedListLazyVisitablePointable(false, listType)
+                        : new VariableListLazyVisitablePointable(false, listType);
+            default:
+                return new FlatLazyVisitablePointable(false, typeTag);
+
+        }
+    }
+}
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/lazy/AbstractLazyVisitablePointable.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/lazy/AbstractLazyVisitablePointable.java
new file mode 100644
index 0000000..e804adf
--- /dev/null
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/lazy/AbstractLazyVisitablePointable.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.om.pointables.lazy;
+
+import org.apache.asterix.om.pointables.base.IVisitablePointable;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.api.IPointable;
+import org.apache.hyracks.data.std.api.IValueReference;
+
+/**
+ * Similar to {@link IVisitablePointable}. The difference is, upon calling {@link #set(byte[], int, int)}, the nested
+ * values (children) will not call {@link #set(byte[], int, int)} recursively. Instead, it will wait until the
+ * child is accessed. Thus, when a processor (a class that implements {@link ILazyVisitablePointableVisitor}) wants
+ * to traverse the object in a <b>DFS mode</b>, the traversal will be done in a single pass - compared to the two passes
+ * when using the {@link IVisitablePointable}, where one pass is done when calling
+ * {@link IVisitablePointable#set(byte[], int, int)} and another pass is done by the processor {@link
+ * ILazyVisitablePointableVisitor}. Also, the lazy visitable-pointable requires less memory as we do not allocate any
+ * temporary buffers.
+ */
+public abstract class AbstractLazyVisitablePointable implements IPointable {
+    private final boolean tagged;
+    private byte[] data;
+    private int offset;
+    private int length;
+
+    AbstractLazyVisitablePointable(boolean tagged) {
+        this.tagged = tagged;
+    }
+
+    @Override
+    public final void set(byte[] data, int offset, int length) {
+        this.data = data;
+        this.offset = offset;
+        this.length = length;
+        reset();
+        init(data, offset, length);
+    }
+
+    @Override
+    public final void set(IValueReference pointer) {
+        set(pointer.getByteArray(), pointer.getStartOffset(), pointer.getLength());
+    }
+
+    @Override
+    public final byte[] getByteArray() {
+        return data;
+    }
+
+    @Override
+    public final int getStartOffset() {
+        return offset;
+    }
+
+    @Override
+    public final int getLength() {
+        return length;
+    }
+
+    /**
+     * @return The serialized type tag
+     */
+    public abstract byte getSerializedTypeTag();
+
+    /**
+     * @return The type tag
+     */
+    public abstract ATypeTag getTypeTag();
+
+    public abstract <R, T> R accept(ILazyVisitablePointableVisitor<R, T> visitor, T arg) throws HyracksDataException;
+
+    /**
+     * @return true if the value contains tag, false otherwise
+     */
+    public final boolean isTagged() {
+        return tagged;
+    }
+
+    /**
+     * Called by {@link #set(byte[], int, int)} to prepare the visitable-pointable for a new value
+     */
+    abstract void reset();
+
+    /**
+     * Called by {@link #set(byte[], int, int)} to initialize the visitable-pointable
+     *
+     * @param data   value's data
+     * @param offset value's start offset
+     * @param length value's length
+     */
+    abstract void init(byte[] data, int offset, int length);
+}
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/lazy/AbstractListLazyVisitablePointable.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/lazy/AbstractListLazyVisitablePointable.java
new file mode 100644
index 0000000..accb524
--- /dev/null
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/lazy/AbstractListLazyVisitablePointable.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.om.pointables.lazy;
+
+import java.util.Objects;
+
+import org.apache.asterix.dataflow.data.nontagged.serde.AInt32SerializerDeserializer;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.asterix.om.types.AbstractCollectionType;
+import org.apache.asterix.om.types.IAType;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+/**
+ * Common implementation for both {@link ATypeTag#ARRAY} and {@link ATypeTag#MULTISET}
+ */
+public abstract class AbstractListLazyVisitablePointable extends AbstractLazyNestedVisitablePointable {
+    private final int headerSize;
+    private final AbstractLazyVisitablePointable itemVisitablePointable;
+    private int numberOfItems;
+    protected int itemsOffset;
+
+    AbstractListLazyVisitablePointable(boolean tagged, AbstractCollectionType listType) {
+        super(tagged, listType.getTypeTag());
+        Objects.requireNonNull(listType);
+        Objects.requireNonNull(listType.getItemType());
+        //1 for typeTag if tagged, 1 for itemTypeTag, 4 for length
+        headerSize = (isTagged() ? 1 : 0) + 1 + 4;
+        itemVisitablePointable = createVisitablePointable(listType.getItemType());
+    }
+
+    @Override
+    public final int getNumberOfChildren() {
+        return numberOfItems;
+    }
+
+    @Override
+    final void init(byte[] data, int offset, int length) {
+        int pointer = headerSize + offset;
+        numberOfItems = AInt32SerializerDeserializer.getInt(data, pointer);
+        itemsOffset = pointer + 4;
+    }
+
+    @Override
+    public AbstractLazyVisitablePointable getChildVisitablePointable(int index) throws HyracksDataException {
+        setInfoIfNeeded(index);
+        itemVisitablePointable.set(getChildValue(index));
+        return itemVisitablePointable;
+    }
+
+    @Override
+    public <R, T> R accept(ILazyVisitablePointableVisitor<R, T> visitor, T arg) throws HyracksDataException {
+        return visitor.visit(this, arg);
+    }
+
+    abstract AbstractLazyVisitablePointable createVisitablePointable(IAType itemType);
+}
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/lazy/FixedListLazyVisitablePointable.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/lazy/FixedListLazyVisitablePointable.java
new file mode 100644
index 0000000..ef630f5
--- /dev/null
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/lazy/FixedListLazyVisitablePointable.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.om.pointables.lazy;
+
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.asterix.om.types.AbstractCollectionType;
+import org.apache.asterix.om.types.IAType;
+import org.apache.asterix.om.utils.NonTaggedFormatUtil;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+/**
+ * This implementation is only for closed {@link ATypeTag#ARRAY} and {@link ATypeTag#MULTISET} with fixed-length
+ * items.
+ */
+public class FixedListLazyVisitablePointable extends AbstractListLazyVisitablePointable {
+    private final int itemSize;
+
+    public FixedListLazyVisitablePointable(boolean tagged, AbstractCollectionType listType) {
+        super(tagged, listType);
+        ATypeTag itemTag = listType.getItemType().getTypeTag();
+        currentChildTypeTag = itemTag.serialize();
+        try {
+            itemSize = NonTaggedFormatUtil.getFieldValueLength(null, -1, itemTag, false);
+        } catch (HyracksDataException e) {
+            throw new IllegalStateException(e);
+        }
+
+    }
+
+    @Override
+    void setInfoIfNeeded(int index) {
+        if (index != currentIndex) {
+            byte[] data = getByteArray();
+            int itemOffset = getStartOffset() + itemsOffset + index * itemSize;
+            currentValue.set(data, itemOffset, itemSize);
+            currentIndex = index;
+        }
+    }
+
+    @Override
+    public boolean isTaggedChild(int index) {
+        return false;
+    }
+
+    @Override
+    AbstractLazyVisitablePointable createVisitablePointable(IAType itemType) {
+        return createVisitable(itemType);
+    }
+}
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/lazy/FlatLazyVisitablePointable.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/lazy/FlatLazyVisitablePointable.java
new file mode 100644
index 0000000..5b105d4
--- /dev/null
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/lazy/FlatLazyVisitablePointable.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.om.pointables.lazy;
+
+import java.util.Objects;
+
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+public class FlatLazyVisitablePointable extends AbstractLazyVisitablePointable {
+    private final ATypeTag typeTag;
+
+    public FlatLazyVisitablePointable(boolean tagged, ATypeTag typeTag) {
+        super(tagged);
+        Objects.requireNonNull(typeTag);
+        this.typeTag = typeTag;
+    }
+
+    @Override
+    public byte getSerializedTypeTag() {
+        if (isTagged()) {
+            return getByteArray()[getStartOffset()];
+        }
+        return typeTag.serialize();
+    }
+
+    @Override
+    public ATypeTag getTypeTag() {
+        if (isTagged()) {
+            return ATypeTag.VALUE_TYPE_MAPPING[getSerializedTypeTag()];
+        }
+        return typeTag;
+    }
+
+    @Override
+    public <R, T> R accept(ILazyVisitablePointableVisitor<R, T> visitor, T arg) throws HyracksDataException {
+        return visitor.visit(this, arg);
+    }
+
+    @Override
+    void init(byte[] data, int offset, int length) {
+        //noOp
+    }
+
+    @Override
+    void reset() {
+        //noOp
+    }
+}
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/lazy/GenericLazyVisitablePointable.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/lazy/GenericLazyVisitablePointable.java
new file mode 100644
index 0000000..984b365
--- /dev/null
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/lazy/GenericLazyVisitablePointable.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.om.pointables.lazy;
+
+import org.apache.asterix.om.pointables.base.DefaultOpenFieldType;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+/**
+ * This is a generic visitable-pointable for tagged values (a.k.a. open values). Each nested visitable-pointable
+ * should only allocate a single instance of this class and reuse it for every open value.
+ */
+public class GenericLazyVisitablePointable extends AbstractLazyVisitablePointable {
+    private RecordLazyVisitablePointable object;
+    private VariableListLazyVisitablePointable array;
+    private VariableListLazyVisitablePointable multiset;
+    private FlatLazyVisitablePointable flat;
+
+    private AbstractLazyVisitablePointable current;
+
+    public GenericLazyVisitablePointable() {
+        super(true);
+    }
+
+    @Override
+    public final byte getSerializedTypeTag() {
+        return current.getSerializedTypeTag();
+    }
+
+    @Override
+    public final ATypeTag getTypeTag() {
+        return current.getTypeTag();
+    }
+
+    @Override
+    public <R, T> R accept(ILazyVisitablePointableVisitor<R, T> visitor, T arg) throws HyracksDataException {
+        switch (current.getTypeTag()) {
+            case OBJECT:
+                return visitor.visit(object, arg);
+            case ARRAY:
+                return visitor.visit(array, arg);
+            case MULTISET:
+                return visitor.visit(multiset, arg);
+            default:
+                return visitor.visit(flat, arg);
+        }
+    }
+
+    @Override
+    void init(byte[] data, int offset, int length) {
+        ATypeTag typeTag = ATypeTag.VALUE_TYPE_MAPPING[data[offset]];
+        AbstractLazyVisitablePointable visitable = getOrCreateVisitablePointable(typeTag);
+        visitable.set(data, offset, length);
+        current = visitable;
+    }
+
+    @Override
+    void reset() {
+        current = null;
+    }
+
+    private AbstractLazyVisitablePointable getOrCreateVisitablePointable(ATypeTag typeTag) {
+        switch (typeTag) {
+            case OBJECT:
+                object = object == null ? new RecordLazyVisitablePointable(true) : object;
+                return object;
+            case ARRAY:
+                array = array == null ? new VariableListLazyVisitablePointable(true,
+                        DefaultOpenFieldType.NESTED_OPEN_AORDERED_LIST_TYPE) : array;
+                return array;
+            case MULTISET:
+                multiset = multiset == null ? new VariableListLazyVisitablePointable(true,
+                        DefaultOpenFieldType.NESTED_OPEN_AUNORDERED_LIST_TYPE) : multiset;
+                return multiset;
+            default:
+                flat = flat == null ? new FlatLazyVisitablePointable(true, ATypeTag.ANY) : flat;
+                return flat;
+
+        }
+    }
+}
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/lazy/ILazyVisitablePointableVisitor.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/lazy/ILazyVisitablePointableVisitor.java
new file mode 100644
index 0000000..f9a07aa
--- /dev/null
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/lazy/ILazyVisitablePointableVisitor.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.om.pointables.lazy;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+public interface ILazyVisitablePointableVisitor<R, T> {
+    R visit(RecordLazyVisitablePointable pointable, T arg) throws HyracksDataException;
+
+    R visit(AbstractListLazyVisitablePointable pointable, T arg) throws HyracksDataException;
+
+    R visit(FlatLazyVisitablePointable pointable, T arg) throws HyracksDataException;
+}
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/lazy/RecordLazyVisitablePointable.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/lazy/RecordLazyVisitablePointable.java
new file mode 100644
index 0000000..5aa4d18
--- /dev/null
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/lazy/RecordLazyVisitablePointable.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.om.pointables.lazy;
+
+import org.apache.asterix.dataflow.data.nontagged.serde.AInt32SerializerDeserializer;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.asterix.om.utils.NonTaggedFormatUtil;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.api.IValueReference;
+import org.apache.hyracks.data.std.primitive.VoidPointable;
+
+/**
+ * This implementation is to handle {@link ATypeTag#OBJECT} with open fields only
+ */
+public class RecordLazyVisitablePointable extends AbstractLazyNestedVisitablePointable {
+    protected final VoidPointable currentFieldName;
+    protected final AbstractLazyVisitablePointable openVisitable;
+    private int openValuesOffset;
+    private int numberOfOpenChildren;
+
+    public RecordLazyVisitablePointable(boolean tagged) {
+        super(tagged, ATypeTag.OBJECT);
+        currentFieldName = new VoidPointable();
+        openVisitable = new GenericLazyVisitablePointable();
+    }
+
+    @Override
+    void setInfoIfNeeded(int index) throws HyracksDataException {
+        if (index != currentIndex) {
+            setOpenValueInfo(index);
+            currentIndex = index;
+        }
+    }
+
+    @Override
+    public boolean isTaggedChild(int index) {
+        return true;
+    }
+
+    @Override
+    public int getNumberOfChildren() {
+        return numberOfOpenChildren;
+    }
+
+    public IValueReference getFieldName(int index) throws HyracksDataException {
+        setInfoIfNeeded(index);
+        return currentFieldName;
+    }
+
+    @Override
+    public AbstractLazyVisitablePointable getChildVisitablePointable(int index) throws HyracksDataException {
+        setInfoIfNeeded(index);
+        openVisitable.set(getChildValue(index));
+        return openVisitable;
+    }
+
+    @Override
+    public final <R, T> R accept(ILazyVisitablePointableVisitor<R, T> visitor, T arg) throws HyracksDataException {
+        return visitor.visit(this, arg);
+    }
+
+    @Override
+    void init(byte[] data, int offset, int length) {
+        initOpenPart(data, offset);
+    }
+
+    protected void setOpenValueInfo(int index) throws HyracksDataException {
+        byte[] data = getByteArray();
+
+        //Compute value offset
+        int hashOffsetPairIndex = openValuesOffset + index * 8;
+        int offset = getStartOffset() + AInt32SerializerDeserializer.getInt(data, hashOffsetPairIndex + 4);
+
+        //set field name
+        int fieldNameLength = NonTaggedFormatUtil.getFieldValueLength(data, offset, ATypeTag.STRING, false);
+        currentFieldName.set(data, offset, fieldNameLength);
+        offset += fieldNameLength;
+
+        //set Type tag
+        currentChildTypeTag = data[offset];
+
+        //set value
+        int valueLength = NonTaggedFormatUtil.getFieldValueLength(data, offset, ATypeTag.ANY, true);
+        currentValue.set(data, offset, valueLength);
+    }
+
+    /* ******************************************************
+     * Init Open part
+     * ******************************************************
+     */
+    protected int initOpenPart(byte[] data, int pointer) {
+        //+1 for type tag and +4 for the length
+        int currentPointer = pointer + 1 + 4;
+        boolean isExpanded = data[currentPointer] == 1;
+        currentPointer++;
+        if (isExpanded) {
+            int openPartStart = pointer + AInt32SerializerDeserializer.getInt(data, currentPointer);
+            //Skip open part offset to the beginning of closed part
+            currentPointer += 4;
+            //Number of children in the open part
+            numberOfOpenChildren = AInt32SerializerDeserializer.getInt(data, openPartStart);
+            //Skip the numberOfOpenChildren to the hashOffset pairs
+            openValuesOffset = openPartStart + 4;
+        } else {
+            numberOfOpenChildren = 0;
+        }
+
+        return currentPointer;
+    }
+}
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/lazy/TypedRecordLazyVisitablePointable.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/lazy/TypedRecordLazyVisitablePointable.java
new file mode 100644
index 0000000..b1068b6
--- /dev/null
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/lazy/TypedRecordLazyVisitablePointable.java
@@ -0,0 +1,221 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.om.pointables.lazy;
+
+import java.util.Objects;
+
+import org.apache.asterix.dataflow.data.nontagged.serde.AInt32SerializerDeserializer;
+import org.apache.asterix.dataflow.data.nontagged.serde.AStringSerializerDeserializer;
+import org.apache.asterix.om.base.AMutableString;
+import org.apache.asterix.om.types.ARecordType;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.asterix.om.types.AUnionType;
+import org.apache.asterix.om.types.IAType;
+import org.apache.asterix.om.utils.NonTaggedFormatUtil;
+import org.apache.asterix.om.utils.RecordUtil;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.api.IPointable;
+import org.apache.hyracks.data.std.api.IValueReference;
+import org.apache.hyracks.data.std.primitive.VoidPointable;
+import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
+import org.apache.hyracks.util.string.UTF8StringWriter;
+
+/**
+ * This implementation extends {@link RecordLazyVisitablePointable} to handle {@link ATypeTag#OBJECT} with open and
+ * closed fields
+ */
+public class TypedRecordLazyVisitablePointable extends RecordLazyVisitablePointable {
+    private static final IPointable MISSING_POINTABLE = createConstantPointable(ATypeTag.MISSING);
+    private static final IPointable NULL_POINTABLE = createConstantPointable(ATypeTag.NULL);
+    private final ARecordType recordType;
+
+    //Closed values
+    private final IValueReference[] closedFieldNames;
+    private final AbstractLazyVisitablePointable[] closedVisitables;
+    private final int numberOfClosedChildren;
+    private final ATypeTag[] closedChildTags;
+    //Record builder computes the fields' offset as if the type tag exists
+    private final int actualChildOffset;
+    private int closedValuesOffset;
+
+    /**
+     * A constructor for the root record
+     *
+     * @param rootRecordType root record type
+     */
+    public TypedRecordLazyVisitablePointable(ARecordType rootRecordType) {
+        this(true, rootRecordType);
+    }
+
+    public TypedRecordLazyVisitablePointable(boolean tagged, ARecordType recordType) {
+        super(tagged);
+        Objects.requireNonNull(recordType);
+        this.recordType = recordType;
+        numberOfClosedChildren = this.recordType.getFieldTypes().length;
+        closedFieldNames = createSerializedClosedFieldNames(this.recordType);
+        closedVisitables = createClosedVisitables(this.recordType);
+        closedChildTags = createInitialClosedTypeTags(this.recordType);
+        //-1 if not tagged. The offsets were calculated as if the tag exists.
+        actualChildOffset = isTagged() ? 0 : -1;
+    }
+
+    private static IPointable createConstantPointable(ATypeTag tag) {
+        byte[] data = { tag.serialize() };
+        IPointable value = new VoidPointable();
+        value.set(data, 0, 1);
+        return value;
+    }
+
+    @Override
+    void init(byte[] data, int offset, int length) {
+        /*
+         * Skip length and the type tag if the current record contains a tag. Only the root can be tagged and typed
+         * at the same time. Nested typed records will not be tagged.
+         */
+        int skipTag = isTagged() ? 1 : 0;
+        //initOpenPart first. It will skip type tag and length.
+        int pointer = recordType.isOpen() ? initOpenPart(data, offset) : offset + skipTag + 4;
+        initClosedPart(pointer, data);
+    }
+
+    @Override
+    void setInfoIfNeeded(int index) throws HyracksDataException {
+        if (currentIndex != index) {
+            if (isTaggedChild(index)) {
+                setOpenValueInfo(index - numberOfClosedChildren);
+            } else {
+                setClosedValueInfo(index);
+            }
+            currentIndex = index;
+        }
+    }
+
+    @Override
+    public boolean isTaggedChild(int index) {
+        return index >= numberOfClosedChildren;
+    }
+
+    @Override
+    public int getNumberOfChildren() {
+        return numberOfClosedChildren + super.getNumberOfChildren();
+    }
+
+    @Override
+    public AbstractLazyVisitablePointable getChildVisitablePointable(int index) throws HyracksDataException {
+        AbstractLazyVisitablePointable visitablePointable;
+        if (isTaggedChild(index)) {
+            visitablePointable = openVisitable;
+        } else {
+            visitablePointable = closedVisitables[index];
+        }
+        visitablePointable.set(getChildValue(index));
+        return visitablePointable;
+    }
+
+    private void setClosedValueInfo(int index) throws HyracksDataException {
+        ATypeTag typeTag = closedChildTags[index];
+        if (typeTag == ATypeTag.NULL) {
+            currentValue.set(NULL_POINTABLE);
+        } else if (typeTag == ATypeTag.MISSING) {
+            currentValue.set(MISSING_POINTABLE);
+        } else {
+            byte[] data = getByteArray();
+            int offset = getStartOffset() + AInt32SerializerDeserializer.getInt(data, closedValuesOffset + 4 * index)
+                    + actualChildOffset;
+            int length = NonTaggedFormatUtil.getFieldValueLength(data, offset, typeTag, false);
+            currentValue.set(data, offset, length);
+        }
+        currentFieldName.set(closedFieldNames[index]);
+        currentChildTypeTag = typeTag.serialize();
+    }
+
+    private void initClosedPart(int pointer, byte[] data) {
+        //+4 to skip the number of closed children
+        int currentPointer = pointer + 4;
+        if (NonTaggedFormatUtil.hasOptionalField(recordType)) {
+            initClosedChildrenTags(data, currentPointer);
+            currentPointer =
+                    (numberOfClosedChildren % 4 == 0 ? numberOfClosedChildren / 4 : numberOfClosedChildren / 4 + 1);
+        }
+        closedValuesOffset = currentPointer;
+    }
+
+    private void initClosedChildrenTags(byte[] data, int nullBitMapOffset) {
+        IAType[] types = recordType.getFieldTypes();
+        for (int i = 0; i < numberOfClosedChildren; i++) {
+            byte nullMissingOrValue = data[nullBitMapOffset + i / 4];
+            if (RecordUtil.isNull(nullMissingOrValue, i)) {
+                closedChildTags[i] = ATypeTag.NULL;
+            } else if (RecordUtil.isMissing(nullMissingOrValue, i)) {
+                closedChildTags[i] = ATypeTag.MISSING;
+            } else {
+                IAType type = types[i];
+                type = type.getTypeTag() == ATypeTag.UNION ? ((AUnionType) type).getActualType() : type;
+                closedChildTags[i] = type.getTypeTag();
+            }
+        }
+    }
+
+    private static ATypeTag[] createInitialClosedTypeTags(ARecordType recordType) {
+        IAType[] types = recordType.getFieldTypes();
+        ATypeTag[] typeTags = new ATypeTag[types.length];
+        for (int i = 0; i < types.length; i++) {
+            IAType type = types[i];
+            if (type.getTypeTag() == ATypeTag.UNION) {
+                type = ((AUnionType) type).getActualType();
+            }
+            typeTags[i] = type.getTypeTag();
+        }
+        return typeTags;
+    }
+
+    private static IValueReference[] createSerializedClosedFieldNames(ARecordType recordType) {
+        UTF8StringWriter writer = new UTF8StringWriter();
+        AMutableString mutableString = new AMutableString("");
+        AStringSerializerDeserializer serDer = new AStringSerializerDeserializer(writer, null);
+
+        String[] fieldNames = recordType.getFieldNames();
+        IValueReference[] fieldNameReferences = new IValueReference[fieldNames.length];
+        for (int i = 0; i < fieldNameReferences.length; i++) {
+            mutableString.setValue(fieldNames[i]);
+            fieldNameReferences[i] = createFieldName(mutableString, serDer);
+        }
+        return fieldNameReferences;
+    }
+
+    private static IValueReference createFieldName(AMutableString mutableString, AStringSerializerDeserializer serDer) {
+        ArrayBackedValueStorage storage = new ArrayBackedValueStorage();
+        try {
+            serDer.serialize(mutableString, storage.getDataOutput());
+        } catch (HyracksDataException e) {
+            throw new IllegalStateException(e);
+        }
+        return storage;
+    }
+
+    private static AbstractLazyVisitablePointable[] createClosedVisitables(ARecordType recordType) {
+        IAType[] types = recordType.getFieldTypes();
+        AbstractLazyVisitablePointable[] visitables = new AbstractLazyVisitablePointable[types.length];
+        for (int i = 0; i < types.length; i++) {
+            visitables[i] = createVisitable(types[i]);
+        }
+        return visitables;
+    }
+
+}
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/lazy/VariableListLazyVisitablePointable.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/lazy/VariableListLazyVisitablePointable.java
new file mode 100644
index 0000000..8835cba
--- /dev/null
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/lazy/VariableListLazyVisitablePointable.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.om.pointables.lazy;
+
+import org.apache.asterix.dataflow.data.nontagged.serde.AInt32SerializerDeserializer;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.asterix.om.types.AbstractCollectionType;
+import org.apache.asterix.om.types.IAType;
+import org.apache.asterix.om.utils.NonTaggedFormatUtil;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+/**
+ * This implementation is for {@link ATypeTag#ARRAY} and {@link ATypeTag#MULTISET} with variable-length
+ * items.
+ */
+public class VariableListLazyVisitablePointable extends AbstractListLazyVisitablePointable {
+    private final ATypeTag itemTag;
+    //List builder computes the items' offset as if the type tag exists
+    private final int actualChildOffset;
+
+    public VariableListLazyVisitablePointable(boolean tagged, AbstractCollectionType listType) {
+        super(tagged, listType);
+        itemTag = listType.getItemType().getTypeTag();
+        //-1 if not tagged. The offsets were calculated as if the tag exists.
+        actualChildOffset = isTagged() ? 0 : -1;
+    }
+
+    @Override
+    void setInfoIfNeeded(int index) throws HyracksDataException {
+        if (index != currentIndex) {
+            byte[] data = getByteArray();
+            int itemOffset = getStartOffset() + AInt32SerializerDeserializer.getInt(data, itemsOffset + index * 4)
+                    + actualChildOffset;
+            ATypeTag itemTypeTag = processTypeTag(data, itemOffset);
+            int itemSize = NonTaggedFormatUtil.getFieldValueLength(data, itemOffset, itemTypeTag, isTaggedChild(index));
+            currentValue.set(data, itemOffset, itemSize);
+            currentIndex = index;
+        }
+    }
+
+    private ATypeTag processTypeTag(byte[] data, int itemOffset) {
+        if (itemTag == ATypeTag.ANY) {
+            currentChildTypeTag = data[itemOffset];
+        }
+        return itemTag;
+    }
+
+    @Override
+    public boolean isTaggedChild(int index) {
+        return itemTag == ATypeTag.ANY;
+    }
+
+    @Override
+    AbstractLazyVisitablePointable createVisitablePointable(IAType itemType) {
+        if (itemType.getTypeTag() != ATypeTag.ANY) {
+            return createVisitable(itemType);
+        }
+        return new GenericLazyVisitablePointable();
+    }
+}
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/types/BuiltinType.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/types/BuiltinType.java
index bdac9e9..7d41551 100644
--- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/types/BuiltinType.java
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/types/BuiltinType.java
@@ -1090,7 +1090,10 @@
     @SuppressWarnings("squid:S1172") // unused parameter
     public static IJsonSerializable fromJson(IPersistedResourceRegistry registry, JsonNode json) {
         byte tag = (byte) json.get(TAG_FIELD).shortValue();
-        ATypeTag typeTag = VALUE_TYPE_MAPPING[tag];
+        return getBuiltinType(VALUE_TYPE_MAPPING[tag]);
+    }
+
+    public static IAType getBuiltinType(ATypeTag typeTag) {
         switch (typeTag) {
             case TYPE:
                 return ALL_TYPE;

-- 
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/12865
To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Change-Id: Ib0877e6cacc2cb1a54c9c8c820f24b239c73f58a
Gerrit-Change-Number: 12865
Gerrit-PatchSet: 1
Gerrit-Owner: Wael Alkowaileet <wa...@gmail.com>
Gerrit-MessageType: newchange

Change in asterixdb[master]: [WIP] Introduce LazyVisitablePointable

Posted by AsterixDB Code Review <do...@asterix-gerrit.ics.uci.edu>.
From Jenkins <je...@fulliautomatix.ics.uci.edu>:

Jenkins has posted comments on this change. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/12865 )

Change subject: [WIP] Introduce LazyVisitablePointable
......................................................................


Patch Set 1: Integration-Tests+1

Integration Tests Successful

https://asterix-jenkins.ics.uci.edu/job/asterix-gerrit-integration-tests/12394/ : SUCCESS


-- 
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/12865
To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Change-Id: Ib0877e6cacc2cb1a54c9c8c820f24b239c73f58a
Gerrit-Change-Number: 12865
Gerrit-PatchSet: 1
Gerrit-Owner: Wael Alkowaileet <wa...@gmail.com>
Gerrit-Reviewer: Jenkins <je...@fulliautomatix.ics.uci.edu>
Gerrit-CC: Anon. E. Moose #1000171
Gerrit-Comment-Date: Tue, 17 Aug 2021 04:25:11 +0000
Gerrit-HasComments: No
Gerrit-Has-Labels: Yes
Gerrit-MessageType: comment

Change in asterixdb[master]: [WIP] Introduce LazyVisitablePointable

Posted by AsterixDB Code Review <do...@asterix-gerrit.ics.uci.edu>.
From Wael Alkowaileet <wa...@gmail.com>:

Wael Alkowaileet has uploaded this change for review. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/12865 )


Change subject: [WIP] Introduce LazyVisitablePointable
......................................................................

[WIP] Introduce LazyVisitablePointable

- user mode changes: no
- storage format changes: no
- interface changes: no

Details:
- The lazy visitable pointable allows for traversing a record
  and process it in a single pass compared to the
  IVisitablePointable, which requires two passes - one for
  calling set() and another when calling accept().
- Does not require to write the nested values into
  temporary buffers.
- However, it is suitable when traversing the records in DFS.

Change-Id: Ib0877e6cacc2cb1a54c9c8c820f24b239c73f58a
---
A asterixdb/asterix-app/src/test/java/org/apache/asterix/test/om/pointable/lazy/LazyVisitablePointableTest.java
A asterixdb/asterix-app/src/test/java/org/apache/asterix/test/om/pointable/lazy/RecordTypeInference.java
A asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/lazy/AbstractLazyNestedVisitablePointable.java
A asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/lazy/AbstractLazyVisitablePointable.java
A asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/lazy/AbstractListLazyVisitablePointable.java
A asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/lazy/FixedListLazyVisitablePointable.java
A asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/lazy/FlatLazyVisitablePointable.java
A asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/lazy/GenericLazyVisitablePointable.java
A asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/lazy/ILazyVisitablePointableVisitor.java
A asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/lazy/RecordLazyVisitablePointable.java
A asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/lazy/TypedRecordLazyVisitablePointable.java
A asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/lazy/VariableListLazyVisitablePointable.java
M asterixdb/asterix-om/src/main/java/org/apache/asterix/om/types/BuiltinType.java
13 files changed, 1,233 insertions(+), 1 deletion(-)



  git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/65/12865/1

diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/om/pointable/lazy/LazyVisitablePointableTest.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/om/pointable/lazy/LazyVisitablePointableTest.java
new file mode 100644
index 0000000..bfa771a
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/om/pointable/lazy/LazyVisitablePointableTest.java
@@ -0,0 +1,114 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.test.om.pointable.lazy;
+
+import java.io.FileInputStream;
+import java.io.IOException;
+
+import org.apache.asterix.external.parser.JSONDataParser;
+import org.apache.asterix.om.pointables.ARecordVisitablePointable;
+import org.apache.asterix.om.pointables.base.DefaultOpenFieldType;
+import org.apache.asterix.om.pointables.base.IVisitablePointable;
+import org.apache.asterix.om.pointables.cast.ACastVisitor;
+import org.apache.asterix.om.pointables.lazy.RecordLazyVisitablePointable;
+import org.apache.asterix.om.pointables.lazy.TypedRecordLazyVisitablePointable;
+import org.apache.asterix.om.types.ARecordType;
+import org.apache.asterix.om.types.IAType;
+import org.apache.asterix.runtime.evaluators.comparisons.DeepEqualAssessor;
+import org.apache.hyracks.algebricks.common.utils.Triple;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
+import org.junit.Assert;
+import org.junit.Test;
+
+import com.fasterxml.jackson.core.JsonFactory;
+
+public class LazyVisitablePointableTest {
+    private static final String BASE_DIR = "data/";
+    private static final String[] FILE_PATHS;
+    private final JSONDataParser parser;
+    private final ACastVisitor castVisitor;
+    private final RecordTypeInference schemaInference;
+    private final DeepEqualAssessor deepEqualAssessor;
+    private final RecordLazyVisitablePointable openLazyPointable;
+    private final ARecordVisitablePointable openPointable;
+    private final ArrayBackedValueStorage recordStorage;
+    private final Triple<IVisitablePointable, IAType, Boolean> arg;
+
+    static {
+        FILE_PATHS = new String[] { BASE_DIR + "hdfs/parquet/dummy_tweet.json", BASE_DIR + "nested01/person2.adm",
+                BASE_DIR + "yelp-checkin/use-case-1.json", BASE_DIR + "yelp-checkin/use-case-2.json",
+                BASE_DIR + "yelp-checkin/use-case-3.json", BASE_DIR + "yelp-checkin/use-case-4.json" };
+    }
+
+    public LazyVisitablePointableTest() {
+        parser = new JSONDataParser(DefaultOpenFieldType.NESTED_OPEN_RECORD_TYPE, new JsonFactory());
+        castVisitor = new ACastVisitor();
+        schemaInference = new RecordTypeInference();
+        deepEqualAssessor = new DeepEqualAssessor();
+        openLazyPointable = new RecordLazyVisitablePointable(true);
+        openPointable = new ARecordVisitablePointable(DefaultOpenFieldType.NESTED_OPEN_RECORD_TYPE);
+        recordStorage = new ArrayBackedValueStorage();
+        arg = new Triple<>(null, null, null);
+        arg.third = Boolean.FALSE;
+    }
+
+    private void prepareParser(String path) throws IOException {
+        FileInputStream inputStream = new FileInputStream(path);
+        parser.setInputStream(inputStream);
+    }
+
+    private void inferAndCompare() throws HyracksDataException {
+        recordStorage.reset();
+        while (parser.parse(recordStorage.getDataOutput())) {
+            openLazyPointable.set(recordStorage);
+
+            //Infer the schema
+            ARecordType inferredFromOpen = (ARecordType) openLazyPointable.accept(schemaInference, "fromOpen");
+            ARecordVisitablePointable closedPointable = new ARecordVisitablePointable(inferredFromOpen);
+            arg.first = closedPointable;
+            arg.second = inferredFromOpen;
+
+            //Cast to closed using the inferred type
+            openPointable.set(recordStorage);
+            openPointable.accept(castVisitor, arg);
+            //Ensure both closed and open records are the same
+            Assert.assertTrue(deepEqualAssessor.isEqual(openPointable, closedPointable));
+
+            //Ensure lazy pointable can handle closed types
+            TypedRecordLazyVisitablePointable closedLazyPointable =
+                    new TypedRecordLazyVisitablePointable(inferredFromOpen);
+            closedLazyPointable.set(closedPointable);
+            //Infer the type (again) but from a closed type
+            ARecordType inferredFromClosed = (ARecordType) closedLazyPointable.accept(schemaInference, "fromClosed");
+            //Ensure both inferred types are the same
+            Assert.assertTrue(inferredFromOpen.deepEqual(inferredFromClosed));
+            recordStorage.reset();
+        }
+    }
+
+    @Test
+    public void runTest() throws IOException {
+        for (String path : FILE_PATHS) {
+            prepareParser(path);
+            inferAndCompare();
+        }
+    }
+
+}
diff --git a/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/om/pointable/lazy/RecordTypeInference.java b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/om/pointable/lazy/RecordTypeInference.java
new file mode 100644
index 0000000..3323a6d
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/java/org/apache/asterix/test/om/pointable/lazy/RecordTypeInference.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.test.om.pointable.lazy;
+
+import java.io.DataInputStream;
+import java.io.IOException;
+
+import org.apache.asterix.om.pointables.lazy.AbstractListLazyVisitablePointable;
+import org.apache.asterix.om.pointables.lazy.FlatLazyVisitablePointable;
+import org.apache.asterix.om.pointables.lazy.ILazyVisitablePointableVisitor;
+import org.apache.asterix.om.pointables.lazy.RecordLazyVisitablePointable;
+import org.apache.asterix.om.types.AOrderedListType;
+import org.apache.asterix.om.types.ARecordType;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.asterix.om.types.AUnorderedListType;
+import org.apache.asterix.om.types.BuiltinType;
+import org.apache.asterix.om.types.IAType;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.api.IValueReference;
+import org.apache.hyracks.data.std.util.ByteArrayAccessibleInputStream;
+import org.apache.hyracks.util.string.UTF8StringReader;
+import org.apache.hyracks.util.string.UTF8StringUtil;
+
+/**
+ * Infers the type of a single record using lazy visitable pointable
+ */
+public class RecordTypeInference implements ILazyVisitablePointableVisitor<IAType, String> {
+    private final ByteArrayAccessibleInputStream in;
+    private final DataInputStream dataIn;
+    private final UTF8StringReader utf8Reader;
+
+    public RecordTypeInference() {
+        in = new ByteArrayAccessibleInputStream(new byte[] {}, 0, 0);
+        dataIn = new DataInputStream(in);
+        utf8Reader = new UTF8StringReader();
+    }
+
+    @Override
+    public IAType visit(RecordLazyVisitablePointable pointable, String arg) throws HyracksDataException {
+        String[] fieldNames = new String[pointable.getNumberOfChildren()];
+        IAType[] fieldTypes = new IAType[pointable.getNumberOfChildren()];
+        for (int i = 0; i < pointable.getNumberOfChildren(); i++) {
+            fieldNames[i] = deserializeString(pointable.getFieldName(i));
+            fieldTypes[i] = pointable.getChildVisitablePointable(i).accept(this, fieldNames[i]);
+        }
+        return new ARecordType(arg, fieldNames, fieldTypes, false);
+    }
+
+    @Override
+    public IAType visit(AbstractListLazyVisitablePointable pointable, String arg) throws HyracksDataException {
+        IAType itemType = BuiltinType.ANY;
+        String itemTypeName = arg + "Item";
+        for (int i = 0; i < pointable.getNumberOfChildren(); i++) {
+            IAType ithItemType = pointable.getChildVisitablePointable(i).accept(this, itemTypeName);
+            if (itemType.getTypeTag() != ATypeTag.ANY && itemType.getTypeTag() != ithItemType.getTypeTag()) {
+                throw new UnsupportedOperationException("Union types are not supported");
+            }
+            itemType = ithItemType;
+        }
+        return pointable.getTypeTag() == ATypeTag.ARRAY ? new AOrderedListType(itemType, arg)
+                : new AUnorderedListType(itemType, arg);
+    }
+
+    @Override
+    public IAType visit(FlatLazyVisitablePointable pointable, String arg) throws HyracksDataException {
+        return BuiltinType.getBuiltinType(pointable.getTypeTag());
+    }
+
+    private String deserializeString(IValueReference stringValue) throws HyracksDataException {
+        in.setContent(stringValue.getByteArray(), stringValue.getStartOffset(), stringValue.getLength());
+        try {
+            return UTF8StringUtil.readUTF8(dataIn, utf8Reader);
+        } catch (IOException e) {
+            throw HyracksDataException.create(e);
+        }
+    }
+}
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/lazy/AbstractLazyNestedVisitablePointable.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/lazy/AbstractLazyNestedVisitablePointable.java
new file mode 100644
index 0000000..c1e97c1
--- /dev/null
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/lazy/AbstractLazyNestedVisitablePointable.java
@@ -0,0 +1,163 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.om.pointables.lazy;
+
+import org.apache.asterix.om.types.ARecordType;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.asterix.om.types.AbstractCollectionType;
+import org.apache.asterix.om.types.IAType;
+import org.apache.asterix.om.utils.NonTaggedFormatUtil;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.api.IValueReference;
+import org.apache.hyracks.data.std.primitive.VoidPointable;
+
+/**
+ * A common implementation for nested values (i.e., {@link ATypeTag#OBJECT}, {@link ATypeTag#ARRAY}, and
+ * {@link ATypeTag#MULTISET}).
+ * <p>
+ * Contract:
+ * <p>
+ * 1- A child's value may or may not contain a type tag. Thus, it is the responsibility of the caller to check if the
+ * child's value contains a type tag by calling {@link #isTaggedChild(int)}.
+ * 2- The returned objects from {@link #getChildVisitablePointable(int)} and {@link #getChildValue(int)}, are reused
+ * when possible. Thus, when the caller does the following for example:
+ * <p>
+ * AbstractLazyVisitablePointable child1 = visitablePointable.getChildVisitablePointable(1);
+ * AbstractLazyVisitablePointable child2 = visitablePointable.getChildVisitablePointable(2);
+ * <p>
+ * both child1 and child2 may have the same value, which is the value of the child with index 2.
+ */
+public abstract class AbstractLazyNestedVisitablePointable extends AbstractLazyVisitablePointable {
+    private final ATypeTag typeTag;
+    protected int currentIndex;
+    protected byte currentChildTypeTag;
+    protected final VoidPointable currentValue;
+
+    AbstractLazyNestedVisitablePointable(boolean tagged, ATypeTag typeTag) {
+        super(tagged);
+        this.typeTag = typeTag;
+        currentValue = new VoidPointable();
+        currentIndex = -1;
+    }
+
+    /**
+     * If the child contains a tag
+     *
+     * @param index of the child
+     * @return true if the child is tagged (open value), false otherwise
+     */
+    public abstract boolean isTaggedChild(int index);
+
+    /**
+     * @return number of children
+     */
+    public abstract int getNumberOfChildren();
+
+    /**
+     * Gets a child visitable-pointable.
+     *
+     * @param index of the child
+     */
+    public abstract AbstractLazyVisitablePointable getChildVisitablePointable(int index) throws HyracksDataException;
+
+    /**
+     * Set all the information of a nested value. We do so as usually request every information about a value. For
+     * example, we usually need the field name, the value, and its type tag from an {@link ATypeTag#OBJECT}.
+     * <p>
+     * {@link TypedRecordLazyVisitablePointable} will set fieldName, the field value, and its type tag
+     * {@link AbstractListLazyVisitablePointable} will set the item value and its type tag
+     *
+     * @param index of the child
+     */
+    abstract void setInfoIfNeeded(int index) throws HyracksDataException;
+
+    /**
+     * Returns a value reference of the child. Note that this is not a visitable-pointable reference.
+     *
+     * @param index of the child
+     */
+    public final IValueReference getChildValue(int index) throws HyracksDataException {
+        setInfoIfNeeded(index);
+        return currentValue;
+    }
+
+    /**
+     * The serialized type tag of a child
+     *
+     * @param index of the child
+     */
+    public final byte getChildSerializedTypeTag(int index) throws HyracksDataException {
+        setInfoIfNeeded(index);
+        return currentChildTypeTag;
+    }
+
+    /**
+     * The type tag of a child
+     *
+     * @param index of the child
+     */
+    public final ATypeTag getChildTypeTag(int index) throws HyracksDataException {
+        setInfoIfNeeded(index);
+        return ATypeTag.VALUE_TYPE_MAPPING[currentChildTypeTag];
+    }
+
+    /**
+     * @return The type tag that corresponds to {@code this} visitable-pointable
+     */
+    @Override
+    public final ATypeTag getTypeTag() {
+        return ATypeTag.VALUE_TYPE_MAPPING[getSerializedTypeTag()];
+    }
+
+    /**
+     * @return The serialized type tag that corresponds to {@code this} visitable-pointable
+     */
+    @Override
+    public final byte getSerializedTypeTag() {
+        return typeTag.serialize();
+    }
+
+    @Override
+    final void reset() {
+        currentIndex = -1;
+    }
+
+    /**
+     * Helper method to create a typed (i.e., non-tagged) visitable-pointable
+     *
+     * @param type the required type
+     * @return a visitable pointable that corresponds to {@code type}
+     */
+    static AbstractLazyVisitablePointable createVisitable(IAType type) {
+        ATypeTag typeTag = type.getTypeTag();
+        switch (typeTag) {
+            case OBJECT:
+                return new TypedRecordLazyVisitablePointable(false, (ARecordType) type);
+            case ARRAY:
+            case MULTISET:
+                AbstractCollectionType listType = (AbstractCollectionType) type;
+                return NonTaggedFormatUtil.isFixedSizedCollection(listType.getItemType())
+                        ? new FixedListLazyVisitablePointable(false, listType)
+                        : new VariableListLazyVisitablePointable(false, listType);
+            default:
+                return new FlatLazyVisitablePointable(false, typeTag);
+
+        }
+    }
+}
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/lazy/AbstractLazyVisitablePointable.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/lazy/AbstractLazyVisitablePointable.java
new file mode 100644
index 0000000..e804adf
--- /dev/null
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/lazy/AbstractLazyVisitablePointable.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.om.pointables.lazy;
+
+import org.apache.asterix.om.pointables.base.IVisitablePointable;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.api.IPointable;
+import org.apache.hyracks.data.std.api.IValueReference;
+
+/**
+ * Similar to {@link IVisitablePointable}. The difference is, upon calling {@link #set(byte[], int, int)}, the nested
+ * values (children) will not call {@link #set(byte[], int, int)} recursively. Instead, it will wait until the
+ * child is accessed. Thus, when a processor (a class that implements {@link ILazyVisitablePointableVisitor}) wants
+ * to traverse the object in a <b>DFS mode</b>, the traversal will be done in a single pass - compared to the two passes
+ * when using the {@link IVisitablePointable}, where one pass is done when calling
+ * {@link IVisitablePointable#set(byte[], int, int)} and another pass is done by the processor {@link
+ * ILazyVisitablePointableVisitor}. Also, the lazy visitable-pointable requires less memory as we do not allocate any
+ * temporary buffers.
+ */
+public abstract class AbstractLazyVisitablePointable implements IPointable {
+    private final boolean tagged;
+    private byte[] data;
+    private int offset;
+    private int length;
+
+    AbstractLazyVisitablePointable(boolean tagged) {
+        this.tagged = tagged;
+    }
+
+    @Override
+    public final void set(byte[] data, int offset, int length) {
+        this.data = data;
+        this.offset = offset;
+        this.length = length;
+        reset();
+        init(data, offset, length);
+    }
+
+    @Override
+    public final void set(IValueReference pointer) {
+        set(pointer.getByteArray(), pointer.getStartOffset(), pointer.getLength());
+    }
+
+    @Override
+    public final byte[] getByteArray() {
+        return data;
+    }
+
+    @Override
+    public final int getStartOffset() {
+        return offset;
+    }
+
+    @Override
+    public final int getLength() {
+        return length;
+    }
+
+    /**
+     * @return The serialized type tag
+     */
+    public abstract byte getSerializedTypeTag();
+
+    /**
+     * @return The type tag
+     */
+    public abstract ATypeTag getTypeTag();
+
+    public abstract <R, T> R accept(ILazyVisitablePointableVisitor<R, T> visitor, T arg) throws HyracksDataException;
+
+    /**
+     * @return true if the value contains tag, false otherwise
+     */
+    public final boolean isTagged() {
+        return tagged;
+    }
+
+    /**
+     * Called by {@link #set(byte[], int, int)} to prepare the visitable-pointable for a new value
+     */
+    abstract void reset();
+
+    /**
+     * Called by {@link #set(byte[], int, int)} to initialize the visitable-pointable
+     *
+     * @param data   value's data
+     * @param offset value's start offset
+     * @param length value's length
+     */
+    abstract void init(byte[] data, int offset, int length);
+}
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/lazy/AbstractListLazyVisitablePointable.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/lazy/AbstractListLazyVisitablePointable.java
new file mode 100644
index 0000000..accb524
--- /dev/null
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/lazy/AbstractListLazyVisitablePointable.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.om.pointables.lazy;
+
+import java.util.Objects;
+
+import org.apache.asterix.dataflow.data.nontagged.serde.AInt32SerializerDeserializer;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.asterix.om.types.AbstractCollectionType;
+import org.apache.asterix.om.types.IAType;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+/**
+ * Common implementation for both {@link ATypeTag#ARRAY} and {@link ATypeTag#MULTISET}
+ */
+public abstract class AbstractListLazyVisitablePointable extends AbstractLazyNestedVisitablePointable {
+    private final int headerSize;
+    private final AbstractLazyVisitablePointable itemVisitablePointable;
+    private int numberOfItems;
+    protected int itemsOffset;
+
+    AbstractListLazyVisitablePointable(boolean tagged, AbstractCollectionType listType) {
+        super(tagged, listType.getTypeTag());
+        Objects.requireNonNull(listType);
+        Objects.requireNonNull(listType.getItemType());
+        //1 for typeTag if tagged, 1 for itemTypeTag, 4 for length
+        headerSize = (isTagged() ? 1 : 0) + 1 + 4;
+        itemVisitablePointable = createVisitablePointable(listType.getItemType());
+    }
+
+    @Override
+    public final int getNumberOfChildren() {
+        return numberOfItems;
+    }
+
+    @Override
+    final void init(byte[] data, int offset, int length) {
+        int pointer = headerSize + offset;
+        numberOfItems = AInt32SerializerDeserializer.getInt(data, pointer);
+        itemsOffset = pointer + 4;
+    }
+
+    @Override
+    public AbstractLazyVisitablePointable getChildVisitablePointable(int index) throws HyracksDataException {
+        setInfoIfNeeded(index);
+        itemVisitablePointable.set(getChildValue(index));
+        return itemVisitablePointable;
+    }
+
+    @Override
+    public <R, T> R accept(ILazyVisitablePointableVisitor<R, T> visitor, T arg) throws HyracksDataException {
+        return visitor.visit(this, arg);
+    }
+
+    abstract AbstractLazyVisitablePointable createVisitablePointable(IAType itemType);
+}
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/lazy/FixedListLazyVisitablePointable.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/lazy/FixedListLazyVisitablePointable.java
new file mode 100644
index 0000000..ef630f5
--- /dev/null
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/lazy/FixedListLazyVisitablePointable.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.om.pointables.lazy;
+
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.asterix.om.types.AbstractCollectionType;
+import org.apache.asterix.om.types.IAType;
+import org.apache.asterix.om.utils.NonTaggedFormatUtil;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+/**
+ * This implementation is only for closed {@link ATypeTag#ARRAY} and {@link ATypeTag#MULTISET} with fixed-length
+ * items.
+ */
+public class FixedListLazyVisitablePointable extends AbstractListLazyVisitablePointable {
+    private final int itemSize;
+
+    public FixedListLazyVisitablePointable(boolean tagged, AbstractCollectionType listType) {
+        super(tagged, listType);
+        ATypeTag itemTag = listType.getItemType().getTypeTag();
+        currentChildTypeTag = itemTag.serialize();
+        try {
+            itemSize = NonTaggedFormatUtil.getFieldValueLength(null, -1, itemTag, false);
+        } catch (HyracksDataException e) {
+            throw new IllegalStateException(e);
+        }
+
+    }
+
+    @Override
+    void setInfoIfNeeded(int index) {
+        if (index != currentIndex) {
+            byte[] data = getByteArray();
+            int itemOffset = getStartOffset() + itemsOffset + index * itemSize;
+            currentValue.set(data, itemOffset, itemSize);
+            currentIndex = index;
+        }
+    }
+
+    @Override
+    public boolean isTaggedChild(int index) {
+        return false;
+    }
+
+    @Override
+    AbstractLazyVisitablePointable createVisitablePointable(IAType itemType) {
+        return createVisitable(itemType);
+    }
+}
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/lazy/FlatLazyVisitablePointable.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/lazy/FlatLazyVisitablePointable.java
new file mode 100644
index 0000000..5b105d4
--- /dev/null
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/lazy/FlatLazyVisitablePointable.java
@@ -0,0 +1,65 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.om.pointables.lazy;
+
+import java.util.Objects;
+
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+public class FlatLazyVisitablePointable extends AbstractLazyVisitablePointable {
+    private final ATypeTag typeTag;
+
+    public FlatLazyVisitablePointable(boolean tagged, ATypeTag typeTag) {
+        super(tagged);
+        Objects.requireNonNull(typeTag);
+        this.typeTag = typeTag;
+    }
+
+    @Override
+    public byte getSerializedTypeTag() {
+        if (isTagged()) {
+            return getByteArray()[getStartOffset()];
+        }
+        return typeTag.serialize();
+    }
+
+    @Override
+    public ATypeTag getTypeTag() {
+        if (isTagged()) {
+            return ATypeTag.VALUE_TYPE_MAPPING[getSerializedTypeTag()];
+        }
+        return typeTag;
+    }
+
+    @Override
+    public <R, T> R accept(ILazyVisitablePointableVisitor<R, T> visitor, T arg) throws HyracksDataException {
+        return visitor.visit(this, arg);
+    }
+
+    @Override
+    void init(byte[] data, int offset, int length) {
+        //noOp
+    }
+
+    @Override
+    void reset() {
+        //noOp
+    }
+}
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/lazy/GenericLazyVisitablePointable.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/lazy/GenericLazyVisitablePointable.java
new file mode 100644
index 0000000..984b365
--- /dev/null
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/lazy/GenericLazyVisitablePointable.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.om.pointables.lazy;
+
+import org.apache.asterix.om.pointables.base.DefaultOpenFieldType;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+/**
+ * This is a generic visitable-pointable for tagged values (a.k.a. open values). Each nested visitable-pointable
+ * should only allocate a single instance of this class and reuse it for every open value.
+ */
+public class GenericLazyVisitablePointable extends AbstractLazyVisitablePointable {
+    private RecordLazyVisitablePointable object;
+    private VariableListLazyVisitablePointable array;
+    private VariableListLazyVisitablePointable multiset;
+    private FlatLazyVisitablePointable flat;
+
+    private AbstractLazyVisitablePointable current;
+
+    public GenericLazyVisitablePointable() {
+        super(true);
+    }
+
+    @Override
+    public final byte getSerializedTypeTag() {
+        return current.getSerializedTypeTag();
+    }
+
+    @Override
+    public final ATypeTag getTypeTag() {
+        return current.getTypeTag();
+    }
+
+    @Override
+    public <R, T> R accept(ILazyVisitablePointableVisitor<R, T> visitor, T arg) throws HyracksDataException {
+        switch (current.getTypeTag()) {
+            case OBJECT:
+                return visitor.visit(object, arg);
+            case ARRAY:
+                return visitor.visit(array, arg);
+            case MULTISET:
+                return visitor.visit(multiset, arg);
+            default:
+                return visitor.visit(flat, arg);
+        }
+    }
+
+    @Override
+    void init(byte[] data, int offset, int length) {
+        ATypeTag typeTag = ATypeTag.VALUE_TYPE_MAPPING[data[offset]];
+        AbstractLazyVisitablePointable visitable = getOrCreateVisitablePointable(typeTag);
+        visitable.set(data, offset, length);
+        current = visitable;
+    }
+
+    @Override
+    void reset() {
+        current = null;
+    }
+
+    private AbstractLazyVisitablePointable getOrCreateVisitablePointable(ATypeTag typeTag) {
+        switch (typeTag) {
+            case OBJECT:
+                object = object == null ? new RecordLazyVisitablePointable(true) : object;
+                return object;
+            case ARRAY:
+                array = array == null ? new VariableListLazyVisitablePointable(true,
+                        DefaultOpenFieldType.NESTED_OPEN_AORDERED_LIST_TYPE) : array;
+                return array;
+            case MULTISET:
+                multiset = multiset == null ? new VariableListLazyVisitablePointable(true,
+                        DefaultOpenFieldType.NESTED_OPEN_AUNORDERED_LIST_TYPE) : multiset;
+                return multiset;
+            default:
+                flat = flat == null ? new FlatLazyVisitablePointable(true, ATypeTag.ANY) : flat;
+                return flat;
+
+        }
+    }
+}
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/lazy/ILazyVisitablePointableVisitor.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/lazy/ILazyVisitablePointableVisitor.java
new file mode 100644
index 0000000..f9a07aa
--- /dev/null
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/lazy/ILazyVisitablePointableVisitor.java
@@ -0,0 +1,29 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.om.pointables.lazy;
+
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+public interface ILazyVisitablePointableVisitor<R, T> {
+    R visit(RecordLazyVisitablePointable pointable, T arg) throws HyracksDataException;
+
+    R visit(AbstractListLazyVisitablePointable pointable, T arg) throws HyracksDataException;
+
+    R visit(FlatLazyVisitablePointable pointable, T arg) throws HyracksDataException;
+}
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/lazy/RecordLazyVisitablePointable.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/lazy/RecordLazyVisitablePointable.java
new file mode 100644
index 0000000..5aa4d18
--- /dev/null
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/lazy/RecordLazyVisitablePointable.java
@@ -0,0 +1,126 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.om.pointables.lazy;
+
+import org.apache.asterix.dataflow.data.nontagged.serde.AInt32SerializerDeserializer;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.asterix.om.utils.NonTaggedFormatUtil;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.api.IValueReference;
+import org.apache.hyracks.data.std.primitive.VoidPointable;
+
+/**
+ * This implementation is to handle {@link ATypeTag#OBJECT} with open fields only
+ */
+public class RecordLazyVisitablePointable extends AbstractLazyNestedVisitablePointable {
+    protected final VoidPointable currentFieldName;
+    protected final AbstractLazyVisitablePointable openVisitable;
+    private int openValuesOffset;
+    private int numberOfOpenChildren;
+
+    public RecordLazyVisitablePointable(boolean tagged) {
+        super(tagged, ATypeTag.OBJECT);
+        currentFieldName = new VoidPointable();
+        openVisitable = new GenericLazyVisitablePointable();
+    }
+
+    @Override
+    void setInfoIfNeeded(int index) throws HyracksDataException {
+        if (index != currentIndex) {
+            setOpenValueInfo(index);
+            currentIndex = index;
+        }
+    }
+
+    @Override
+    public boolean isTaggedChild(int index) {
+        return true;
+    }
+
+    @Override
+    public int getNumberOfChildren() {
+        return numberOfOpenChildren;
+    }
+
+    public IValueReference getFieldName(int index) throws HyracksDataException {
+        setInfoIfNeeded(index);
+        return currentFieldName;
+    }
+
+    @Override
+    public AbstractLazyVisitablePointable getChildVisitablePointable(int index) throws HyracksDataException {
+        setInfoIfNeeded(index);
+        openVisitable.set(getChildValue(index));
+        return openVisitable;
+    }
+
+    @Override
+    public final <R, T> R accept(ILazyVisitablePointableVisitor<R, T> visitor, T arg) throws HyracksDataException {
+        return visitor.visit(this, arg);
+    }
+
+    @Override
+    void init(byte[] data, int offset, int length) {
+        initOpenPart(data, offset);
+    }
+
+    protected void setOpenValueInfo(int index) throws HyracksDataException {
+        byte[] data = getByteArray();
+
+        //Compute value offset
+        int hashOffsetPairIndex = openValuesOffset + index * 8;
+        int offset = getStartOffset() + AInt32SerializerDeserializer.getInt(data, hashOffsetPairIndex + 4);
+
+        //set field name
+        int fieldNameLength = NonTaggedFormatUtil.getFieldValueLength(data, offset, ATypeTag.STRING, false);
+        currentFieldName.set(data, offset, fieldNameLength);
+        offset += fieldNameLength;
+
+        //set Type tag
+        currentChildTypeTag = data[offset];
+
+        //set value
+        int valueLength = NonTaggedFormatUtil.getFieldValueLength(data, offset, ATypeTag.ANY, true);
+        currentValue.set(data, offset, valueLength);
+    }
+
+    /* ******************************************************
+     * Init Open part
+     * ******************************************************
+     */
+    protected int initOpenPart(byte[] data, int pointer) {
+        //+1 for type tag and +4 for the length
+        int currentPointer = pointer + 1 + 4;
+        boolean isExpanded = data[currentPointer] == 1;
+        currentPointer++;
+        if (isExpanded) {
+            int openPartStart = pointer + AInt32SerializerDeserializer.getInt(data, currentPointer);
+            //Skip open part offset to the beginning of closed part
+            currentPointer += 4;
+            //Number of children in the open part
+            numberOfOpenChildren = AInt32SerializerDeserializer.getInt(data, openPartStart);
+            //Skip the numberOfOpenChildren to the hashOffset pairs
+            openValuesOffset = openPartStart + 4;
+        } else {
+            numberOfOpenChildren = 0;
+        }
+
+        return currentPointer;
+    }
+}
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/lazy/TypedRecordLazyVisitablePointable.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/lazy/TypedRecordLazyVisitablePointable.java
new file mode 100644
index 0000000..b1068b6
--- /dev/null
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/lazy/TypedRecordLazyVisitablePointable.java
@@ -0,0 +1,221 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.om.pointables.lazy;
+
+import java.util.Objects;
+
+import org.apache.asterix.dataflow.data.nontagged.serde.AInt32SerializerDeserializer;
+import org.apache.asterix.dataflow.data.nontagged.serde.AStringSerializerDeserializer;
+import org.apache.asterix.om.base.AMutableString;
+import org.apache.asterix.om.types.ARecordType;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.asterix.om.types.AUnionType;
+import org.apache.asterix.om.types.IAType;
+import org.apache.asterix.om.utils.NonTaggedFormatUtil;
+import org.apache.asterix.om.utils.RecordUtil;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.api.IPointable;
+import org.apache.hyracks.data.std.api.IValueReference;
+import org.apache.hyracks.data.std.primitive.VoidPointable;
+import org.apache.hyracks.data.std.util.ArrayBackedValueStorage;
+import org.apache.hyracks.util.string.UTF8StringWriter;
+
+/**
+ * This implementation extends {@link RecordLazyVisitablePointable} to handle {@link ATypeTag#OBJECT} with open and
+ * closed fields
+ */
+public class TypedRecordLazyVisitablePointable extends RecordLazyVisitablePointable {
+    private static final IPointable MISSING_POINTABLE = createConstantPointable(ATypeTag.MISSING);
+    private static final IPointable NULL_POINTABLE = createConstantPointable(ATypeTag.NULL);
+    private final ARecordType recordType;
+
+    //Closed values
+    private final IValueReference[] closedFieldNames;
+    private final AbstractLazyVisitablePointable[] closedVisitables;
+    private final int numberOfClosedChildren;
+    private final ATypeTag[] closedChildTags;
+    //Record builder computes the fields' offset as if the type tag exists
+    private final int actualChildOffset;
+    private int closedValuesOffset;
+
+    /**
+     * A constructor for the root record
+     *
+     * @param rootRecordType root record type
+     */
+    public TypedRecordLazyVisitablePointable(ARecordType rootRecordType) {
+        this(true, rootRecordType);
+    }
+
+    public TypedRecordLazyVisitablePointable(boolean tagged, ARecordType recordType) {
+        super(tagged);
+        Objects.requireNonNull(recordType);
+        this.recordType = recordType;
+        numberOfClosedChildren = this.recordType.getFieldTypes().length;
+        closedFieldNames = createSerializedClosedFieldNames(this.recordType);
+        closedVisitables = createClosedVisitables(this.recordType);
+        closedChildTags = createInitialClosedTypeTags(this.recordType);
+        //-1 if not tagged. The offsets were calculated as if the tag exists.
+        actualChildOffset = isTagged() ? 0 : -1;
+    }
+
+    private static IPointable createConstantPointable(ATypeTag tag) {
+        byte[] data = { tag.serialize() };
+        IPointable value = new VoidPointable();
+        value.set(data, 0, 1);
+        return value;
+    }
+
+    @Override
+    void init(byte[] data, int offset, int length) {
+        /*
+         * Skip length and the type tag if the current record contains a tag. Only the root can be tagged and typed
+         * at the same time. Nested typed records will not be tagged.
+         */
+        int skipTag = isTagged() ? 1 : 0;
+        //initOpenPart first. It will skip type tag and length.
+        int pointer = recordType.isOpen() ? initOpenPart(data, offset) : offset + skipTag + 4;
+        initClosedPart(pointer, data);
+    }
+
+    @Override
+    void setInfoIfNeeded(int index) throws HyracksDataException {
+        if (currentIndex != index) {
+            if (isTaggedChild(index)) {
+                setOpenValueInfo(index - numberOfClosedChildren);
+            } else {
+                setClosedValueInfo(index);
+            }
+            currentIndex = index;
+        }
+    }
+
+    @Override
+    public boolean isTaggedChild(int index) {
+        return index >= numberOfClosedChildren;
+    }
+
+    @Override
+    public int getNumberOfChildren() {
+        return numberOfClosedChildren + super.getNumberOfChildren();
+    }
+
+    @Override
+    public AbstractLazyVisitablePointable getChildVisitablePointable(int index) throws HyracksDataException {
+        AbstractLazyVisitablePointable visitablePointable;
+        if (isTaggedChild(index)) {
+            visitablePointable = openVisitable;
+        } else {
+            visitablePointable = closedVisitables[index];
+        }
+        visitablePointable.set(getChildValue(index));
+        return visitablePointable;
+    }
+
+    private void setClosedValueInfo(int index) throws HyracksDataException {
+        ATypeTag typeTag = closedChildTags[index];
+        if (typeTag == ATypeTag.NULL) {
+            currentValue.set(NULL_POINTABLE);
+        } else if (typeTag == ATypeTag.MISSING) {
+            currentValue.set(MISSING_POINTABLE);
+        } else {
+            byte[] data = getByteArray();
+            int offset = getStartOffset() + AInt32SerializerDeserializer.getInt(data, closedValuesOffset + 4 * index)
+                    + actualChildOffset;
+            int length = NonTaggedFormatUtil.getFieldValueLength(data, offset, typeTag, false);
+            currentValue.set(data, offset, length);
+        }
+        currentFieldName.set(closedFieldNames[index]);
+        currentChildTypeTag = typeTag.serialize();
+    }
+
+    private void initClosedPart(int pointer, byte[] data) {
+        //+4 to skip the number of closed children
+        int currentPointer = pointer + 4;
+        if (NonTaggedFormatUtil.hasOptionalField(recordType)) {
+            initClosedChildrenTags(data, currentPointer);
+            currentPointer =
+                    (numberOfClosedChildren % 4 == 0 ? numberOfClosedChildren / 4 : numberOfClosedChildren / 4 + 1);
+        }
+        closedValuesOffset = currentPointer;
+    }
+
+    private void initClosedChildrenTags(byte[] data, int nullBitMapOffset) {
+        IAType[] types = recordType.getFieldTypes();
+        for (int i = 0; i < numberOfClosedChildren; i++) {
+            byte nullMissingOrValue = data[nullBitMapOffset + i / 4];
+            if (RecordUtil.isNull(nullMissingOrValue, i)) {
+                closedChildTags[i] = ATypeTag.NULL;
+            } else if (RecordUtil.isMissing(nullMissingOrValue, i)) {
+                closedChildTags[i] = ATypeTag.MISSING;
+            } else {
+                IAType type = types[i];
+                type = type.getTypeTag() == ATypeTag.UNION ? ((AUnionType) type).getActualType() : type;
+                closedChildTags[i] = type.getTypeTag();
+            }
+        }
+    }
+
+    private static ATypeTag[] createInitialClosedTypeTags(ARecordType recordType) {
+        IAType[] types = recordType.getFieldTypes();
+        ATypeTag[] typeTags = new ATypeTag[types.length];
+        for (int i = 0; i < types.length; i++) {
+            IAType type = types[i];
+            if (type.getTypeTag() == ATypeTag.UNION) {
+                type = ((AUnionType) type).getActualType();
+            }
+            typeTags[i] = type.getTypeTag();
+        }
+        return typeTags;
+    }
+
+    private static IValueReference[] createSerializedClosedFieldNames(ARecordType recordType) {
+        UTF8StringWriter writer = new UTF8StringWriter();
+        AMutableString mutableString = new AMutableString("");
+        AStringSerializerDeserializer serDer = new AStringSerializerDeserializer(writer, null);
+
+        String[] fieldNames = recordType.getFieldNames();
+        IValueReference[] fieldNameReferences = new IValueReference[fieldNames.length];
+        for (int i = 0; i < fieldNameReferences.length; i++) {
+            mutableString.setValue(fieldNames[i]);
+            fieldNameReferences[i] = createFieldName(mutableString, serDer);
+        }
+        return fieldNameReferences;
+    }
+
+    private static IValueReference createFieldName(AMutableString mutableString, AStringSerializerDeserializer serDer) {
+        ArrayBackedValueStorage storage = new ArrayBackedValueStorage();
+        try {
+            serDer.serialize(mutableString, storage.getDataOutput());
+        } catch (HyracksDataException e) {
+            throw new IllegalStateException(e);
+        }
+        return storage;
+    }
+
+    private static AbstractLazyVisitablePointable[] createClosedVisitables(ARecordType recordType) {
+        IAType[] types = recordType.getFieldTypes();
+        AbstractLazyVisitablePointable[] visitables = new AbstractLazyVisitablePointable[types.length];
+        for (int i = 0; i < types.length; i++) {
+            visitables[i] = createVisitable(types[i]);
+        }
+        return visitables;
+    }
+
+}
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/lazy/VariableListLazyVisitablePointable.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/lazy/VariableListLazyVisitablePointable.java
new file mode 100644
index 0000000..8835cba
--- /dev/null
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/pointables/lazy/VariableListLazyVisitablePointable.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements.  See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership.  The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License.  You may obtain a copy of the License at
+ *
+ *   http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied.  See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.om.pointables.lazy;
+
+import org.apache.asterix.dataflow.data.nontagged.serde.AInt32SerializerDeserializer;
+import org.apache.asterix.om.types.ATypeTag;
+import org.apache.asterix.om.types.AbstractCollectionType;
+import org.apache.asterix.om.types.IAType;
+import org.apache.asterix.om.utils.NonTaggedFormatUtil;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+
+/**
+ * This implementation is for {@link ATypeTag#ARRAY} and {@link ATypeTag#MULTISET} with variable-length
+ * items.
+ */
+public class VariableListLazyVisitablePointable extends AbstractListLazyVisitablePointable {
+    private final ATypeTag itemTag;
+    //List builder computes the items' offset as if the type tag exists
+    private final int actualChildOffset;
+
+    public VariableListLazyVisitablePointable(boolean tagged, AbstractCollectionType listType) {
+        super(tagged, listType);
+        itemTag = listType.getItemType().getTypeTag();
+        //-1 if not tagged. The offsets were calculated as if the tag exists.
+        actualChildOffset = isTagged() ? 0 : -1;
+    }
+
+    @Override
+    void setInfoIfNeeded(int index) throws HyracksDataException {
+        if (index != currentIndex) {
+            byte[] data = getByteArray();
+            int itemOffset = getStartOffset() + AInt32SerializerDeserializer.getInt(data, itemsOffset + index * 4)
+                    + actualChildOffset;
+            ATypeTag itemTypeTag = processTypeTag(data, itemOffset);
+            int itemSize = NonTaggedFormatUtil.getFieldValueLength(data, itemOffset, itemTypeTag, isTaggedChild(index));
+            currentValue.set(data, itemOffset, itemSize);
+            currentIndex = index;
+        }
+    }
+
+    private ATypeTag processTypeTag(byte[] data, int itemOffset) {
+        if (itemTag == ATypeTag.ANY) {
+            currentChildTypeTag = data[itemOffset];
+        }
+        return itemTag;
+    }
+
+    @Override
+    public boolean isTaggedChild(int index) {
+        return itemTag == ATypeTag.ANY;
+    }
+
+    @Override
+    AbstractLazyVisitablePointable createVisitablePointable(IAType itemType) {
+        if (itemType.getTypeTag() != ATypeTag.ANY) {
+            return createVisitable(itemType);
+        }
+        return new GenericLazyVisitablePointable();
+    }
+}
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/types/BuiltinType.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/types/BuiltinType.java
index bdac9e9..7d41551 100644
--- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/types/BuiltinType.java
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/om/types/BuiltinType.java
@@ -1090,7 +1090,10 @@
     @SuppressWarnings("squid:S1172") // unused parameter
     public static IJsonSerializable fromJson(IPersistedResourceRegistry registry, JsonNode json) {
         byte tag = (byte) json.get(TAG_FIELD).shortValue();
-        ATypeTag typeTag = VALUE_TYPE_MAPPING[tag];
+        return getBuiltinType(VALUE_TYPE_MAPPING[tag]);
+    }
+
+    public static IAType getBuiltinType(ATypeTag typeTag) {
         switch (typeTag) {
             case TYPE:
                 return ALL_TYPE;

-- 
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/12865
To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings

Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Change-Id: Ib0877e6cacc2cb1a54c9c8c820f24b239c73f58a
Gerrit-Change-Number: 12865
Gerrit-PatchSet: 1
Gerrit-Owner: Wael Alkowaileet <wa...@gmail.com>
Gerrit-MessageType: newchange