You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@asterixdb.apache.org by al...@apache.org on 2019/07/18 19:14:01 UTC
[asterixdb] branch master updated: [ASTERIXDB-2613][*DB] Fix
serialization of schemaless records
This is an automated email from the ASF dual-hosted git repository.
alsuliman pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/asterixdb.git
The following commit(s) were added to refs/heads/master by this push:
new c30c5ea [ASTERIXDB-2613][*DB] Fix serialization of schemaless records
c30c5ea is described below
commit c30c5ea7941efa3eeabcf100082f373586ddb640
Author: Ali Alsuliman <al...@gmail.com>
AuthorDate: Wed Jul 17 22:56:41 2019 -0700
[ASTERIXDB-2613][*DB] Fix serialization of schemaless records
- user model changes: no
- storage format changes: no
- interface changes: no
Details:
When serializing schemaless records, the record serializer should
continue serializing the fields as schemaless fields to handle fields
that are also records (nested records).
Change-Id: Ibc7257987d24905e82d5e81f27840fba81281706
Reviewed-on: https://asterix-gerrit.ics.uci.edu/3501
Contrib: Jenkins <je...@fulliautomatix.ics.uci.edu>
Tested-by: Jenkins <je...@fulliautomatix.ics.uci.edu>
Integration-Tests: Jenkins <je...@fulliautomatix.ics.uci.edu>
Reviewed-by: Ali Alsuliman <al...@gmail.com>
Reviewed-by: Dmitry Lychagin <dm...@couchbase.com>
---
...cord-serialization-ASTERIXDB-2613.1.query.sqlpp | 23 +++++++++
.../record-serialization-ASTERIXDB-2613.1.adm | 1 +
.../test/resources/runtimets/testsuite_sqlpp.xml | 5 ++
.../serde/ARecordSerializerDeserializer.java | 57 ++++++----------------
4 files changed, 45 insertions(+), 41 deletions(-)
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/record-serialization-ASTERIXDB-2613/record-serialization-ASTERIXDB-2613.1.query.sqlpp b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/record-serialization-ASTERIXDB-2613/record-serialization-ASTERIXDB-2613.1.query.sqlpp
new file mode 100644
index 0000000..b2762f8
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/queries_sqlpp/misc/record-serialization-ASTERIXDB-2613/record-serialization-ASTERIXDB-2613.1.query.sqlpp
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+// testing fix for asterix record object serialization where the record has nested records. the record needs to be
+// serialized as open since it will be inside of a list of ANY. every field should also be serialized as open
+
+select value array_put([3], {"f": {"ff": 4}});
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/record-serialization-ASTERIXDB-2613/record-serialization-ASTERIXDB-2613.1.adm b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/record-serialization-ASTERIXDB-2613/record-serialization-ASTERIXDB-2613.1.adm
new file mode 100644
index 0000000..4c65118
--- /dev/null
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/results/misc/record-serialization-ASTERIXDB-2613/record-serialization-ASTERIXDB-2613.1.adm
@@ -0,0 +1 @@
+[ 3, { "f": { "ff": 4 } } ]
\ No newline at end of file
diff --git a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
index 5d461f8..511feab 100644
--- a/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
+++ b/asterixdb/asterix-app/src/test/resources/runtimets/testsuite_sqlpp.xml
@@ -5691,6 +5691,11 @@
</compilation-unit>
</test-case>
<test-case FilePath="misc">
+ <compilation-unit name="record-serialization-ASTERIXDB-2613">
+ <output-dir compare="Text">record-serialization-ASTERIXDB-2613</output-dir>
+ </compilation-unit>
+ </test-case>
+ <test-case FilePath="misc">
<compilation-unit name="field_access-ASTERIXDB-2289">
<output-dir compare="Text">field_access-ASTERIXDB-2289</output-dir>
</compilation-unit>
diff --git a/asterixdb/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/serde/ARecordSerializerDeserializer.java b/asterixdb/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/serde/ARecordSerializerDeserializer.java
index 1d891e1..3f8102e 100644
--- a/asterixdb/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/serde/ARecordSerializerDeserializer.java
+++ b/asterixdb/asterix-om/src/main/java/org/apache/asterix/dataflow/data/nontagged/serde/ARecordSerializerDeserializer.java
@@ -21,12 +21,12 @@ package org.apache.asterix.dataflow.data.nontagged.serde;
import java.io.DataInput;
import java.io.DataOutput;
import java.io.IOException;
-import java.util.List;
import org.apache.asterix.builders.IARecordBuilder;
import org.apache.asterix.builders.RecordBuilder;
import org.apache.asterix.formats.nontagged.SerializerDeserializerProvider;
import org.apache.asterix.om.base.AMissing;
+import org.apache.asterix.om.base.AMutableString;
import org.apache.asterix.om.base.ANull;
import org.apache.asterix.om.base.ARecord;
import org.apache.asterix.om.base.AString;
@@ -38,7 +38,6 @@ import org.apache.asterix.om.types.BuiltinType;
import org.apache.asterix.om.types.IAType;
import org.apache.asterix.om.utils.NonTaggedFormatUtil;
import org.apache.asterix.om.utils.RecordUtil;
-import org.apache.hyracks.algebricks.common.utils.Pair;
import org.apache.hyracks.api.dataflow.value.IBinaryComparator;
import org.apache.hyracks.api.dataflow.value.IBinaryHashFunction;
import org.apache.hyracks.api.dataflow.value.ISerializerDeserializer;
@@ -168,19 +167,15 @@ public class ARecordSerializerDeserializer implements ISerializerDeserializer<AR
return schemaFields;
}
- @Override
- public void serialize(ARecord instance, DataOutput out) throws HyracksDataException {
- this.serialize(instance, out, false);
- }
-
// This serialize method will NOT work if <code>recordType</code> is not equal to the type of the instance.
@SuppressWarnings("unchecked")
- public void serialize(ARecord instance, DataOutput out, boolean writeTypeTag) throws HyracksDataException {
- IARecordBuilder recordBuilder = new RecordBuilder();
- ArrayBackedValueStorage fieldValue = new ArrayBackedValueStorage();
- recordBuilder.reset(recordType);
- recordBuilder.init();
+ @Override
+ public void serialize(ARecord instance, DataOutput out) throws HyracksDataException {
if (recordType != null) {
+ IARecordBuilder recordBuilder = new RecordBuilder();
+ ArrayBackedValueStorage fieldValue = new ArrayBackedValueStorage();
+ recordBuilder.reset(recordType);
+ recordBuilder.init();
IAType[] fieldTypes = recordType.getFieldTypes();
for (int fieldIndex = 0; fieldIndex < recordType.getFieldNames().length; ++fieldIndex) {
fieldValue.reset();
@@ -196,51 +191,31 @@ public class ARecordSerializerDeserializer implements ISerializerDeserializer<AR
}
recordBuilder.addField(fieldIndex, fieldValue);
}
- recordBuilder.write(out, writeTypeTag);
+ recordBuilder.write(out, false);
} else {
- serializeSchemalessRecord(instance, out, writeTypeTag);
+ serializeSchemalessRecord(instance, out);
}
}
@SuppressWarnings({ "unchecked", "rawtypes" })
- private static void serializeSchemalessRecord(ARecord record, DataOutput dataOutput, boolean writeTypeTag)
- throws HyracksDataException {
+ private static void serializeSchemalessRecord(ARecord record, DataOutput dataOutput) throws HyracksDataException {
ISerializerDeserializer<AString> stringSerde =
SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(BuiltinType.ASTRING);
RecordBuilder confRecordBuilder = new RecordBuilder();
confRecordBuilder.reset(RecordUtil.FULLY_OPEN_RECORD_TYPE);
ArrayBackedValueStorage fieldNameBytes = new ArrayBackedValueStorage();
ArrayBackedValueStorage fieldValueBytes = new ArrayBackedValueStorage();
+ AMutableString mutableString = new AMutableString(null);
for (int i = 0; i < record.getType().getFieldNames().length; i++) {
String fieldName = record.getType().getFieldNames()[i];
fieldValueBytes.reset();
fieldNameBytes.reset();
- stringSerde.serialize(new AString(fieldName), fieldNameBytes.getDataOutput());
- ISerializerDeserializer valueSerde = SerializerDeserializerProvider.INSTANCE
- .getSerializerDeserializer(record.getType().getFieldTypes()[i]);
- valueSerde.serialize(record.getValueByPos(i), fieldValueBytes.getDataOutput());
- confRecordBuilder.addField(fieldNameBytes, fieldValueBytes);
- }
- confRecordBuilder.write(dataOutput, writeTypeTag);
- }
-
- @SuppressWarnings("unchecked")
- public static void serializeSimpleSchemalessRecord(List<Pair<String, String>> record, DataOutput dataOutput,
- boolean writeTypeTag) throws HyracksDataException {
- ISerializerDeserializer<AString> stringSerde =
- SerializerDeserializerProvider.INSTANCE.getSerializerDeserializer(BuiltinType.ASTRING);
- RecordBuilder confRecordBuilder = new RecordBuilder();
- confRecordBuilder.reset(RecordUtil.FULLY_OPEN_RECORD_TYPE);
- ArrayBackedValueStorage fieldNameBytes = new ArrayBackedValueStorage();
- ArrayBackedValueStorage fieldValueBytes = new ArrayBackedValueStorage();
- for (int i = 0; i < record.size(); i++) {
- fieldValueBytes.reset();
- fieldNameBytes.reset();
- stringSerde.serialize(new AString(record.get(i).first), fieldNameBytes.getDataOutput());
- stringSerde.serialize(new AString(record.get(i).second), fieldValueBytes.getDataOutput());
+ mutableString.setValue(fieldName);
+ stringSerde.serialize(mutableString, fieldNameBytes.getDataOutput());
+ AObjectSerializerDeserializer.INSTANCE.serialize(record.getValueByPos(i), fieldValueBytes.getDataOutput());
confRecordBuilder.addField(fieldNameBytes, fieldValueBytes);
}
- confRecordBuilder.write(dataOutput, writeTypeTag);
+ confRecordBuilder.write(dataOutput, false);
}
public static IAObject[] mergeFields(IAObject[] closedFields, IAObject[] openFields) {
@@ -374,6 +349,6 @@ public class ARecordSerializerDeserializer implements ISerializerDeserializer<AR
@Override
public String toString() {
- return " ";
+ return recordType != null ? recordType.toString() : "";
}
}