You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@asterixdb.apache.org by AsterixDB Code Review <do...@asterix-gerrit.ics.uci.edu> on 2024/06/13 09:23:41 UTC
Change in asterixdb[master]: [ASTERIXDB-3430] Log schema as a one-liner JSON
From <pr...@couchbase.com>:
preetham.poluparthi@couchbase.com has uploaded this change for review. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18374 )
Change subject: [ASTERIXDB-3430] Log schema as a one-liner JSON
......................................................................
[ASTERIXDB-3430] Log schema as a one-liner JSON
Change-Id: I9c8f6d1739a3e5aafb1ead1ae6c19417c6f0a060
---
A asterixdb/asterix-column/src/test/java/org/apache/asterix/column/metadata/schema/visitor/SchemaJSONBuilderVisitor.java
M asterixdb/asterix-column/src/test/java/org/apache/asterix/column/test/dummy/TransformerTest.java
M asterixdb/asterix-column/src/main/java/org/apache/asterix/column/util/RunLengthIntArray.java
3 files changed, 300 insertions(+), 4 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/74/18374/1
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/util/RunLengthIntArray.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/util/RunLengthIntArray.java
index df238cb..bbfe245 100644
--- a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/util/RunLengthIntArray.java
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/util/RunLengthIntArray.java
@@ -32,7 +32,7 @@
* Example:
* Original Array: [1,1,1,1,1,1,2,2,2,1,1,1]
* blockValues: [1,2,1]
- * blockCounts: [6,10,13]
+ * blockCounts: [6,9,12]
*/
public final class RunLengthIntArray {
private final IntArrayList blockValues;
diff --git a/asterixdb/asterix-column/src/test/java/org/apache/asterix/column/metadata/schema/visitor/SchemaJSONBuilderVisitor.java b/asterixdb/asterix-column/src/test/java/org/apache/asterix/column/metadata/schema/visitor/SchemaJSONBuilderVisitor.java
new file mode 100644
index 0000000..547b9d8
--- /dev/null
+++ b/asterixdb/asterix-column/src/test/java/org/apache/asterix/column/metadata/schema/visitor/SchemaJSONBuilderVisitor.java
@@ -0,0 +1,287 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.asterix.column.metadata.schema.visitor;
+
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Map;
+import java.util.regex.Matcher;
+import java.util.regex.Pattern;
+
+import org.apache.asterix.column.metadata.schema.AbstractSchemaNode;
+import org.apache.asterix.column.metadata.schema.ISchemaNodeVisitor;
+import org.apache.asterix.column.metadata.schema.ObjectSchemaNode;
+import org.apache.asterix.column.metadata.schema.UnionSchemaNode;
+import org.apache.asterix.column.metadata.schema.collection.AbstractCollectionSchemaNode;
+import org.apache.asterix.column.metadata.schema.primitive.PrimitiveSchemaNode;
+import org.apache.asterix.column.operation.lsm.flush.FlushColumnMetadata;
+import org.apache.asterix.column.values.writer.DummyColumnValuesWriter;
+import org.apache.asterix.dataflow.data.nontagged.serde.AStringSerializerDeserializer;
+import org.apache.asterix.om.base.AString;
+import org.apache.hyracks.api.exceptions.HyracksDataException;
+import org.apache.hyracks.data.std.api.IValueReference;
+import org.apache.hyracks.data.std.util.ByteArrayAccessibleDataInputStream;
+import org.apache.hyracks.data.std.util.ByteArrayAccessibleInputStream;
+import org.apache.hyracks.util.string.UTF8StringReader;
+import org.apache.hyracks.util.string.UTF8StringWriter;
+
+import com.fasterxml.jackson.core.JsonProcessingException;
+import com.fasterxml.jackson.databind.JsonNode;
+import com.fasterxml.jackson.databind.ObjectMapper;
+import com.fasterxml.jackson.databind.node.ArrayNode;
+import com.fasterxml.jackson.databind.node.JsonNodeFactory;
+import com.fasterxml.jackson.databind.node.ObjectNode;
+import com.fasterxml.jackson.databind.node.TextNode;
+
+import it.unimi.dsi.fastutil.ints.IntList;
+
+/**
+ * Schema is encoded into a JSON.
+ * UnionSchema is encoded as an array with first element being "UNION", the next elements will be the union contents.
+ * FlatSchema is encoded as - "($typetag)(INDEX:$index)$actual_value"
+ * ObjectSchema, Array schema are encoded as is.
+ */
+public class SchemaJSONBuilderVisitor implements ISchemaNodeVisitor<JsonNode, Void> {
+ private static final String UNION_FLAG = "UNION";
+
+ private final FlushColumnMetadata context;
+ private final List<String> fieldNames;
+
+ public SchemaJSONBuilderVisitor(FlushColumnMetadata context) throws HyracksDataException {
+ this.context = context;
+ this.fieldNames = new ArrayList<>();
+ AStringSerializerDeserializer stringSerDer =
+ new AStringSerializerDeserializer(new UTF8StringWriter(), new UTF8StringReader());
+ List<IValueReference> extractedFieldNames = context.getFieldNamesDictionary().getFieldNames();
+
+ //Deserialize field names
+ ByteArrayAccessibleInputStream in = new ByteArrayAccessibleInputStream(new byte[0], 0, 0);
+ ByteArrayAccessibleDataInputStream dataIn = new ByteArrayAccessibleDataInputStream(in);
+ for (IValueReference serFieldName : extractedFieldNames) {
+ in.setContent(serFieldName.getByteArray(), 0, serFieldName.getLength());
+ AString fieldName = stringSerDer.deserialize(dataIn);
+ this.fieldNames.add(fieldName.getStringValue());
+ }
+ }
+
+ public String build(ObjectSchemaNode root) throws HyracksDataException {
+ JsonNode jsonNode = visit(root, null);
+ return jsonNode.toString();
+ }
+
+ @Override
+ public JsonNode visit(ObjectSchemaNode objectNode, Void arg) throws HyracksDataException {
+
+ List<AbstractSchemaNode> children = objectNode.getChildren();
+ IntList fieldNameIndexes = objectNode.getChildrenFieldNameIndexes();
+ ObjectNode jsonNode = JsonNodeFactory.instance.objectNode();
+
+ for (int i = 0; i < children.size(); i++) {
+ String fieldName = fieldNames.get(fieldNameIndexes.getInt(i));
+ AbstractSchemaNode child = children.get(i);
+ JsonNode childNode = child.accept(this, null);
+ jsonNode.putIfAbsent(fieldName, childNode);
+ }
+ return jsonNode;
+ }
+
+ @Override
+ public JsonNode visit(AbstractCollectionSchemaNode collectionNode, Void arg) throws HyracksDataException {
+ AbstractSchemaNode itemNode = collectionNode.getItemNode();
+ ArrayNode arrayNode = JsonNodeFactory.instance.arrayNode();
+ arrayNode.add(itemNode.accept(this, null));
+ return arrayNode;
+ }
+
+ @Override
+ public JsonNode visit(UnionSchemaNode unionNode, Void arg) throws HyracksDataException {
+ ArrayNode jsonNode = JsonNodeFactory.instance.arrayNode();
+ jsonNode.add(UNION_FLAG);
+
+ for (AbstractSchemaNode child : unionNode.getChildren().values()) {
+ jsonNode.add(child.accept(this, null));
+ }
+ return jsonNode;
+ }
+
+ @Override
+ public JsonNode visit(PrimitiveSchemaNode primitiveNode, Void arg) throws HyracksDataException {
+ DummyColumnValuesWriter writer = (DummyColumnValuesWriter) context.getWriter(primitiveNode.getColumnIndex());
+ String encodedString = String.format("(%s)(INDEX:%d)%s", primitiveNode.getTypeTag(),
+ primitiveNode.getColumnIndex(), writer.getDefinitionLevelsString());
+ return JsonNodeFactory.instance.textNode(encodedString);
+ }
+
+ public static String JSONtoPretty(String jsonString) {
+ JSONtoPrettyConverter stringConverter = new JSONtoPrettyConverter();
+
+ ObjectMapper objectMapper = new ObjectMapper();
+
+ JsonNode jsonNode;
+
+ try {
+ jsonNode = objectMapper.readTree(jsonString);
+ } catch (JsonProcessingException e) {
+ throw new RuntimeException(e);
+ }
+
+ return stringConverter.convert(jsonNode);
+ }
+
+ static class JSONtoPrettyConverter {
+
+ private final StringBuilder builder;
+ private int level;
+ private int indent;
+
+ private static String FLAT_SCHEMA_MATCHING_REGEX = "^\\(([a-zA-Z]+)\\)\\(INDEX:(\\d+)\\)(.*)$";
+
+ public JSONtoPrettyConverter() {
+ builder = new StringBuilder();
+ level = 0;
+ indent = 0;
+
+ }
+
+ public String convert(JsonNode jsonNode) {
+ builder.append("root\n");
+ visit(jsonNode);
+ return builder.toString();
+ }
+
+ private void visit(JsonNode jsonNode) {
+ if (jsonNode.isObject()) {
+ visitObject((ObjectNode) jsonNode);
+ } else if (jsonNode.isArray() && jsonNode.size() == 1) {
+ visitArray((ArrayNode) jsonNode);
+ } else if (jsonNode.isArray()) {
+ visitUnion((ArrayNode) jsonNode);
+ } else {
+ visitFlat((TextNode) jsonNode);
+ }
+ }
+
+ private void visitObject(ObjectNode objectNode) {
+ level++;
+ indent++;
+
+ for (Iterator<Map.Entry<String, JsonNode>> it = objectNode.fields(); it.hasNext();) {
+ Map.Entry<String, JsonNode> entry = it.next();
+ String fieldName = entry.getKey();
+ JsonNode child = entry.getValue();
+ append(fieldName, child);
+ visit(child);
+ }
+ level--;
+ indent--;
+ }
+
+ private void visitArray(ArrayNode arrayNode) {
+ level++;
+ indent++;
+ JsonNode itemNode = arrayNode.get(0);
+ append("item", itemNode);
+ visit(itemNode);
+ level--;
+ indent--;
+ }
+
+ private void visitUnion(ArrayNode unionNode) {
+ indent++;
+ for (int i = 1; i < unionNode.size(); i++) {
+ JsonNode child = unionNode.get(i);
+ append(getTypeTag(child), child);
+ visit(child);
+ }
+ indent--;
+ }
+
+ private void visitFlat(TextNode textNode) {
+ indent++;
+ appendLevels(getValue(textNode));
+ indent--;
+ }
+
+ private void appendLevels(String levels) {
+ appendDecor();
+ builder.append("Def ");
+ builder.append(levels);
+ builder.append('\n');
+ }
+
+ private void appendDecor() {
+ builder.append("| ".repeat(Math.max(0, indent - 1)));
+ builder.append("|-- ");
+ }
+
+ private void append(String key, JsonNode node) {
+ appendDecor();
+ builder.append(key);
+ builder.append(": ");
+ builder.append(getTypeTag(node).toString());
+ builder.append(" <level: ");
+ builder.append(level);
+ if (node.isTextual()) {
+ final TextNode textNode = (TextNode) node;
+ builder.append(", index: ");
+ builder.append(getColumnIndex(textNode));
+ }
+ builder.append(">\n");
+ }
+
+ private String getTypeTag(JsonNode jsonNode) {
+ if (jsonNode.isObject()) {
+ return "object";
+ } else if (jsonNode.isArray() && jsonNode.size() == 1) {
+ return "array";
+ } else if (jsonNode.isArray()) {
+ return "union";
+ } else {
+ return getFlatTypeTag((TextNode) jsonNode);
+ }
+ }
+
+ private String getFlatTypeTag(TextNode textNode) {
+ String encodedString = textNode.textValue();
+ Pattern pattern = Pattern.compile(FLAT_SCHEMA_MATCHING_REGEX);
+ Matcher matcher = pattern.matcher(encodedString);
+ matcher.find();
+ return matcher.group(1);
+ }
+
+ private String getColumnIndex(TextNode textNode) {
+ String encodedString = textNode.textValue();
+ Pattern pattern = Pattern.compile(FLAT_SCHEMA_MATCHING_REGEX);
+ Matcher matcher = pattern.matcher(encodedString);
+ matcher.find();
+ return matcher.group(2);
+ }
+
+ private String getValue(TextNode textNode) {
+ String encodedString = textNode.textValue();
+ Pattern pattern = Pattern.compile(FLAT_SCHEMA_MATCHING_REGEX);
+ Matcher matcher = pattern.matcher(encodedString);
+ matcher.find();
+ return matcher.group(3);
+ }
+
+ }
+
+}
diff --git a/asterixdb/asterix-column/src/test/java/org/apache/asterix/column/test/dummy/TransformerTest.java b/asterixdb/asterix-column/src/test/java/org/apache/asterix/column/test/dummy/TransformerTest.java
index c457eeb..2901128 100644
--- a/asterixdb/asterix-column/src/test/java/org/apache/asterix/column/test/dummy/TransformerTest.java
+++ b/asterixdb/asterix-column/src/test/java/org/apache/asterix/column/test/dummy/TransformerTest.java
@@ -26,7 +26,7 @@
import org.apache.asterix.column.common.test.TestCase;
import org.apache.asterix.column.metadata.schema.ObjectSchemaNode;
-import org.apache.asterix.column.metadata.schema.visitor.SchemaStringBuilderVisitor;
+import org.apache.asterix.column.metadata.schema.visitor.SchemaJSONBuilderVisitor;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -74,11 +74,11 @@
private void writeResult(ObjectSchemaNode root) throws IOException {
File resultFile = testCase.getOutputFile();
- SchemaStringBuilderVisitor schemaBuilder = new SchemaStringBuilderVisitor(columnMetadata);
+ SchemaJSONBuilderVisitor schemaBuilder = new SchemaJSONBuilderVisitor(columnMetadata);
String schema = schemaBuilder.build(root);
try (PrintStream ps = new PrintStream(new FileOutputStream(resultFile))) {
- ps.print(schema);
+ ps.print(SchemaJSONBuilderVisitor.JSONtoPretty(schema));
}
}
}
--
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/18374
To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-Project: asterixdb
Gerrit-Branch: master
Gerrit-Change-Id: I9c8f6d1739a3e5aafb1ead1ae6c19417c6f0a060
Gerrit-Change-Number: 18374
Gerrit-PatchSet: 1
Gerrit-Owner: preetham.poluparthi@couchbase.com
Gerrit-MessageType: newchange