You are viewing a plain text version of this content. The canonical link for it is here.
Posted to notifications@asterixdb.apache.org by AsterixDB Code Review <do...@asterix-gerrit.ics.uci.edu> on 2024/12/09 22:59:03 UTC
Change in asterixdb[goldfish]: [WIP] Added Log lines for debugging
From Ritik Raj <ra...@gmail.com>:
Ritik Raj has uploaded this change for review. ( https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/19186 )
Change subject: [WIP] Added Log lines for debugging
......................................................................
[WIP] Added Log lines for debugging
Change-Id: I5a5455a88933284dfe8a5c0cdfb73154f675f4d0
---
M asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/stream/in/MultiByteBufferInputStream.java
M asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/decoder/ParquetRunLengthBitPackingHybridDecoder.java
M asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/AbstractColumnValuesReader.java
3 files changed, 76 insertions(+), 3 deletions(-)
git pull ssh://asterix-gerrit.ics.uci.edu:29418/asterixdb refs/changes/86/19186/1
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/decoder/ParquetRunLengthBitPackingHybridDecoder.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/decoder/ParquetRunLengthBitPackingHybridDecoder.java
index 4607dc2..f02364f 100644
--- a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/decoder/ParquetRunLengthBitPackingHybridDecoder.java
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/decoder/ParquetRunLengthBitPackingHybridDecoder.java
@@ -31,6 +31,9 @@
import org.apache.parquet.column.values.rle.RunLengthBitPackingHybridDecoder;
import org.apache.parquet.io.ParquetDecodingException;
+import it.unimi.dsi.fastutil.ints.IntArrayList;
+import it.unimi.dsi.fastutil.ints.IntList;
+
/**
* Re-implementation of {@link RunLengthBitPackingHybridDecoder}
*/
@@ -42,6 +45,7 @@
private final int bitWidth;
private final BytePacker packer;
+ private final IntList lastReadValues;
private InputStream in;
private MODE mode;
@@ -55,17 +59,25 @@
Preconditions.checkArgument(bitWidth >= 0 && bitWidth <= 32, "bitWidth must be >= 0 and <= 32");
this.bitWidth = bitWidth;
this.packer = Packer.LITTLE_ENDIAN.newBytePacker(bitWidth);
+ this.lastReadValues = new IntArrayList();
}
public void reset(InputStream in) {
this.in = in;
currentCount = 0;
currentBufferLength = 0;
+ lastReadValues.clear();
+ }
+
+ public IntList getLastReadValues() {
+ return lastReadValues;
}
public int readInt() throws HyracksDataException {
try {
- return nextInt();
+ int val = nextInt();
+ lastReadValues.add(val);
+ return val;
} catch (IOException e) {
throw HyracksDataException.create(e);
}
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/stream/in/MultiByteBufferInputStream.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/stream/in/MultiByteBufferInputStream.java
index 31f8179..c46c42e 100644
--- a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/stream/in/MultiByteBufferInputStream.java
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/bytes/stream/in/MultiByteBufferInputStream.java
@@ -260,6 +260,11 @@
this.length = 0;
}
+ @Override
+ public ByteBuffer getBuffer() {
+ return current;
+ }
+
private List<ByteBuffer> sliceBuffers(long length) throws EOFException {
if (length <= 0) {
return Collections.emptyList();
diff --git a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/AbstractColumnValuesReader.java b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/AbstractColumnValuesReader.java
index 0942a23..c102b85 100644
--- a/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/AbstractColumnValuesReader.java
+++ b/asterixdb/asterix-column/src/main/java/org/apache/asterix/column/values/reader/AbstractColumnValuesReader.java
@@ -19,6 +19,7 @@
package org.apache.asterix.column.values.reader;
import java.io.IOException;
+import java.nio.ByteBuffer;
import org.apache.asterix.column.bytes.decoder.ParquetRunLengthBitPackingHybridDecoder;
import org.apache.asterix.column.bytes.stream.in.AbstractBytesInputStream;
@@ -31,11 +32,17 @@
import org.apache.asterix.om.types.ATypeTag;
import org.apache.hyracks.api.exceptions.HyracksDataException;
import org.apache.hyracks.data.std.api.IValueReference;
+import org.apache.logging.log4j.LogManager;
+import org.apache.logging.log4j.Logger;
import org.apache.parquet.bytes.BytesUtils;
+import com.fasterxml.jackson.databind.ObjectMapper;
import com.fasterxml.jackson.databind.node.ObjectNode;
abstract class AbstractColumnValuesReader implements IColumnValuesReader {
+ private static final Logger LOGGER = LogManager.getLogger();
+ private static final ObjectMapper OBJECT_MAPPER = new ObjectMapper();
+
protected final AbstractValueReader valueReader;
protected final int columnIndex;
protected final int maxLevel;
@@ -53,6 +60,8 @@
// For logging purposes only
private int numberOfEncounteredMissing;
private int numberOfEncounteredNull;
+ private ByteBuffer lastBuffer;
+ private int lastPosition;
AbstractColumnValuesReader(AbstractValueReader valueReader, int columnIndex, int maxLevel, boolean primaryKey) {
this.valueReader = valueReader;
@@ -69,8 +78,20 @@
if (allMissing) {
return;
}
-
- int actualLevel = definitionLevels.readInt();
+ int actualLevel;
+ try {
+ actualLevel = definitionLevels.readInt();
+ } catch (Exception e) {
+ lastBuffer.position(lastPosition);
+ ObjectNode infoNode = OBJECT_MAPPER.createObjectNode();
+ appendReaderInformation(infoNode);
+ LOGGER.info("currentValueIndex: {}, currentValueCount: {}, definitionLevels: {}, readerInfo: {}",
+ valueIndex, valueCount, definitionLevels.getLastReadValues(), infoNode);
+ // not sure, about this as for Non-Key columns, the inputStream actually contains multiple buffers.
+ // but trying to print the buffer.
+ logByteBuffer(lastBuffer);
+ throw e;
+ }
//Check whether the level is for a null value
nullLevel = ColumnValuesUtil.isNull(nullBitMask, actualLevel);
//Clear the null bit to allow repeated value readers determine the correct delimiter for null values
@@ -81,6 +102,30 @@
numberOfEncounteredNull += isNull() ? 1 : 0;
}
+ public void logByteBuffer(ByteBuffer buffer) {
+ if (buffer == null) {
+ LOGGER.warn("ByteBuffer is null!");
+ return;
+ }
+
+ int originalPosition = buffer.position();
+ int capacity = buffer.capacity();
+ StringBuilder sb = new StringBuilder();
+
+ sb.append("ByteBuffer contents from position ").append(originalPosition).append(" to capacity ")
+ .append(capacity).append(": [");
+
+ for (int i = originalPosition; i < capacity; i++) {
+ sb.append(String.format("%02X", buffer.get(i)));
+ if (i < capacity - 1) {
+ sb.append(", ");
+ }
+ }
+ sb.append("]");
+
+ LOGGER.info(sb.toString());
+ }
+
abstract void resetValues();
@Override
@@ -96,6 +141,8 @@
}
allMissing = false;
try {
+ lastBuffer = in.getBuffer();
+ lastPosition = lastBuffer.position();
nullBitMask = ColumnValuesUtil.getNullMask(BytesUtils.readZigZagVarInt(in));
int defLevelsSize = BytesUtils.readZigZagVarInt(in);
valueCount = BytesUtils.readZigZagVarInt(in);
--
To view, visit https://asterix-gerrit.ics.uci.edu/c/asterixdb/+/19186
To unsubscribe, or for help writing mail filters, visit https://asterix-gerrit.ics.uci.edu/settings
Gerrit-Project: asterixdb
Gerrit-Branch: goldfish
Gerrit-Change-Id: I5a5455a88933284dfe8a5c0cdfb73154f675f4d0
Gerrit-Change-Number: 19186
Gerrit-PatchSet: 1
Gerrit-Owner: Ritik Raj <ra...@gmail.com>
Gerrit-MessageType: newchange