You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@phoenix.apache.org by ap...@apache.org on 2025/06/26 17:19:55 UTC
(phoenix) 01/02: PHOENIX-7565 Replication log file, and replication log file writer and reader classes (#2133)
This is an automated email from the ASF dual-hosted git repository.
apurtell pushed a commit to branch PHOENIX-7562-feature
in repository https://gitbox.apache.org/repos/asf/phoenix.git
commit 9c29755851134dd3a82d0e5733f14a0ca7131826
Author: Andrew Purtell <ap...@apache.org>
AuthorDate: Fri May 9 10:45:32 2025 -0700
PHOENIX-7565 Replication log file, and replication log file writer and reader classes (#2133)
---
phoenix-core-server/pom.xml | 4 +
.../phoenix/replication/log/AsyncFSDataOutput.java | 165 ++++++
.../org/apache/phoenix/replication/log/CRC64.java | 76 +++
.../phoenix/replication/log/FSDataInput.java | 158 ++++++
.../phoenix/replication/log/HDFSDataOutput.java | 132 +++++
.../phoenix/replication/log/LogBlockHeader.java | 125 +++++
.../apache/phoenix/replication/log/LogFile.java | 438 +++++++++++++++
.../phoenix/replication/log/LogFileCodec.java | 283 ++++++++++
.../replication/log/LogFileFormatReader.java | 389 ++++++++++++++
.../replication/log/LogFileFormatWriter.java | 241 +++++++++
.../phoenix/replication/log/LogFileHeader.java | 134 +++++
.../replication/log/LogFileIterationException.java | 68 +++
.../phoenix/replication/log/LogFileReader.java | 150 ++++++
.../replication/log/LogFileReaderContext.java | 157 ++++++
.../phoenix/replication/log/LogFileRecord.java | 182 +++++++
.../phoenix/replication/log/LogFileTrailer.java | 210 ++++++++
.../phoenix/replication/log/LogFileWriter.java | 118 +++++
.../replication/log/LogFileWriterContext.java | 129 +++++
.../phoenix/replication/log/SeekableDataInput.java | 36 ++
.../replication/log/SyncableDataOutput.java | 49 ++
.../phoenix/replication/log/package-info.java | 53 ++
phoenix-core/pom.xml | 5 +
.../apache/phoenix/replication/log/CRC64Test.java | 82 +++
.../phoenix/replication/log/LogFileCodecTest.java | 322 +++++++++++
.../replication/log/LogFileCompressionTest.java | 225 ++++++++
.../phoenix/replication/log/LogFileFormatTest.java | 588 +++++++++++++++++++++
.../phoenix/replication/log/LogFileTestUtil.java | 434 +++++++++++++++
.../replication/log/LogFileWriterSyncTest.java | 194 +++++++
.../phoenix/replication/log/LogFileWriterTest.java | 152 ++++++
pom.xml | 11 +
30 files changed, 5310 insertions(+)
diff --git a/phoenix-core-server/pom.xml b/phoenix-core-server/pom.xml
index bb55875b46..86e9f9f7af 100644
--- a/phoenix-core-server/pom.xml
+++ b/phoenix-core-server/pom.xml
@@ -73,6 +73,10 @@
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-metrics</artifactId>
</dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-asyncfs</artifactId>
+ </dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-server</artifactId>
diff --git a/phoenix-core-server/src/main/java/org/apache/phoenix/replication/log/AsyncFSDataOutput.java b/phoenix-core-server/src/main/java/org/apache/phoenix/replication/log/AsyncFSDataOutput.java
new file mode 100644
index 0000000000..d2287d8674
--- /dev/null
+++ b/phoenix-core-server/src/main/java/org/apache/phoenix/replication/log/AsyncFSDataOutput.java
@@ -0,0 +1,165 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.replication.log;
+
+import java.io.IOException;
+import java.io.InterruptedIOException;
+import java.util.concurrent.ExecutionException;
+
+import org.apache.hadoop.hbase.io.asyncfs.AsyncFSOutput;
+import org.apache.hadoop.hbase.util.Bytes;
+
+/**
+ * SyncableDataOutput implementation that delegates to a hbase-async AsyncFSOutput.
+ */
+@edu.umd.cs.findbugs.annotations.SuppressWarnings(value = { "EI_EXPOSE_REP", "EI_EXPOSE_REP2" },
+ justification = "Intentional")
+public class AsyncFSDataOutput implements SyncableDataOutput {
+
+ private final AsyncFSOutput delegate;
+
+ public AsyncFSDataOutput(AsyncFSOutput delegate) {
+ this.delegate = delegate;
+ }
+
+ @Override
+ public long getPos() throws IOException {
+ return delegate.getSyncedLength() + delegate.buffered();
+ }
+
+ @Override
+ public void close() throws IOException {
+ delegate.close();
+ }
+
+ private byte[] byteBuf = new byte[1];
+
+ @Override
+ public void write(int b) throws IOException {
+ byteBuf[0] = (byte) b;
+ delegate.write(byteBuf);
+ }
+
+ @Override
+ public void write(byte[] b) throws IOException {
+ delegate.write(b);
+ }
+
+ @Override
+ public void write(byte[] b, int off, int len) throws IOException {
+ delegate.write(b, off, len);
+ }
+
+ @Override
+ public void writeByte(int v) throws IOException {
+ byteBuf[0] = (byte) v;
+ delegate.write(byteBuf);
+ }
+
+ @Override
+ public void writeBoolean(boolean v) throws IOException {
+ writeByte(v ? 1 : 0);
+ }
+
+ @Override
+ public void writeShort(int v) throws IOException {
+ // This code is equivalent to what DataOutputStream in the JRE does.
+ writeByte((v >>> 8) & 0xFF);
+ writeByte((v >>> 0) & 0xFF);
+ }
+
+ @Override
+ public void writeChar(int v) throws IOException {
+ // What DataOutputStream in the JRE does is equivalent to writeShort.
+ writeShort(v);
+ }
+
+ @Override
+ public void writeInt(int v) throws IOException {
+ delegate.writeInt(v);
+ }
+
+ @Override
+ public void writeLong(long v) throws IOException {
+ delegate.write(Bytes.toBytes(v));
+ }
+
+ @Override
+ public void writeFloat(float v) throws IOException {
+ delegate.write(Bytes.toBytes(v));
+ }
+
+ @Override
+ public void writeDouble(double v) throws IOException {
+ delegate.write(Bytes.toBytes(v));
+ }
+
+ @Override
+ public void writeBytes(String s) throws IOException {
+ writeUTF(s); // Simplify here by unconditionally coding strings as UTF-8.
+ }
+
+ @Override
+ public void writeChars(String s) throws IOException {
+ // DataOutputStream in the JRE does the equivalent of writeShort on each code point.
+ int len = s.length();
+ for (int i = 0; i < len; i++) {
+ int v = s.charAt(i);
+ writeShort(v);
+ }
+ }
+
+ @Override
+ public void writeUTF(String s) throws IOException {
+ delegate.write(Bytes.toBytes(s));
+ }
+
+ @Override
+ public void hflush() throws IOException {
+ try {
+ // Our sync method is synchronous, as intended.
+ delegate.flush(false).get();
+ } catch (InterruptedException e) {
+ InterruptedIOException ioe = new InterruptedIOException();
+ ioe.initCause(e);
+ throw ioe;
+ } catch (ExecutionException e) {
+ throw new IOException(e);
+ }
+ }
+
+ @Override
+ public void hsync() throws IOException {
+ try {
+ // Our sync method is synchronous, as intended.
+ delegate.flush(true).get();
+ } catch (InterruptedException e) {
+ InterruptedIOException ioe = new InterruptedIOException();
+ ioe.initCause(e);
+ throw ioe;
+ } catch (ExecutionException e) {
+ throw new IOException(e);
+ }
+ }
+
+ @Override
+ public void sync() throws IOException {
+ hsync();
+ }
+
+}
diff --git a/phoenix-core-server/src/main/java/org/apache/phoenix/replication/log/CRC64.java b/phoenix-core-server/src/main/java/org/apache/phoenix/replication/log/CRC64.java
new file mode 100644
index 0000000000..c4b39a045b
--- /dev/null
+++ b/phoenix-core-server/src/main/java/org/apache/phoenix/replication/log/CRC64.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.replication.log;
+
+public class CRC64 {
+ private static final long POLY = 0xC96C5795D7870F42L; // ECMA-182
+
+ private static final int TABLE_LENGTH = 256;
+ private static final long[] TABLE = new long[TABLE_LENGTH];
+ private long value = -1; // Initial CRC value is -1
+
+ static {
+ // Initialize a table constructed from POLY
+ for (int n = 0; n < TABLE_LENGTH; ++n) {
+ long crc = n;
+ for (int i = 0; i < 8; ++i) {
+ if ((crc & 1) == 1) {
+ crc = (crc >>> 1) ^ POLY;
+ } else {
+ crc >>>= 1;
+ }
+ }
+ TABLE[n] = crc;
+ }
+ }
+
+ /** Resets the CRC calculation to the initial value. */
+ public void reset() {
+ value = -1;
+ }
+
+ /**
+ * Updates the CRC value with a specified portion of a byte array.
+ * @param input the byte array to update the CRC value with
+ * @param off the start offset of the data
+ * @param len the number of bytes to use for the update
+ */
+ public void update(byte[] input, int off, int len) {
+ // Update CRC calculation using the lookup table
+ for (int i = off; i < off + len; i++) {
+ value = TABLE[(input[i] ^ (int) value) & 0xFF] ^ (value >>> 8);
+ }
+ }
+
+ /**
+ * Updates the CRC value with an entire byte array.
+ * @param input the byte array to update the CRC value with
+ */
+ public void update(byte[] input) {
+ update(input, 0, input.length);
+ }
+
+ /**
+ * Returns the current CRC value. The final checksum requires inverting the bits.
+ * @return the current CRC value (before final inversion)
+ */
+ public long getValue() {
+ return ~value;
+ }
+
+}
diff --git a/phoenix-core-server/src/main/java/org/apache/phoenix/replication/log/FSDataInput.java b/phoenix-core-server/src/main/java/org/apache/phoenix/replication/log/FSDataInput.java
new file mode 100644
index 0000000000..56b7d9c2d3
--- /dev/null
+++ b/phoenix-core-server/src/main/java/org/apache/phoenix/replication/log/FSDataInput.java
@@ -0,0 +1,158 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.replication.log;
+
+import java.io.Closeable;
+import java.io.DataInput;
+import java.io.IOException;
+
+import org.apache.hadoop.fs.FSDataInputStream;
+
+/**
+ * Input side abstraction, mirroring the write side FSDataOutput.
+ */
+@edu.umd.cs.findbugs.annotations.SuppressWarnings(value = { "EI_EXPOSE_REP", "EI_EXPOSE_REP2" },
+ justification = "Intentional")
+public class FSDataInput implements DataInput, Closeable, SeekableDataInput {
+
+ private final FSDataInputStream delegate;
+
+ public FSDataInput(FSDataInputStream delegate) {
+ this.delegate = delegate;
+ }
+
+ @Override
+ public void readFully(byte[] b) throws IOException {
+ delegate.readFully(b);
+ }
+
+ @Override
+ public void readFully(byte[] b, int off, int len) throws IOException {
+ delegate.readFully(b, off, len);
+ }
+
+ @Override
+ public int skipBytes(int n) throws IOException {
+ return delegate.skipBytes(n);
+ }
+
+ @Override
+ public boolean readBoolean() throws IOException {
+ return delegate.readBoolean();
+ }
+
+ @Override
+ public byte readByte() throws IOException {
+ return delegate.readByte();
+ }
+
+ @Override
+ public int readUnsignedByte() throws IOException {
+ return delegate.readUnsignedByte();
+ }
+
+ @Override
+ public short readShort() throws IOException {
+ return delegate.readShort();
+ }
+
+ @Override
+ public int readUnsignedShort() throws IOException {
+ return delegate.readUnsignedShort();
+ }
+
+ @Override
+ public char readChar() throws IOException {
+ return delegate.readChar();
+ }
+
+ @Override
+ public int readInt() throws IOException {
+ return delegate.readInt();
+ }
+
+ @Override
+ public long readLong() throws IOException {
+ return delegate.readLong();
+ }
+
+ @Override
+ public float readFloat() throws IOException {
+ return delegate.readFloat();
+ }
+
+ @Override
+ public double readDouble() throws IOException {
+ return delegate.readDouble();
+ }
+
+ @Override
+ @Deprecated
+ public String readLine() throws IOException {
+ return delegate.readLine();
+ }
+
+ @Override
+ public String readUTF() throws IOException {
+ return delegate.readUTF();
+ }
+
+ @Override
+ public void seek(long pos) throws IOException {
+ delegate.seek(pos);
+ }
+
+ @Override
+ public long getPos() throws IOException {
+ return delegate.getPos();
+ }
+
+ @Override
+ public boolean seekToNewSource(long targetPos) throws IOException {
+ return delegate.seekToNewSource(targetPos);
+ }
+
+ public int read(byte[] buffer) throws IOException {
+ return delegate.read(buffer);
+ }
+
+ public int read(byte[] buffer, int pos, int len) throws IOException {
+ return delegate.read(buffer, pos, len);
+ }
+
+ @Override
+ public int read(long position, byte[] buffer, int offset, int length) throws IOException {
+ return delegate.read(position, buffer, offset, length);
+ }
+
+ @Override
+ public void readFully(long position, byte[] buffer, int offset, int length) throws IOException {
+ delegate.readFully(position, buffer, offset, length);
+ }
+
+ @Override
+ public void readFully(long position, byte[] buffer) throws IOException {
+ delegate.readFully(position, buffer);
+ }
+
+ @Override
+ public void close() throws IOException {
+ delegate.close();
+ }
+
+}
diff --git a/phoenix-core-server/src/main/java/org/apache/phoenix/replication/log/HDFSDataOutput.java b/phoenix-core-server/src/main/java/org/apache/phoenix/replication/log/HDFSDataOutput.java
new file mode 100644
index 0000000000..5d09173854
--- /dev/null
+++ b/phoenix-core-server/src/main/java/org/apache/phoenix/replication/log/HDFSDataOutput.java
@@ -0,0 +1,132 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.replication.log;
+
+import java.io.IOException;
+
+import org.apache.hadoop.fs.FSDataOutputStream;
+
+/**
+ * SyncableDataOutput implementation that delegates to a standard FSDataOutputStream.
+ */
+@edu.umd.cs.findbugs.annotations.SuppressWarnings(value = { "EI_EXPOSE_REP", "EI_EXPOSE_REP2" },
+ justification = "Intentional")
+public class HDFSDataOutput implements SyncableDataOutput {
+
+ private final FSDataOutputStream delegate;
+
+ public HDFSDataOutput(FSDataOutputStream delegate) {
+ this.delegate = delegate;
+ }
+
+ @Override
+ public long getPos() throws IOException {
+ return delegate.getPos();
+ }
+
+ @Override
+ public void close() throws IOException {
+ delegate.close();
+ }
+
+ @Override
+ public void write(int b) throws IOException {
+ delegate.write(b);
+ }
+
+ @Override
+ public void write(byte[] b) throws IOException {
+ delegate.write(b);
+ }
+
+ @Override
+ public void write(byte[] b, int off, int len) throws IOException {
+ delegate.write(b, off, len);
+ }
+
+ @Override
+ public void writeBoolean(boolean v) throws IOException {
+ delegate.writeBoolean(v);
+ }
+
+ @Override
+ public void writeByte(int v) throws IOException {
+ delegate.writeByte(v);
+ }
+
+ @Override
+ public void writeShort(int v) throws IOException {
+ delegate.writeShort(v);
+ }
+
+ @Override
+ public void writeChar(int v) throws IOException {
+ delegate.writeChar(v);
+ }
+
+ @Override
+ public void writeInt(int v) throws IOException {
+ delegate.writeInt(v);
+ }
+
+ @Override
+ public void writeLong(long v) throws IOException {
+ delegate.writeLong(v);
+ }
+
+ @Override
+ public void writeFloat(float v) throws IOException {
+ delegate.writeFloat(v);
+ }
+
+ @Override
+ public void writeDouble(double v) throws IOException {
+ delegate.writeDouble(v);
+ }
+
+ @Override
+ public void writeBytes(String s) throws IOException {
+ delegate.writeBytes(s);
+ }
+
+ @Override
+ public void writeChars(String s) throws IOException {
+ delegate.writeChars(s);
+ }
+
+ @Override
+ public void writeUTF(String s) throws IOException {
+ delegate.writeUTF(s);
+ }
+
+ @Override
+ public void hflush() throws IOException {
+ delegate.hflush();
+ }
+
+ @Override
+ public void hsync() throws IOException {
+ delegate.hsync();
+ }
+
+ @Override
+ public void sync() throws IOException {
+ delegate.hsync();
+ }
+
+}
diff --git a/phoenix-core-server/src/main/java/org/apache/phoenix/replication/log/LogBlockHeader.java b/phoenix-core-server/src/main/java/org/apache/phoenix/replication/log/LogBlockHeader.java
new file mode 100644
index 0000000000..5b55fbedd6
--- /dev/null
+++ b/phoenix-core-server/src/main/java/org/apache/phoenix/replication/log/LogBlockHeader.java
@@ -0,0 +1,125 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.replication.log;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.Arrays;
+
+import org.apache.hadoop.hbase.io.compress.Compression;
+import org.apache.hadoop.hbase.util.Bytes;
+
+public class LogBlockHeader implements LogFile.BlockHeader {
+ private int version;
+ private Compression.Algorithm compression;
+ private int uncompressedSize;
+ private int compressedSize;
+
+ /** Magic for replication log blocks */
+ static final byte[] MAGIC = Bytes.toBytes("PBLK");
+ /** Current version of the replication log block header */
+ static final byte VERSION = 1;
+
+ static final int HEADER_SIZE = MAGIC.length + (2 * Bytes.SIZEOF_BYTE)
+ + (2 * Bytes.SIZEOF_INT);
+
+ public LogBlockHeader() {
+ this.version = VERSION;
+ this.compression = Compression.Algorithm.NONE;
+ }
+
+ @Override
+ public int getVersion() {
+ return version;
+ }
+
+ @Override
+ public Compression.Algorithm getDataCompression() {
+ return compression;
+ }
+
+ @Override
+ public LogFile.BlockHeader setDataCompression(Compression.Algorithm compression) {
+ this.compression = compression;
+ return this;
+ }
+
+ @Override
+ public int getUncompressedDataSize() {
+ return uncompressedSize;
+ }
+
+ @Override
+ public LogFile.BlockHeader setUncompressedDataSize(int uncompressedSize) {
+ this.uncompressedSize = uncompressedSize;
+ return this;
+ }
+
+ @Override
+ public int getCompressedDataSize() {
+ return compressedSize;
+ }
+
+ @Override
+ public LogFile.BlockHeader setCompressedDataSize(int compressedSize) {
+ this.compressedSize = compressedSize;
+ return this;
+ }
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ byte[] magic = new byte[MAGIC.length];
+ in.readFully(magic);
+ if (!Arrays.equals(MAGIC, magic)) {
+ throw new IOException("Invalid Log block magic. Got " + Bytes.toStringBinary(magic)
+ + ", expected " + Bytes.toStringBinary(MAGIC));
+ }
+ version = in.readByte();
+ if (version != VERSION) {
+ throw new IOException("Unsupported Log block header version. Got " + version
+ + ", expected " + VERSION);
+ }
+ int ordinal = in.readByte();
+ compression = Compression.Algorithm.values()[ordinal];
+ uncompressedSize = in.readInt();
+ compressedSize = in.readInt();
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ out.write(MAGIC);
+ out.writeByte(version);
+ out.writeByte((byte) compression.ordinal());
+ out.writeInt(uncompressedSize);
+ out.writeInt(compressedSize);
+ }
+
+ @Override
+ public int getSerializedHeaderLength() {
+ return HEADER_SIZE;
+ }
+
+ @Override
+ public String toString() {
+ return "LogBlockHeader [version=" + version + ", compressionAlgorithm="
+ + compression + ", uncompressedSize=" + uncompressedSize + ", compressedSize="
+ + compressedSize + "]";
+ }
+
+}
diff --git a/phoenix-core-server/src/main/java/org/apache/phoenix/replication/log/LogFile.java b/phoenix-core-server/src/main/java/org/apache/phoenix/replication/log/LogFile.java
new file mode 100644
index 0000000000..419313bd7e
--- /dev/null
+++ b/phoenix-core-server/src/main/java/org/apache/phoenix/replication/log/LogFile.java
@@ -0,0 +1,438 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.replication.log;
+
+import java.io.Closeable;
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.client.Mutation;
+import org.apache.hadoop.hbase.io.compress.Compression;
+
+/**
+ * Defines the structure and constants for Phoenix Replication Log files.
+ * Provides interfaces for reading and writing these logs.
+ */
+public interface LogFile {
+
+ /** Represents the file header */
+ interface Header {
+ /**
+ * Gets the major version of the log file format.
+ * @return The major version number.
+ */
+ int getMajorVersion();
+
+ /**
+ * Sets the major version of the log file format.
+ * @param majorVersion The major version number to set.
+ * @return This Header instance for chaining.
+ */
+ Header setMajorVersion(int majorVersion);
+
+ /**
+ * Gets the minor version of the log file format.
+ * @return The minor version number.
+ */
+ int getMinorVersion();
+
+ /**
+ * Sets the minor version of the log file format.
+ * @param minorVersion The minor version number to set.
+ * @return This Header instance for chaining.
+ */
+ Header setMinorVersion(int minorVersion);
+
+ /**
+ * Gets the fixed serialized length of the header in bytes.
+ * @return The length of the serialized header.
+ */
+ int getSerializedLength();
+
+ /**
+ * Reads the header fields from the provided DataInput stream.
+ * @param in The DataInput stream to read from.
+ * @throws IOException if an I/O error occurs or the header format is invalid.
+ */
+ void readFields(DataInput in) throws IOException;
+
+ /**
+ * Writes the header fields to the provided DataOutput stream.
+ * @param out The DataOutput stream to write to.
+ * @throws IOException if an I/O error occurs.
+ */
+ void write(DataOutput out) throws IOException;
+ }
+
+ /** Represents the header of a single block within the log file */
+ interface BlockHeader {
+ /**
+ * Gets the version of this block header format.
+ * @return The block header version number.
+ */
+ int getVersion();
+
+ /**
+ * Gets the compression algorithm used for the data payload of this block.
+ * @return The compression algorithm.
+ */
+ Compression.Algorithm getDataCompression();
+
+ /**
+ * Sets the compression algorithm used for the data payload of this block.
+ * @param compression The compression algorithm to set.
+ * @return This BlockHeader instance for chaining.
+ */
+ BlockHeader setDataCompression(Compression.Algorithm compression);
+
+ /**
+ * Gets the size of the data payload before compression.
+ * @return The uncompressed data size in bytes.
+ */
+ int getUncompressedDataSize();
+
+ /**
+ * Sets the size of the data payload before compression.
+ * @param uncompressedSize The uncompressed data size in bytes.
+ * @return This BlockHeader instance for chaining.
+ */
+ BlockHeader setUncompressedDataSize(int uncompressedSize);
+
+ /**
+ * Gets the size of the data payload after compression. If compression is NONE,
+ * this will be the same as the uncompressed size.
+ * @return The compressed data size in bytes.
+ */
+ int getCompressedDataSize();
+
+ /**
+ * Sets the size of the data payload after compression.
+ * @param compressedSize The compressed data size in bytes.
+ * @return This BlockHeader instance for chaining.
+ */
+ BlockHeader setCompressedDataSize(int compressedSize);
+
+ /**
+ * Gets the fixed serialized length of the block header itself in bytes.
+ * @return The length of the serialized block header.
+ */
+ int getSerializedHeaderLength();
+
+ /**
+ * Reads the block header fields from the provided DataInput stream.
+ * @param in The DataInput stream to read from.
+ * @throws IOException if an I/O error occurs or the block header format is invalid.
+ */
+ void readFields(DataInput in) throws IOException;
+
+ /**
+ * Writes the block header fields to the provided DataOutput stream.
+ * @param out The DataOutput stream to write to.
+ * @throws IOException if an I/O error occurs.
+ */
+ void write(DataOutput out) throws IOException;
+ }
+
+ /** Represents the file trailer */
+ interface Trailer {
+ /**
+ * Gets the major version of the log file format stored in the trailer.
+ * Useful for validation when reading from the end of the file.
+ * @return The major version number.
+ */
+ int getMajorVersion();
+
+ /**
+ * Sets the major version of the log file format in the trailer.
+ * @param majorVersion The major version number to set.
+ * @return This Trailer instance for chaining.
+ */
+ Trailer setMajorVersion(int majorVersion);
+
+ /**
+ * Gets the minor version of the log file format stored in the trailer.
+ * Useful for validation when reading from the end of the file.
+ * @return The minor version number.
+ */
+ int getMinorVersion();
+
+ /**
+ * Sets the minor version of the log file format in the trailer.
+ * @param minorVersion The minor version number to set.
+ * @return This Trailer instance for chaining.
+ */
+ Trailer setMinorVersion(int minorVersion);
+
+ /**
+ * Gets the total number of records contained in the log file.
+ * @return The total record count.
+ */
+ long getRecordCount();
+
+ /**
+ * Sets the total number of records contained in the log file.
+ * @param recordCount The total record count to set.
+ * @return This Trailer instance for chaining.
+ */
+ Trailer setRecordCount(long recordCount);
+
+ /**
+ * Gets the total number of blocks contained in the log file.
+ * @return The total block count.
+ */
+ long getBlockCount();
+
+ /**
+ * Sets the total number of blocks contained in the log file.
+ * @param blockCount The total block count to set.
+ * @return This Trailer instance for chaining.
+ */
+ Trailer setBlockCount(long blockCount);
+
+ /**
+ * Gets the byte offset within the file where the first block begins (after the header).
+ * @return The starting offset of the first block.
+ */
+ long getBlocksStartOffset();
+
+ /**
+ * Sets the byte offset within the file where the first block begins.
+ * @param offset The starting offset of the first block.
+ * @return This Trailer instance for chaining.
+ */
+ Trailer setBlocksStartOffset(long offset);
+
+ /**
+ * Gets the byte offset within the file where the trailer itself begins.
+ * @return The starting offset of the trailer.
+ */
+ long getTrailerStartOffset();
+
+ /**
+ * Sets the byte offset within the file where the trailer itself begins.
+ * @param offset The starting offset of the trailer.
+ * @return This Trailer instance for chaining.
+ */
+ Trailer setTrailerStartOffset(long offset);
+
+ /**
+ * Gets the total serialized length of the trailer in bytes, including any variable-length
+ * metadata.
+ * @return The total length of the serialized trailer.
+ */
+ int getSerializedLength();
+
+ /**
+ * Reads the trailer fields from the provided DataInput stream. Assumes the stream
+ * is positioned at the start of the trailer.
+ * @param in The DataInput stream to read from.
+ * @throws IOException if an I/O error occurs or the trailer format is invalid.
+ */
+ void readFields(DataInput in) throws IOException;
+
+ /**
+ * Writes the trailer fields to the provided DataOutput stream.
+ * @param out The DataOutput stream to write to.
+ * @throws IOException if an I/O error occurs.
+ */
+ void write(DataOutput out) throws IOException;
+ }
+
+ /** Represents a single logical change */
+ interface Record {
+ /**
+ * Gets the mutation this record represents.
+ * @return The Mutation.
+ */
+ Mutation getMutation();
+
+ /**
+ * Sets the mutation this record represents.
+ * @param mutation The Mutation to set.
+ * @return This Record instance for chaining.
+ */
+ Record setMutation(Mutation mutation);
+
+ /**
+ * Gets the name of the HBase table this record pertains to.
+ * @return The HBase table name.
+ */
+ String getHBaseTableName();
+
+ /**
+ * Sets the name of the HBase table this record pertains to.
+ * @param tableName The HBase table name to set.
+ * @return This Record instance for chaining.
+ */
+ Record setHBaseTableName(String tableName);
+
+ /**
+ * Gets the commit ID or System Change Number (SCN) associated with this record.
+ * @return The commit ID/SCN.
+ */
+ long getCommitId();
+
+ /**
+ * Sets the commit ID or System Change Number (SCN) for this record.
+ * @param commitId The commit ID/SCN to set.
+ * @return This Record instance for chaining.
+ */
+ Record setCommitId(long commitId);
+
+ /**
+ * Gets the total serialized length of this record in bytes, including any length prefixes
+ * used by the codec. This value should be set by the codec after writing or reading.
+ * @return The total serialized length of the record.
+ */
+ int getSerializedLength();
+
+ /**
+ * Sets the total serialized length of this record. This is typically called by the codec.
+ * @param serializedLength The total serialized length in bytes.
+ * @return This Record instance for chaining.
+ */
+ Record setSerializedLength(int serializedLength);
+ }
+
+ /** Interface for writing replication logs */
+ interface Writer extends Closeable {
+ /**
+ * Initializes the writer with necessary context.
+ * @param context The context containing initialization parameters.
+ * @throws IOException if initialization fails.
+ */
+ void init(LogFileWriterContext context) throws IOException;
+
+ /**
+ * Appends an HBase mutation to the log file. The log record may be buffered internally.
+ * @param tableName The HBase table name
+ * @param commitId The commit identifier
+ * @param mutation The mutation to append.
+ * @throws IOException if an I/O error occurs during append.
+ */
+ void append(String tableName, long commitId, Mutation mutation) throws IOException;
+
+ /**
+ * Flushes any buffered data to the underlying storage and ensures it is durable
+ * (e.g., by calling hsync on the FSDataOutputStream). This guarantees that records
+ * appended before the sync are persisted.
+ * @throws IOException if an I/O error occurs during sync.
+ */
+ void sync() throws IOException;
+
+ /**
+ * Returns the current length of the log file being written, in bytes.
+ * This typically reflects the position of the underlying output stream.
+ * @return The current file length.
+ * @throws IOException if the length cannot be determined.
+ */
+ long getLength() throws IOException;
+ }
+
+ /** Interface for reading replication logs */
+ interface Reader extends Closeable, Iterable<Record> {
+ /**
+ * Initializes the reader with necessary context.
+ * This typically involves opening the file and reading the header.
+ * @param context The context containing initialization parameters.
+ * @throws IOException if initialization fails (e.g., file not found, invalid header).
+ */
+ void init(LogFileReaderContext context) throws IOException;
+
+ /**
+ * Reads and returns the next record from the log file, as an HBase mutation.
+ * @return The next Record, or null if the end of the file has been reached.
+ * @throws IOException if an I/O error occurs during reading or parsing.
+ */
+ Record next() throws IOException;
+
+ /**
+ * Returns the Header information read from the log file during initialization.
+ * @return The file Header.
+ * @throws IllegalStateException if the reader has not been initialized.
+ */
+ Header getHeader();
+
+ /**
+ * Returns the Trailer information read from the log file during initialization.
+ * If the trailer was missing or corrupt, this may return null.
+ * @return The file Trailer, or null if not present or invalid.
+ * @throws IllegalStateException if the reader has not been initialized.
+ */
+ Trailer getTrailer();
+ }
+
+ /** Interface for encoding/decoding Records within a block's buffer */
+ interface Codec {
+ Encoder getEncoder(DataOutput out);
+ Decoder getDecoder(DataInput in);
+ Decoder getDecoder(ByteBuffer buffer);
+
+ /** Interface for encoding Records to a DataOutput stream */
+ interface Encoder {
+ void write(Record record) throws IOException;
+ // NOTE: The encoder must set serializedLength in the Record after writing.
+ }
+
+ /** Interface for decoding Records from a DataInput stream or ByteBuffer */
+ interface Decoder {
+ /**
+ * Moves the decoder to the next record in the buffer. If a non-null {@code reuse}
+ * object is provided and is compatible, it will be populated with the data instead
+ * of creating a new object internally.
+ * @return true if a record was successfully read, false if the end of the stream
+ * is reached.
+ * @throws IOException if an error occurs during reading.
+ */
+ boolean advance() throws IOException;
+ /**
+ * Returns the current record, which might be the object passed in the last successful
+ * call to {@code advance(reuse)} or an internal object if reuse was null.
+ * Throws IllegalStateException if advance() has not been called successfully first.
+ */
+ Record current();
+ // NOTE: The decoder must set serializedLength in the Record after reading.
+ }
+ }
+
+ /**
+ * Utility for determining if a file is a valid replication log file.
+ * @param fs The FileSystem
+ * @param path Path to the potential replication log file
+ * @return true if the file is a valid replication log file, false otherwise
+ * @throws IOException if an I/O problem was encountered
+ */
+ static boolean isValidLogFile(final FileSystem fs, final Path path) throws IOException {
+ long length = fs.getFileStatus(path).getLen();
+ try (FSDataInputStream in = fs.open(path)) {
+ if (LogFileTrailer.isValidTrailer(in, length)) {
+ return true;
+ } else {
+ // Not a valid trailer, do we need to do something (set a flag)?
+ // Fall back to checking the header.
+ return LogFileHeader.isValidHeader(in);
+ }
+ }
+ }
+
+}
diff --git a/phoenix-core-server/src/main/java/org/apache/phoenix/replication/log/LogFileCodec.java b/phoenix-core-server/src/main/java/org/apache/phoenix/replication/log/LogFileCodec.java
new file mode 100644
index 0000000000..d56325b88d
--- /dev/null
+++ b/phoenix-core-server/src/main/java/org/apache/phoenix/replication/log/LogFileCodec.java
@@ -0,0 +1,283 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.replication.log;
+
+import java.io.ByteArrayOutputStream;
+import java.io.DataInput;
+import java.io.DataInputStream;
+import java.io.DataOutput;
+import java.io.DataOutputStream;
+import java.io.EOFException;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Mutation;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.io.ByteBuffInputStream;
+import org.apache.hadoop.hbase.nio.ByteBuff;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.io.WritableUtils;
+
+/**
+ * Default Codec for encoding and decoding ReplicationLog Records within a block buffer.
+ * This implementation uses standard Java DataInput/DataOutput for serialization.
+ *
+ * Record Format within a block:
+ * <pre>
+ * +--------------------------------------------+
+ * | RECORD LENGTH (vint) |
+ * +--------------------------------------------+
+ * | RECORD HEADER |
+ * | - Mutation type (byte) |
+ * | - HBase table name length (vint) |
+ * | - HBase table name (byte[]) |
+ * | - Transaction/SCN or commit ID (vint) |
+ * +--------------------------------------------+
+ * | ROW KEY LENGTH (vint) |
+ * | ROW KEY (byte[]) |
+ * +--------------------------------------------+
+ * | MUTATION TIMESTAMP (vint) |
+ * +--------------------------------------------+
+ * | NUMBER OF COLUMN FAMILIES CHANGED (vint) |
+ * +--------------------------------------------+
+ * | PER-FAMILY DATA (repeated) |
+ * | +--------------------------------------+ |
+ * | | COLUMN FAMILY NAME LENGTH (vint) | |
+ * | | COLUMN FAMILY NAME (byte[]) | |
+ * | | NUMBER OF CELLS IN FAMILY (vint) | |
+ * | +--------------------------------------+ |
+ * | | PER-CELL DATA (repeated) | |
+ * | | +––––––––––––----------------–--–+ | |
+ * | | | COLUMN QUALIFIER LENGTH (vint) | | |
+ * | | | COLUMN QUALIFIER (byte[]) | | |
+ * | | | VALUE LENGTH (vint) | | |
+ * | | | VALUE (byte[]) | | |
+ * | | +–––––––––––––--------------––--–+ | |
+ * | +--------------------------------------+ |
+ * +--------------------------------------------+
+ * </pre>
+ */
+public class LogFileCodec implements LogFile.Codec {
+
+ @Override
+ public Encoder getEncoder(DataOutput out) {
+ return new RecordEncoder(out);
+ }
+
+ @Override
+ public Decoder getDecoder(DataInput in) {
+ return new RecordDecoder(in);
+ }
+
+ @Override
+ public Decoder getDecoder(ByteBuffer buffer) {
+ // We use HBase ByteBuff and ByteBuffInputStream which avoids copying, because our buffers
+ // are known to be heap based.
+ ByteBuff wrapByteBuf = ByteBuff.wrap(buffer);
+ try {
+ return new RecordDecoder(new DataInputStream(new ByteBuffInputStream(wrapByteBuf)));
+ } finally {
+ wrapByteBuf.release();
+ }
+ }
+
+ private static class RecordEncoder implements LogFile.Codec.Encoder {
+ private final DataOutput out;
+ private final ByteArrayOutputStream currentRecord;
+
+ RecordEncoder(DataOutput out) {
+ this.out = out;
+ currentRecord = new ByteArrayOutputStream();
+ }
+
+
+ @Override
+ public void write(LogFile.Record record) throws IOException {
+
+ DataOutput recordOut = new DataOutputStream(currentRecord);
+
+ // Write record fields
+
+ Mutation mutation = record.getMutation();
+ LogFileRecord.MutationType mutationType = LogFileRecord.MutationType.get(mutation);
+ recordOut.writeByte(mutationType.getCode());
+ byte[] nameBytes = record.getHBaseTableName().getBytes(StandardCharsets.UTF_8);
+ WritableUtils.writeVInt(recordOut, nameBytes.length);
+ recordOut.write(nameBytes);
+ WritableUtils.writeVLong(recordOut, record.getCommitId());
+ byte[] rowKey = mutation.getRow();
+ WritableUtils.writeVInt(recordOut, rowKey.length);
+ recordOut.write(rowKey);
+ recordOut.writeLong(mutation.getTimestamp());
+
+ Map<byte[], List<Cell>> familyMap = mutation.getFamilyCellMap();
+ int cfCount = familyMap.size();
+ WritableUtils.writeVInt(recordOut, cfCount);
+
+ for (Map.Entry<byte[], List<Cell>> entry: familyMap.entrySet()) {
+ byte[] cf = entry.getKey();
+ WritableUtils.writeVInt(recordOut, cf.length);
+ recordOut.write(cf);
+ List<Cell> cells = entry.getValue();
+ WritableUtils.writeVInt(recordOut, cells.size());
+ for (Cell cell: cells) {
+ WritableUtils.writeVInt(recordOut, cell.getQualifierLength());
+ recordOut.write(cell.getQualifierArray(), cell.getQualifierOffset(),
+ cell.getQualifierLength());
+ WritableUtils.writeVInt(recordOut, cell.getValueLength());
+ if (cell.getValueLength() > 0) {
+ recordOut.write(cell.getValueArray(), cell.getValueOffset(),
+ cell.getValueLength());
+ }
+ }
+ }
+
+ byte[] currentRecordBytes = currentRecord.toByteArray();
+ // Write total record length
+ WritableUtils.writeVInt(out, currentRecordBytes.length);
+ // Write the record
+ out.write(currentRecordBytes);
+
+ // Set the size (including the vint prefix) on the record object
+ ((LogFileRecord) record).setSerializedLength(currentRecordBytes.length
+ + WritableUtils.getVIntSize(currentRecordBytes.length));
+
+ // Reset the ByteArrayOutputStream to release resources
+ currentRecord.reset();
+ }
+ }
+
+ private static class RecordDecoder implements LogFile.Codec.Decoder {
+ private final DataInput in;
+ // A reference to the object populated by the last successful advance()
+ private LogFileRecord current = null;
+
+ RecordDecoder(DataInput in) {
+ this.in = in;
+ }
+
+ @Override
+ public boolean advance() throws IOException {
+ try {
+ int recordDataLength = WritableUtils.readVInt(in);
+ recordDataLength += WritableUtils.getVIntSize(recordDataLength);
+
+ current = new LogFileRecord();
+ // Set the total serialized length on the record
+ current.setSerializedLength(recordDataLength);
+
+ LogFileRecord.MutationType type =
+ LogFileRecord.MutationType.codeToType(in.readByte());
+
+ int nameBytesLen = WritableUtils.readVInt(in);
+ byte[] nameBytes = new byte[nameBytesLen];
+ in.readFully(nameBytes);
+
+ current.setHBaseTableName(Bytes.toString(nameBytes));
+
+ current.setCommitId(WritableUtils.readVLong(in));
+
+ int rowKeyLen = WritableUtils.readVInt(in);
+ byte[] rowKey = new byte[rowKeyLen];
+ in.readFully(rowKey);
+
+ Mutation mutation;
+ switch (type) {
+ case PUT:
+ mutation = new Put(rowKey);
+ break;
+ case DELETE:
+ case DELETEFAMILYVERSION:
+ case DELETECOLUMN:
+ case DELETEFAMILY:
+ mutation = new Delete(rowKey);
+ break;
+ default:
+ throw new UnsupportedOperationException("Unhandled mutation type " + type);
+ }
+ current.setMutation(mutation);
+
+ long ts = in.readLong();
+ mutation.setTimestamp(ts);
+
+ int cfCount = WritableUtils.readVInt(in);
+ for (int i = 0; i < cfCount; i++) {
+ // Col name
+ int cfLen = WritableUtils.readVInt(in);
+ byte[] cf = new byte[cfLen];
+ in.readFully(cf);
+ // Qualifiers+Values Count
+ int columnValuePairsCount = WritableUtils.readVInt(in);
+ for (int j = 0; j < columnValuePairsCount; j++) {
+ // Qualifier name
+ int qualLen = WritableUtils.readVInt(in);
+ byte[] qual = new byte[qualLen];
+ if (qualLen > 0) {
+ in.readFully(qual);
+ }
+ // Value
+ int valueLen = WritableUtils.readVInt(in);
+ byte[] value = new byte[valueLen];
+ if (valueLen > 0) {
+ in.readFully(value);
+ }
+ switch (type) {
+ case PUT:
+ ((Put) mutation).addColumn(cf, qual, ts, value);
+ break;
+ case DELETE:
+ case DELETECOLUMN:
+ ((Delete) mutation).addColumn(cf, qual, ts);
+ break;
+ case DELETEFAMILYVERSION:
+ ((Delete) mutation).addFamilyVersion(cf, ts);
+ break;
+ case DELETEFAMILY:
+ ((Delete) mutation).addFamily(cf);
+ break;
+ default:
+ throw new UnsupportedOperationException("Unhandled mutation type "
+ + type);
+ }
+ }
+ }
+
+ // Successfully read a record
+ return true;
+ } catch (EOFException e) {
+ // End of stream
+ current = null;
+ return false;
+ }
+ }
+
+ @Override
+ public LogFile.Record current() {
+ if (current == null) {
+ throw new IllegalStateException("Call advance() first, or end of stream reached");
+ }
+ return current;
+ }
+ }
+
+}
diff --git a/phoenix-core-server/src/main/java/org/apache/phoenix/replication/log/LogFileFormatReader.java b/phoenix-core-server/src/main/java/org/apache/phoenix/replication/log/LogFileFormatReader.java
new file mode 100644
index 0000000000..194abe7f87
--- /dev/null
+++ b/phoenix-core-server/src/main/java/org/apache/phoenix/replication/log/LogFileFormatReader.java
@@ -0,0 +1,389 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.replication.log;
+
+import java.io.Closeable;
+import java.io.EOFException;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import org.apache.hadoop.hbase.io.compress.Compression;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.io.compress.Decompressor;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Handles the low-level reading of headers, blocks, and trailers for Log files. Manages reading
+ * from the underlying stream, checksum validation, decompression, and providing access to block
+ * data for the Codec.
+ */
+@edu.umd.cs.findbugs.annotations.SuppressWarnings(value = { "EI_EXPOSE_REP", "EI_EXPOSE_REP2" },
+ justification = "Intentional")
+public class LogFileFormatReader implements Closeable {
+
+ private static final Logger LOG = LoggerFactory.getLogger(LogFileFormatReader.class);
+
+ private LogFileReaderContext context;
+ private LogFile.Codec.Decoder decoder;
+ private SeekableDataInput input;
+ private LogFile.Header header;
+ private LogFile.Trailer trailer;
+ private long currentPosition;
+ private ByteBuffer currentBlockBuffer;
+ private long currentBlockDataBytes;
+ private long currentBlockConsumedBytes;
+ private boolean trailerValidated;
+ private CRC64 crc = new CRC64();
+
+ public LogFileFormatReader() {
+
+ }
+
+ public void init(LogFileReaderContext context, SeekableDataInput input) throws IOException {
+ this.context = context;
+ this.input = input;
+ this.currentBlockDataBytes = -1;
+ this.currentBlockConsumedBytes = 0;
+ try {
+ readAndValidateTrailer();
+ trailerValidated = true;
+ } catch (IOException e) {
+ // Log warning, trailer might be missing or corrupt, proceed without it
+ LOG.warn("Failed to read or validate Log trailer for path: "
+ + (context != null ? context.getFilePath() : "unknown")
+ + ". Proceeding without trailer.", e);
+ trailer = null; // Ensure trailer is null if reading/validation failed
+ }
+ this.decoder = null;
+ // Seek to start of the file and read the header
+ readHeader();
+ currentPosition = input.getPos(); // Should be the offset of the first block
+ }
+
+ private void readAndValidateTrailer() throws IOException {
+ if (context.getFileSize() < LogFileTrailer.FIXED_TRAILER_SIZE) {
+ throw new IOException("File size " + context.getFileSize()
+ + " is smaller than the fixed trailer size " + LogFileTrailer.FIXED_TRAILER_SIZE);
+ }
+ LogFileTrailer ourTrailer = new LogFileTrailer();
+ // Fixed trailer fields will be LogTrailer.FIXED_TRAILER_SIZE bytes back from end of file.
+ input.seek(context.getFileSize() - LogFileTrailer.FIXED_TRAILER_SIZE);
+ // Read fixed fields
+ ourTrailer.readFixedFields(input);
+ // Now read the variable length protobuf message if present
+ input.seek(ourTrailer.getTrailerStartOffset());
+ ourTrailer.readMetadata(input);
+ trailer = ourTrailer;
+ }
+
+ private void readHeader() throws IOException {
+ header = new LogFileHeader();
+ // Seek to start of file
+ input.seek(0);
+ header.readFields(input);
+ }
+
+ public LogFile.Record next() throws IOException {
+ while (true) {
+ // Check if we need a new block. Either we haven't started yet or the current block
+ // is fully consumed.
+ if (decoder == null || currentBlockConsumedBytes >= currentBlockDataBytes) {
+ currentBlockBuffer = readNextBlock();
+ if (currentBlockBuffer == null) {
+ // End of file or unrecoverable error after skipping blocks
+ validateReadCounts(); // Validate counts if trailer was read
+ return null;
+ }
+ currentBlockConsumedBytes = 0;
+ // The limit of the buffer will be the size of the data in the block.
+ currentBlockDataBytes = currentBlockBuffer.limit();
+ decoder = context.getCodec().getDecoder(currentBlockBuffer);
+ }
+ // Advance the codec.
+ try {
+ if (decoder.advance()) {
+ // We advanced the codec, process the record.
+ LogFile.Record record = decoder.current();
+ int recordSize = record.getSerializedLength();
+ if (recordSize <= 0) {
+ // The codec must set the serialized length correctly. If it didn't this is
+ // likely a codec implementation problem but we will consider it an
+ // indication of block corruption.
+ throw new IOException("Codec failed to set serialized length for record at"
+ + " position " + (currentPosition + currentBlockConsumedBytes));
+ }
+ currentBlockConsumedBytes += recordSize;
+ context.incrementRecordsRead();
+ return record;
+ } else {
+ // We failed to advance the codec but know we have bytes left in the block.
+ throw new IOException("Decoder failed to advance at position "
+ + (currentPosition + currentBlockConsumedBytes));
+ }
+ } catch (IOException e) {
+ String message =
+ String.format("Encountered corrupt block at offset %d for path: %s",
+ currentPosition + currentBlockConsumedBytes, context.getFilePath());
+ LOG.warn(message, e);
+ context.incrementCorruptBlocksSkipped();
+ if (!context.isSkipCorruptBlocks()) {
+ throw new IOException(message, e);
+ }
+ // Resync from the start of the current block
+ if (!resyncReader(currentPosition)) {
+ return null;
+ }
+ decoder = null; // Force reading the next block
+ continue; // Retry reading from the next block
+ }
+ }
+ }
+
+ // Reads the next block header, payload, checksum. Validates checksum. Decompresses.
+ // Returns the decompressed block buffer, or null if EOF or unrecoverable error.
+ // Manages skipping corrupt blocks if configured.
+ private ByteBuffer readNextBlock() throws IOException {
+ while (currentPosition < getEndOfDataOffset()) {
+ long blockStartOffset = currentPosition;
+ ByteBuffer decompressedBuffer = null;
+ LogBlockHeader blockHeader = new LogBlockHeader();
+ try {
+ // Read Header
+ blockHeader.readFields(input);
+ currentPosition = input.getPos(); // Position after block header
+
+ // Read Payload
+ int payloadSize = blockHeader.getCompressedDataSize();
+ ByteBuffer payloadBuffer = ByteBuffer.allocate(payloadSize);
+
+ try {
+ input.readFully(payloadBuffer.array(), payloadBuffer.arrayOffset(),
+ payloadSize);
+ payloadBuffer.limit(payloadSize);
+ currentPosition += payloadSize;
+
+ // Read Checksum
+ long expectedChecksum = input.readLong();
+ currentPosition += Bytes.SIZEOF_LONG;
+
+ // Validate Checksum
+ crc.reset();
+ // Checksum is on the raw payload bytes
+ crc.update(payloadBuffer.array(), payloadBuffer.arrayOffset(), payloadSize);
+ long actualChecksum = crc.getValue();
+
+ if (expectedChecksum != actualChecksum) {
+ throw new IOException("Checksum mismatch for block at offset "
+ + blockStartOffset + ", expected: " + expectedChecksum + ", actual: "
+ + actualChecksum);
+ }
+
+ // Decompress if necessary
+ if (blockHeader.getDataCompression() != Compression.Algorithm.NONE) {
+ decompressedBuffer = decompressBlock(payloadBuffer, blockHeader);
+ } else {
+ decompressedBuffer = payloadBuffer;
+ }
+ } finally {
+ payloadBuffer = null;
+ }
+
+ context.incrementBlocksRead();
+ return decompressedBuffer; // Successfully read and processed the block
+ } catch (IOException | IllegalArgumentException e) {
+ context.incrementCorruptBlocksSkipped();
+ LOG.warn("Encountered corrupt block at offset " + blockStartOffset + " for path: "
+ + context.getFilePath(), e);
+ if (!context.isSkipCorruptBlocks()) {
+ decompressedBuffer = null;
+ throw new IOException("Failed to read block at offset " + blockStartOffset, e);
+ }
+ // Attempt to skip this block and find the next one
+ LOG.warn("Skipping corrupt block and attempting to resync...");
+ if (!resyncReader(blockStartOffset)) {
+ // Cannot resync, likely EOF or further corruption
+ decompressedBuffer = null;
+ return null;
+ }
+ // Continue the loop to read the next block after resync
+ }
+ }
+ return null;
+ }
+
+ // Decompresses the payload buffer using the specified algorithm.
+ // Manages obtaining/releasing decompressors.
+ private ByteBuffer decompressBlock(ByteBuffer compressedBuffer, LogFile.BlockHeader header)
+ throws IOException {
+ ByteBuffer decompressedBuffer;
+ Compression.Algorithm compression = header.getDataCompression();
+ Decompressor decompressor = compression.getDecompressor();
+ try {
+ decompressedBuffer = ByteBuffer.allocate(header.getUncompressedDataSize());
+ decompressor.reset();
+ decompressor.setInput(compressedBuffer.array(), compressedBuffer.arrayOffset(),
+ header.getCompressedDataSize());
+ int decompressedSize =
+ decompressor.decompress(decompressedBuffer.array(),
+ decompressedBuffer.arrayOffset(), header.getUncompressedDataSize());
+ if (decompressedSize != header.getUncompressedDataSize()) {
+ throw new IOException("Decompression size mismatch: expected="
+ + header.getUncompressedDataSize() + ", actual=" + decompressedSize);
+ }
+ decompressedBuffer.limit(decompressedSize);
+ return decompressedBuffer;
+ } finally {
+ compression.returnDecompressor(decompressor);
+ }
+ }
+
+ // Tries to find the start of the next valid block after corruption.
+ private boolean resyncReader(long offset) throws IOException {
+ long nextOffset = offset + 1; // Start searching after the point of failure
+ long endOfDataOffset = getEndOfDataOffset();
+ while (nextOffset < endOfDataOffset) {
+ long startPos = seekToMagic(nextOffset, LogBlockHeader.MAGIC);
+ if (startPos < 0) {
+ LOG.warn("Could not find next block magic after offset {}", nextOffset);
+ return false; // EOF reached without finding magic bytes
+ }
+ // Found what look like magic bytes, now validate the header
+ try {
+ input.seek(startPos);
+ LogBlockHeader blockHeader = new LogBlockHeader();
+ blockHeader.readFields(input); // This reads exactly HEADER_SIZE bytes
+ // Basic validation (readFields already checks magic and version)
+ if (blockHeader.getUncompressedDataSize() < 0
+ || blockHeader.getCompressedDataSize() < 0) {
+ throw new IOException("Invalid block header found at offset " + startPos);
+ }
+ // Check if the block fits within the data boundary
+ long blockEndOffset = startPos + blockHeader.getSerializedHeaderLength()
+ + blockHeader.getCompressedDataSize() + Bytes.SIZEOF_LONG;
+ if (blockEndOffset > endOfDataOffset) {
+ throw new IOException("Possible block at offset " + startPos
+ + " extends beyond end of data offset " + endOfDataOffset);
+ }
+ // If we reached here, the header seems structurally valid.
+ LOG.warn("Found valid block header at offset {}", startPos);
+ input.seek(startPos);
+ currentPosition = startPos;
+ // Invalidate current decoder
+ this.decoder = null;
+ return true;
+ } catch (EOFException e) {
+ // Found magic bytes too close to the end of the file
+ LOG.warn("Found magic bytes at offset {} but hit EOF trying to read header",
+ startPos);
+ return false;
+ } catch (IOException | IllegalArgumentException e) {
+ // Header was invalid (bad magic, version, compression, sizes, etc.)
+ LOG.warn("Found magic bytes at offset {} but header validation failed: {},"
+ + " continuing", startPos, e.getMessage());
+ nextOffset = startPos + 1; // Continue searching after the magic bytes
+ }
+ }
+ // Reached end of data without finding a valid block
+ LOG.warn("Reached of data offset {} without finding a valid block", endOfDataOffset);
+ return false;
+ }
+
+ // Helper to seek to the next occurrence of a magic byte sequence
+ private long seekToMagic(long startOffset, byte[] magic) throws IOException {
+ input.seek(startOffset);
+ byte[] buffer = new byte[8192]; // Read in chunks, 8K seems reasonable. (Should be larger?)
+ int magicPos = 0;
+ while (true) {
+ int bytesRead = input.read(buffer);
+ if (bytesRead == -1) {
+ return -1; // EOF
+ }
+ // This is brute force, is there something more efficient?
+ for (int i = 0; i < bytesRead; i++) {
+ if (buffer[i] == magic[magicPos]) {
+ magicPos++;
+ if (magicPos == magic.length) {
+ // Found block magic, position stream right before it.
+ long posAfterMagic = startOffset + (i + 1);
+ long magicStartPos = posAfterMagic - magic.length;
+ return magicStartPos;
+ }
+ } else {
+ // Restart the match
+ magicPos = 0;
+ if (buffer[i] == magic[0]) {
+ magicPos = 1;
+ }
+ }
+ }
+ startOffset += bytesRead;
+ }
+ }
+
+ // Returns the offset where data blocks end, either EOF or start of trailer.
+ private long getEndOfDataOffset() throws IOException {
+ return trailer != null ? trailer.getTrailerStartOffset() : context.getFileSize();
+ }
+
+ // Validates read counts against trailer counts if trailer was successfully read
+ private void validateReadCounts() {
+ if (!trailerValidated || trailer == null) {
+ return;
+ }
+ if (trailer.getBlockCount() != context.getBlocksRead()) {
+ LOG.warn("Trailer block count mismatch! expected=" + trailer.getBlockCount()
+ + ", actual=" + context.getBlocksRead());
+ }
+ if (trailer.getRecordCount() != context.getRecordsRead()) {
+ LOG.warn("Trailer record count mismatch! expected=" + trailer.getRecordCount()
+ + ", actual=" + context.getRecordsRead());
+ }
+ }
+
+ @Override
+ public void close() throws IOException {
+ currentBlockBuffer = null;
+ if (input != null) {
+ try {
+ input.close();
+ } finally {
+ input = null;
+ }
+ }
+ }
+
+ @Override
+ public String toString() {
+ return "LogFileFormatReader [context=" + context + ", decoder=" + decoder + ", input="
+ + input + ", header=" + header + ", trailer=" + trailer + ", currentPosition="
+ + currentPosition + ", currentBlockBuffer=" + currentBlockBuffer
+ + ", currentBlockUncompressedSize=" + currentBlockDataBytes
+ + ", currentBlockConsumedBytes=" + currentBlockConsumedBytes
+ + ", trailerValidated=" + trailerValidated + "]";
+ }
+
+ LogFile.Header getHeader() {
+ return header;
+ }
+
+ LogFile.Trailer getTrailer() {
+ return trailer;
+ }
+
+}
diff --git a/phoenix-core-server/src/main/java/org/apache/phoenix/replication/log/LogFileFormatWriter.java b/phoenix-core-server/src/main/java/org/apache/phoenix/replication/log/LogFileFormatWriter.java
new file mode 100644
index 0000000000..7e7f3c1931
--- /dev/null
+++ b/phoenix-core-server/src/main/java/org/apache/phoenix/replication/log/LogFileFormatWriter.java
@@ -0,0 +1,241 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.replication.log;
+
+import java.io.ByteArrayOutputStream;
+import java.io.Closeable;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import org.apache.hadoop.hbase.io.compress.Compression;
+import org.apache.hadoop.io.compress.Compressor;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Handles the low-level writing of headers, blocks, and trailers for Log files.
+ * Manages buffering, compression, checksums, and writing to the underlying stream.
+ */
+public class LogFileFormatWriter implements Closeable {
+
+ private static final Logger LOG = LoggerFactory.getLogger(LogFileFormatWriter.class);
+
+ private LogFileWriterContext context;
+ private LogFile.Codec.Encoder encoder;
+ private SyncableDataOutput output;
+ private ByteArrayOutputStream currentBlockBytes;
+ private DataOutputStream blockDataStream;
+ private boolean headerWritten = false;
+ private boolean trailerWritten = false;
+ private long recordCount = 0;
+ private long blockCount = 0;
+ private long blocksStartOffset = -1;
+ private CRC64 crc = new CRC64(); // Indirect this when we have more than one type
+ // Cached buffer for compression for performance
+ ByteBuffer compressBuf = null;
+
+ public LogFileFormatWriter() {
+
+ }
+
+ public void init(LogFileWriterContext context, SyncableDataOutput output) throws IOException {
+ this.output = output;
+ this.context = context;
+ this.currentBlockBytes = new ByteArrayOutputStream();
+ this.blockDataStream = new DataOutputStream(currentBlockBytes);
+ this.encoder = context.getCodec().getEncoder(blockDataStream);
+ }
+
+ private void writeFileHeader() throws IOException {
+ if (!headerWritten) {
+ LogFileHeader header = new LogFileHeader();
+ header.write(output);
+ blocksStartOffset = output.getPos(); // First block starts after header
+ headerWritten = true;
+ }
+ }
+
+ public long getBlocksStartOffset() {
+ return blocksStartOffset;
+ }
+
+ public void append(LogFile.Record record) throws IOException {
+ if (!headerWritten) {
+ // Lazily write file header
+ writeFileHeader();
+ }
+ if (trailerWritten) {
+ throw new IOException("Cannot append record after trailer has been written");
+ }
+ if (blockDataStream == null) {
+ startBlock(); // Start the block if needed
+ }
+ encoder.write(record);
+ recordCount++;
+
+ // Check if the current block size exceeds the limit AFTER writing the record
+ if (currentBlockBytes.size() >= context.getMaxBlockSize()) {
+ // To close the block, we do a sync(), which not only closes the block and opens a
+ // new one, it syncs the finalized block.
+ sync();
+ }
+ }
+
+ // Should be called before writing the first record.
+ public void startBlock() throws IOException {
+ if (blockDataStream == null) {
+ this.currentBlockBytes = new ByteArrayOutputStream();
+ this.blockDataStream = new DataOutputStream(currentBlockBytes);
+ // Re-initialize encoder for the new stream if necessary. This depends on the codec
+ // implementation details. For now we assume it is necessary.
+ this.encoder = context.getCodec().getEncoder(blockDataStream);
+ }
+ }
+
+ // Closes the current block being written, compresses it (if applicable),
+ // calculates checksum, and writes the block (header, payload, checksum) to the output stream.
+ public void closeBlock() throws IOException {
+ if (blockDataStream == null || currentBlockBytes.size() == 0) {
+ return; // No active block or block is empty
+ }
+ blockDataStream.flush(); // Ensure all encoded records are in the byte array
+ byte[] uncompressedBytes = currentBlockBytes.toByteArray();
+ ByteBuffer writeBuf;
+ int writeLen;
+ Compression.Algorithm ourCompression = context.getCompression();
+ if (ourCompression != Compression.Algorithm.NONE) {
+ Compressor compressor = ourCompression.getCompressor();
+ try {
+ compressor.reset();
+ compressor.setInput(uncompressedBytes, 0, uncompressedBytes.length);
+ compressor.finish(); // We are going to one-shot this.
+ // Give 25% overhead for pathological cases
+ // We can't go below this by much because the Snappy compressor will require more
+ // than 20% overhead in some cases or else it will refuse to try.
+ int compressBuffNeeded = (int) (uncompressedBytes.length * 1.25f);
+ if (compressBuf == null || compressBuf.capacity() < compressBuffNeeded) {
+ compressBuf = ByteBuffer.allocate(compressBuffNeeded);
+ }
+ compressBuf.clear();
+ writeLen = compressor.compress(compressBuf.array(), compressBuf.arrayOffset(),
+ compressBuffNeeded);
+ if (!compressor.finished()) {
+ throw new IOException("Compressor did not finish");
+ }
+ writeBuf = compressBuf;
+ } finally {
+ context.getCompression().returnCompressor(compressor);
+ }
+ } else {
+ writeBuf = ByteBuffer.wrap(uncompressedBytes);
+ writeLen = uncompressedBytes.length;
+ }
+
+ // Write block header
+ LogFile.BlockHeader blockHeader = new LogBlockHeader()
+ .setDataCompression(ourCompression)
+ .setUncompressedDataSize(uncompressedBytes.length)
+ .setCompressedDataSize(writeLen);
+ blockHeader.write(output);
+
+ output.write(writeBuf.array(), writeBuf.arrayOffset(), writeLen);
+ // Calculate checksum on the payload
+ crc.reset();
+ crc.update(writeBuf.array(), writeBuf.arrayOffset(), writeLen);
+ long checksum = crc.getValue();
+ output.writeLong(checksum); // Write CRC64 of header and payload
+
+ blockCount++;
+
+ // Reset for the next block
+ // blockDataStream remains wrapping the reset currentBlockUncompressedBytes
+ currentBlockBytes.reset();
+ blockDataStream = null;
+ }
+
+ public void sync() throws IOException {
+ // Ensure the current block data is flushed to the FSDataOutputStream.
+ if (blockDataStream != null && currentBlockBytes.size() > 0) {
+ // Closing the current block forces its header, data (potentially compressed),
+ // and checksum into the outputStream buffer.
+ closeBlock();
+ // Flush and sync the underlying output
+ output.sync();
+ // Start a new block for subsequent appends.
+ startBlock();
+ }
+ }
+
+ public long getPosition() throws IOException {
+ return output.getPos();
+ }
+
+ @Override
+ public void close() throws IOException {
+ // We use the fact we have already written the trailer as the boolean "closed" condition.
+ if (trailerWritten) {
+ return;
+ }
+ try {
+ // We might be closing an empty file, handle this case correctly.
+ if (!headerWritten) {
+ writeFileHeader();
+ }
+ // Close any outstanding block.
+ closeBlock();
+ // After we write the trailer we consider the file closed.
+ writeTrailer();
+ } finally {
+ if (output != null) {
+ // We need to catch the exception in order to prevent a compressor leak.
+ try {
+ output.close();
+ } catch (IOException e) {
+ LOG.error("Exception while closing LogFormatWriter", e);
+ }
+ }
+ }
+ }
+
+ private void writeTrailer() throws IOException {
+ LogFile.Trailer trailer = new LogFileTrailer()
+ .setRecordCount(recordCount)
+ .setBlockCount(blockCount)
+ .setBlocksStartOffset(blocksStartOffset)
+ .setTrailerStartOffset(output.getPos());
+ trailer.write(output);
+ trailerWritten = true;
+ try {
+ output.sync();
+ } catch (IOException e) {
+ // Failed sync on trailer write isn't a fatal event.
+ LOG.warn("Exception while syncing Log trailer", e);
+ }
+ }
+
+ @Override
+ public String toString() {
+ return "LogFileFormatWriter [writerContext=" + context
+ + ", currentBlockUncompressedBytes=" + currentBlockBytes
+ + ", headerWritten=" + headerWritten + ", trailerWritten=" + trailerWritten
+ + ", recordCount=" + recordCount + ", blockCount=" + blockCount
+ + ", blocksStartOffset=" + blocksStartOffset + "]";
+ }
+
+}
diff --git a/phoenix-core-server/src/main/java/org/apache/phoenix/replication/log/LogFileHeader.java b/phoenix-core-server/src/main/java/org/apache/phoenix/replication/log/LogFileHeader.java
new file mode 100644
index 0000000000..bfc07b76d9
--- /dev/null
+++ b/phoenix-core-server/src/main/java/org/apache/phoenix/replication/log/LogFileHeader.java
@@ -0,0 +1,134 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.replication.log;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.Arrays;
+
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.util.Bytes;
+
+public class LogFileHeader implements LogFile.Header {
+
+ /** Magic number for Phoenix Replication Log files */
+ static final byte[] MAGIC = Bytes.toBytes("PLOG");
+ /** Current major version of the replication log format */
+ static final int VERSION_MAJOR = 1;
+ /** Current minor version of the replication log format */
+ static final int VERSION_MINOR = 0;
+
+ static final int HEADERSIZE = MAGIC.length + 3 * Bytes.SIZEOF_BYTE;
+
+ private int majorVersion = VERSION_MAJOR;
+ private int minorVersion = VERSION_MINOR;
+
+ public LogFileHeader() {
+
+ }
+
+ @Override
+ public int getMajorVersion() {
+ return majorVersion;
+ }
+
+ @Override
+ public LogFile.Header setMajorVersion(int majorVersion) {
+ this.majorVersion = majorVersion;
+ return this;
+ }
+
+ @Override
+ public int getMinorVersion() {
+ return minorVersion;
+ }
+
+ @Override
+ public LogFile.Header setMinorVersion(int minorVersion) {
+ this.minorVersion = minorVersion;
+ return this;
+ }
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ byte[] magic = new byte[MAGIC.length];
+ in.readFully(magic);
+ if (!Arrays.equals(MAGIC, magic)) {
+ throw new IOException("Invalid LogFile magic. Got " + Bytes.toStringBinary(magic)
+ + ", expected " + Bytes.toStringBinary(MAGIC));
+ }
+ majorVersion = in.readByte();
+ minorVersion = in.readByte();
+ // Basic version check for now. We assume semver conventions where only higher major
+ // versions may be incompatible.
+ if (majorVersion > VERSION_MAJOR) {
+ throw new IOException("Unsupported LogFile version. Got major=" + majorVersion
+ + " minor=" + minorVersion + ", expected major=" + VERSION_MAJOR
+ + " minor=" + VERSION_MINOR);
+ }
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ out.write(MAGIC);
+ out.writeByte(majorVersion);
+ out.writeByte(minorVersion);
+ }
+
+ @Override
+ public int getSerializedLength() {
+ return HEADERSIZE;
+ }
+
+ public static boolean isValidHeader(final FileSystem fs, final Path path)
+ throws IOException {
+ if (fs.getFileStatus(path).getLen() < HEADERSIZE) {
+ return false;
+ }
+ try (FSDataInputStream in = fs.open(path)) {
+ return isValidHeader(in);
+ }
+ }
+
+ public static boolean isValidHeader(FSDataInputStream in) throws IOException {
+ in.seek(0);
+ byte[] magic = new byte[MAGIC.length];
+ in.readFully(magic);
+ if (!Arrays.equals(MAGIC, magic)) {
+ return false;
+ }
+ int majorVersion = in.readByte();
+ in.readByte(); // minorVersion, for now we don't use it
+ // Basic version check for now. We assume semver conventions where only higher major
+ // versions may be incompatible.
+ if (majorVersion > VERSION_MAJOR) {
+ return false;
+ }
+ return true;
+ }
+
+ @Override
+ public String toString() {
+ return "LogFileHeader [majorVersion=" + majorVersion + ", minorVersion=" + minorVersion
+ + "]";
+ }
+
+}
diff --git a/phoenix-core-server/src/main/java/org/apache/phoenix/replication/log/LogFileIterationException.java b/phoenix-core-server/src/main/java/org/apache/phoenix/replication/log/LogFileIterationException.java
new file mode 100644
index 0000000000..850505b6b1
--- /dev/null
+++ b/phoenix-core-server/src/main/java/org/apache/phoenix/replication/log/LogFileIterationException.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.replication.log;
+
+import java.io.IOException;
+
+/**
+ * Unchecked exception thrown when an IOException occurs during iteration over a replication log
+ * using its Iterator interface.
+ */
+public class LogFileIterationException extends RuntimeException {
+
+ private static final long serialVersionUID = 1L; // Recommended for RuntimeExceptions
+
+ /**
+ * Constructs a new LogIterationException with the specified cause.
+ * A standard detail message incorporating the cause's message is generated.
+ *
+ * @param cause the cause (the {@code IOException} that occurred during iteration).
+ * (A {@code null} value is permitted, and indicates that the cause is
+ * nonexistent or unknown.)
+ */
+ public LogFileIterationException(IOException cause) {
+ super("IOException occurred during Replication Log iteration: "
+ + (cause == null ? "Unknown cause" : cause.getMessage()), cause);
+ }
+
+ /**
+ * Constructs a new LogIterationException with the specified detail message and cause.
+ *
+ * @param message the detail message.
+ * @param cause the cause (the {@code IOException} that occurred during iteration).
+ * (A {@code null} value is permitted, and indicates that the cause is
+ * nonexistent or unknown.)
+ */
+ public LogFileIterationException(String message, IOException cause) {
+ super(message, cause);
+ }
+
+ /**
+ * Returns the cause of this exception (the {@code IOException} that
+ * occurred during iteration). Overrides the standard getCause to return
+ * the more specific type directly.
+ *
+ * @return the {@code IOException} which is the cause of this exception,
+ * or {@code null} if the cause is nonexistent or unknown.
+ */
+ @Override
+ public synchronized IOException getCause() {
+ Throwable cause = super.getCause();
+ return (cause instanceof IOException) ? (IOException) cause : null;
+ }
+}
diff --git a/phoenix-core-server/src/main/java/org/apache/phoenix/replication/log/LogFileReader.java b/phoenix-core-server/src/main/java/org/apache/phoenix/replication/log/LogFileReader.java
new file mode 100644
index 0000000000..6b3fa028b2
--- /dev/null
+++ b/phoenix-core-server/src/main/java/org/apache/phoenix/replication/log/LogFileReader.java
@@ -0,0 +1,150 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.replication.log;
+
+import java.io.IOException;
+import java.util.Iterator;
+import java.util.NoSuchElementException;
+
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Reader for Phoenix Replication Log files.
+ * Manages reading the header, trailer, and iterating through blocks and records via
+ * the LogFormatReader.
+ */
+public class LogFileReader implements LogFile.Reader {
+
+ private static final Logger LOG = LoggerFactory.getLogger(LogFileReader.class);
+ private LogFileReaderContext context;
+ private LogFileFormatReader reader;
+ private boolean closed = false;
+
+ public LogFileReader() {
+
+ }
+
+ public LogFileReaderContext getContext() {
+ return context;
+ }
+
+ @Override
+ public void init(LogFileReaderContext context) throws IOException {
+ this.context = context;
+ this.reader = new LogFileFormatReader(); // Instantiate from conf when more than one
+ this.reader.init(context,
+ new FSDataInput(context.getFileSystem().open(context.getFilePath())));
+ LOG.debug("Initialized LogFileReader for path {}", context.getFilePath());
+ }
+
+ @Override
+ public LogFile.Record next() throws IOException {
+ if (closed) {
+ throw new IOException("LogFileReader has been closed");
+ }
+ LogFile.Record r = reader.next();
+ if (r == null) {
+ return null;
+ }
+ return r;
+ }
+
+ @Override
+ public Iterator<LogFile.Record> iterator() {
+ return new Iterator<LogFile.Record>() {
+ private LogFile.Record next = null;
+ private boolean fetched = false;
+
+ @Override
+ public boolean hasNext() {
+ if (closed) {
+ return false;
+ }
+ if (!fetched) {
+ try {
+ next = LogFileReader.this.next();
+ fetched = true;
+ } catch (IOException e) {
+ throw new LogFileIterationException(e);
+ }
+ }
+ return next != null;
+ }
+
+ @Override
+ public LogFile.Record next() {
+ if (!hasNext()) {
+ throw new NoSuchElementException("No more records in the Log");
+ }
+ LogFile.Record r = next;
+ // Reset state for the next hasNext() call
+ next = null;
+ fetched = false;
+ return r;
+ }
+
+ @Override
+ public void remove() {
+ throw new UnsupportedOperationException(
+ "Remove operation is not supported by this iterator");
+ }
+ };
+ }
+
+ @Override
+ public LogFile.Header getHeader() {
+ if (reader == null) {
+ throw new IllegalStateException("LogFileReader not initialized");
+ }
+ return reader.getHeader();
+ }
+
+ @Override
+ public LogFile.Trailer getTrailer() {
+ if (reader == null) {
+ throw new IllegalStateException("LogFileReader not initialized");
+ }
+ return reader.getTrailer();
+ }
+
+ @Override
+ public void close() throws IOException {
+ if (closed) {
+ return;
+ }
+ try {
+ if (reader != null) {
+ reader.close();
+ }
+ } catch (IOException e) {
+ LOG.error("Error closing LogFileReader for path " + context.getFilePath(), e);
+ throw e;
+ } finally {
+ closed = true;
+ LOG.debug("Closed LogFileReader for path {}", context.getFilePath());
+ }
+ }
+
+ @Override
+ public String toString() {
+ return "LogFileReader [readerContext=" + context + ", formatReader=" + reader
+ + ", closed=" + closed + "]";
+ }
+
+}
diff --git a/phoenix-core-server/src/main/java/org/apache/phoenix/replication/log/LogFileReaderContext.java b/phoenix-core-server/src/main/java/org/apache/phoenix/replication/log/LogFileReaderContext.java
new file mode 100644
index 0000000000..52fbdb0cbb
--- /dev/null
+++ b/phoenix-core-server/src/main/java/org/apache/phoenix/replication/log/LogFileReaderContext.java
@@ -0,0 +1,157 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.replication.log;
+
+import java.io.IOException;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+
+/**
+ * Context for LogFileReader. Uses Builder pattern.
+ */
+@edu.umd.cs.findbugs.annotations.SuppressWarnings(value = { "EI_EXPOSE_REP", "EI_EXPOSE_REP2" },
+ justification = "Intentional")
+public class LogFileReaderContext {
+
+ /** Configuration key for skipping corrupt blocks */
+ public static final String LOGFILE_SKIP_CORRUPT_BLOCKS =
+ "phoenix.replication.logfile.skip.corrupt.blocks";
+ /** Default for skipping corrupt blocks */
+ public static final boolean DEFAULT_LOGFILE_SKIP_CORRUPT_BLOCKS = true;
+
+ private final Configuration conf;
+ private FileSystem fs;
+ private Path path;
+ private LogFileCodec codec;
+ private long fileSize = -1;
+ private boolean isSkipCorruptBlocks;
+ private long blocksRead;
+ private long recordsRead;
+ private long corruptBlocksSkipped;
+
+ public LogFileReaderContext(Configuration conf) {
+ this.conf = conf;
+ this.isSkipCorruptBlocks = conf.getBoolean(LOGFILE_SKIP_CORRUPT_BLOCKS,
+ DEFAULT_LOGFILE_SKIP_CORRUPT_BLOCKS);
+ // Note: When we have multiple codec types, instantiate the appropriate type based on
+ // configuration;
+ this.codec = new LogFileCodec();
+ }
+
+ public Configuration getConfiguration() {
+ return conf;
+ }
+
+ public FileSystem getFileSystem() {
+ return fs;
+ }
+
+ public LogFileReaderContext setFileSystem(FileSystem fileSystem) {
+ this.fs = fileSystem;
+ return this;
+ }
+
+ public Path getFilePath() {
+ return path;
+ }
+
+ public LogFileReaderContext setFilePath(Path filePath) {
+ this.path = filePath;
+ return this;
+ }
+
+ public long getFileSize() throws IOException {
+ if (fileSize < 0) {
+ fileSize = fs.getFileStatus(path).getLen();
+ }
+ return fileSize;
+ }
+
+ public LogFileReaderContext setFileSize(long fileSize) {
+ this.fileSize = fileSize;
+ return this;
+ }
+
+ public boolean isSkipCorruptBlocks() {
+ return isSkipCorruptBlocks;
+ }
+
+ public LogFileReaderContext setSkipCorruptBlocks(boolean isSkipCorruptBlocks) {
+ this.isSkipCorruptBlocks = isSkipCorruptBlocks;
+ return this;
+ }
+
+ public LogFileCodec getCodec() {
+ return codec;
+ }
+
+ public LogFileReaderContext setCodec(LogFileCodec codec) {
+ this.codec = codec;
+ return this;
+ }
+
+ public void incrementBlocksRead() {
+ blocksRead++;
+ }
+
+ public long getBlocksRead() {
+ return blocksRead;
+ }
+
+ public LogFileReaderContext setBlocksRead(long value) {
+ blocksRead = value;
+ return this;
+ }
+
+ public void incrementRecordsRead() {
+ recordsRead++;
+ }
+
+ public long getRecordsRead() {
+ return recordsRead;
+ }
+
+ public LogFileReaderContext setRecordsRead(long value) {
+ recordsRead = value;
+ return this;
+ }
+
+ public void incrementCorruptBlocksSkipped() {
+ corruptBlocksSkipped++;
+ }
+
+ public long getCorruptBlocksSkipped() {
+ return corruptBlocksSkipped;
+ }
+
+ public LogFileReaderContext setCorruptBlocksSkipped(long value) {
+ corruptBlocksSkipped = value;
+ return this;
+ }
+
+ @Override
+ public String toString() {
+ return "LogFileReaderContext [filePath=" + path + ", fileSize=" + fileSize
+ + ", isSkipCorruptBlocks=" + isSkipCorruptBlocks + ", codec=" + codec + ", blocksRead="
+ + blocksRead + ", recordsRead=" + recordsRead + ", corruptBlocksSkipped="
+ + corruptBlocksSkipped + "]";
+ }
+
+}
diff --git a/phoenix-core-server/src/main/java/org/apache/phoenix/replication/log/LogFileRecord.java b/phoenix-core-server/src/main/java/org/apache/phoenix/replication/log/LogFileRecord.java
new file mode 100644
index 0000000000..a36c5ef986
--- /dev/null
+++ b/phoenix-core-server/src/main/java/org/apache/phoenix/replication/log/LogFileRecord.java
@@ -0,0 +1,182 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.replication.log;
+
+import java.io.IOException;
+
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellScanner;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Mutation;
+import org.apache.hadoop.hbase.client.Put;
+
+@edu.umd.cs.findbugs.annotations.SuppressWarnings(value = { "EI_EXPOSE_REP", "EI_EXPOSE_REP2" },
+ justification = "Intentional")
+public class LogFileRecord implements LogFile.Record {
+
+ private String tableName;
+ private long commitId;
+ private Mutation mutation;
+ private int serializedLength;
+
+ public LogFileRecord() {
+ }
+
+ @Override
+ public String getHBaseTableName() {
+ return tableName;
+ }
+
+ @Override
+ public LogFile.Record setHBaseTableName(String tableName) {
+ this.tableName = tableName;
+ return this;
+ }
+
+ @Override
+ public long getCommitId() {
+ return commitId;
+ }
+
+ @Override
+ public LogFile.Record setCommitId(long commitId) {
+ this.commitId = commitId;
+ return this;
+ }
+
+ @Override
+ public Mutation getMutation() {
+ return this.mutation;
+ }
+
+ @Override
+ public LogFile.Record setMutation(Mutation mutation) {
+ this.mutation = mutation;
+ return this;
+ }
+
+ @Override
+ public int getSerializedLength() {
+ // NOTE: Should be set by the Codec using setSerializedLength after reading or writing
+ // the record.
+ return this.serializedLength;
+ }
+
+ @Override
+ public LogFile.Record setSerializedLength(int serializedLength) {
+ this.serializedLength = serializedLength;
+ return this;
+ }
+
+ @Override
+ public int hashCode() {
+ int code = tableName.hashCode();
+ code ^= Long.hashCode(commitId);
+ code ^= mutation.toString().hashCode();
+ return code;
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj) {
+ return true;
+ }
+ if (obj == null) {
+ return false;
+ }
+ if (getClass() != obj.getClass()) {
+ return false;
+ }
+ LogFileRecord other = (LogFileRecord) obj;
+ return tableName.equals(other.tableName)
+ && commitId == other.commitId
+ && mutation.toString().equals(other.mutation.toString());
+ }
+
+ @Override
+ public String toString() {
+ return "LogFileRecord [mutation=" + mutation.toString() + ", tableName=" + tableName
+ + ", commitId=" + commitId + " ]";
+ }
+
+ // Internals only below. Not for LogFile interface consumer use.
+
+ /**
+ * The Phoenix concept of HBase mutation type, which is currently a 1:1 mapping with HBase's,
+ * with different code values (they don't need to match), but may potentially diverge in the
+ * future.
+ */
+ protected enum MutationType {
+ PUT(1),
+ DELETE(2),
+ DELETEFAMILYVERSION(3),
+ DELETECOLUMN(4),
+ DELETEFAMILY(5);
+
+ private int code;
+
+ MutationType(int code) {
+ this.code = code;
+ }
+
+ int getCode() {
+ return code;
+ }
+
+ static MutationType get(Mutation mutation) throws IOException {
+ if (mutation instanceof Put) {
+ return PUT;
+ } else if (mutation instanceof Delete) {
+ CellScanner s = mutation.cellScanner();
+ if (!s.advance()) {
+ // No cell in delete. A simple delete of a row.
+ return DELETE;
+ }
+ // This assumes that either there is only one cell in the Delete, or all cells in
+ // the delete have the same cell type, which is correct as of today. We only need
+ // to look at the first.
+ Cell cell = s.current();
+ switch (cell.getType()) {
+ case Delete:
+ return DELETE;
+ case DeleteFamilyVersion:
+ return DELETEFAMILYVERSION;
+ case DeleteColumn:
+ return DELETECOLUMN;
+ case DeleteFamily:
+ return DELETEFAMILY;
+ default:
+ // Fall through to throw the UnsupportedOperationException
+ break;
+ }
+ } // Fall through to throw the UnsupportedOperationException
+ throw new UnsupportedOperationException("Unsupported mutation type: " + mutation);
+ }
+
+ static MutationType codeToType(int code) {
+ for (MutationType type: MutationType.values()) {
+ if (type.code == code) {
+ return type;
+ }
+ }
+ throw new UnsupportedOperationException("Unsupported mutation code: " + code);
+ }
+
+ }
+
+}
diff --git a/phoenix-core-server/src/main/java/org/apache/phoenix/replication/log/LogFileTrailer.java b/phoenix-core-server/src/main/java/org/apache/phoenix/replication/log/LogFileTrailer.java
new file mode 100644
index 0000000000..b05da86aae
--- /dev/null
+++ b/phoenix-core-server/src/main/java/org/apache/phoenix/replication/log/LogFileTrailer.java
@@ -0,0 +1,210 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.replication.log;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.Arrays;
+
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.util.Bytes;
+
+public class LogFileTrailer implements LogFile.Trailer {
+ private int majorVersion = LogFileHeader.VERSION_MAJOR;
+ private int minorVersion = LogFileHeader.VERSION_MINOR;
+ private long recordCount;
+ private long blockCount;
+ private long blocksStartOffset;
+ private long trailerStartOffset;
+
+ static final int VERSION_AND_MAGIC_SIZE = LogFileHeader.MAGIC.length + 2 * Bytes.SIZEOF_BYTE;
+ static final int FIXED_TRAILER_SIZE = Bytes.SIZEOF_LONG * 4 + VERSION_AND_MAGIC_SIZE;
+
+ public LogFileTrailer() {
+
+ }
+
+ // Add methods to set/get Protobuf metadata when needed in the future, e.g.:
+ // public void setMetadata(LogTrailerMetadata metadata) { ... }
+ // public LogTrailerMetadata getMetadata() throws InvalidProtocolBufferException { ... }
+
+ @Override
+ public int getMajorVersion() {
+ return majorVersion;
+ }
+
+ @Override
+ public int getMinorVersion() {
+ return minorVersion;
+ }
+
+ @Override
+ public LogFile.Trailer setMajorVersion(int majorVersion) {
+ this.majorVersion = majorVersion;
+ return this;
+ }
+
+ @Override
+ public LogFile.Trailer setMinorVersion(int minorVersion) {
+ this.minorVersion = minorVersion;
+ return this;
+ }
+
+ @Override
+ public long getRecordCount() {
+ return recordCount;
+ }
+
+ @Override
+ public LogFile.Trailer setRecordCount(long recordCount) {
+ this.recordCount = recordCount;
+ return this;
+ }
+
+ @Override
+ public long getBlockCount() {
+ return blockCount;
+ }
+
+ @Override
+ public LogFile.Trailer setBlockCount(long blockCount) {
+ this.blockCount = blockCount;
+ return this;
+ }
+
+ @Override
+ public long getBlocksStartOffset() {
+ return blocksStartOffset;
+ }
+
+ @Override
+ public LogFile.Trailer setBlocksStartOffset(long offset) {
+ this.blocksStartOffset = offset;
+ return this;
+ }
+
+ @Override
+ public long getTrailerStartOffset() {
+ return trailerStartOffset;
+ }
+
+ @Override
+ public LogFile.Trailer setTrailerStartOffset(long offset) {
+ this.trailerStartOffset = offset;
+ return this;
+ }
+
+ public void readFixedFields(DataInput in) throws IOException {
+ this.recordCount = in.readLong();
+ this.blockCount = in.readLong();
+ this.blocksStartOffset = in.readLong();
+ this.trailerStartOffset = in.readLong();
+ this.majorVersion = in.readByte();
+ this.minorVersion = in.readByte();
+ // Basic version check for now. We assume semver conventions where only higher major
+ // versions may be incompatible.
+ if (majorVersion > LogFileHeader.VERSION_MAJOR) {
+ throw new IOException("Unsupported LogFile version. Got major=" + majorVersion
+ + " minor=" + minorVersion + ", expected major=" + LogFileHeader.VERSION_MAJOR
+ + " minor=" + LogFileHeader.VERSION_MINOR);
+ }
+ byte[] magic = new byte[LogFileHeader.MAGIC.length];
+ in.readFully(magic);
+ if (!Arrays.equals(LogFileHeader.MAGIC, magic)) {
+ throw new IOException("Invalid LogFile magic. Got " + Bytes.toStringBinary(magic)
+ + ", expected " + Bytes.toStringBinary(LogFileHeader.MAGIC));
+ }
+ }
+
+ public void readMetadata(DataInput in) throws IOException {
+ int protoSize = in.readInt();
+ if (protoSize < 0) {
+ throw new IOException("Invalid Protobuf message size in LogFile trailer");
+ }
+ if (protoSize > 0) {
+ byte[] protoBytes = new byte[protoSize];
+ in.readFully(protoBytes);
+ // Deserialize protobuf when we have something to read
+ }
+ }
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ readMetadata(in);
+ readFixedFields(in);
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ out.writeInt(0); // Protobuf bytes
+ // Serialize protobuf when we have a message to write
+ out.writeLong(this.recordCount);
+ out.writeLong(this.blockCount);
+ out.writeLong(this.blocksStartOffset);
+ out.writeLong(this.trailerStartOffset);
+ out.writeByte(this.getMajorVersion());
+ out.writeByte(this.getMinorVersion());
+ out.write(LogFileHeader.MAGIC);
+ }
+
+ @Override
+ public int getSerializedLength() {
+ return Bytes.SIZEOF_INT // Protobuf message length
+ // Add the size of the Protobuf serialized message when we have one
+ + FIXED_TRAILER_SIZE;
+ }
+
+ public static boolean isValidTrailer(final FileSystem fs, final Path path) throws IOException {
+ try (FSDataInputStream in = fs.open(path)) {
+ return isValidTrailer(in, fs.getFileStatus(path).getLen());
+ }
+ }
+
+ public static boolean isValidTrailer(FSDataInputStream in, long length) throws IOException {
+ long offset = length - VERSION_AND_MAGIC_SIZE;
+ if (offset < 0) {
+ return false;
+ }
+ in.seek(offset);
+ byte[] magic = new byte[LogFileHeader.MAGIC.length];
+ in.readFully(magic);
+ if (!Arrays.equals(LogFileHeader.MAGIC, magic)) {
+ return false;
+ }
+ int majorVersion = in.readByte();
+ in.readByte(); // minorVersion, for now we don't use it
+ // Basic version check for now. We assume semver conventions where only higher major
+ // versions may be incompatible.
+ if (majorVersion > LogFileHeader.VERSION_MAJOR) {
+ return false;
+ }
+ return true;
+ }
+
+ @Override
+ public String toString() {
+ return "LogFileTrailer [majorVersion=" + majorVersion + ", minorVersion=" + minorVersion
+ + ", recordCount=" + recordCount + ", blockCount=" + blockCount
+ + ", blocksStartOffset=" + blocksStartOffset + ", trailerStartOffset="
+ + trailerStartOffset + "]";
+ }
+
+}
diff --git a/phoenix-core-server/src/main/java/org/apache/phoenix/replication/log/LogFileWriter.java b/phoenix-core-server/src/main/java/org/apache/phoenix/replication/log/LogFileWriter.java
new file mode 100644
index 0000000000..3bf71e1f99
--- /dev/null
+++ b/phoenix-core-server/src/main/java/org/apache/phoenix/replication/log/LogFileWriter.java
@@ -0,0 +1,118 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.replication.log;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.hbase.client.Mutation;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Writer for Phoenix Replication Log files.
+ * Manages writing the header, blocks (via LogFormatWriter), and trailer.
+ */
+public class LogFileWriter implements LogFile.Writer {
+
+ private static final Logger LOG = LoggerFactory.getLogger(LogFileWriter.class);
+
+ private LogFileWriterContext context;
+ private LogFileFormatWriter writer;
+ private boolean closed = false;
+
+ public LogFileWriter() {
+
+ }
+
+ public LogFileWriterContext getContext() {
+ return context;
+ }
+
+ @Override
+ public void init(LogFileWriterContext context) throws IOException {
+ this.context = context;
+ this.writer = new LogFileFormatWriter();
+ // TODO: Handle stream creation with proper permissions and overwrite options based on
+ // config. For now we overwrite.
+ FileSystem fs = context.getFileSystem();
+ FSDataOutputStream out = fs.create(context.getFilePath(), true);
+ this.writer.init(context, new HDFSDataOutput(out));
+ LOG.debug("Initialized LogFileWriter for path {}", context.getFilePath());
+ }
+
+ @Override
+ public void append(String tableName, long commitId, Mutation mutation) throws IOException {
+ if (closed) {
+ throw new IOException("Writer has been closed");
+ }
+ writer.append(new LogFileRecord().setHBaseTableName(tableName).setCommitId(commitId)
+ .setMutation(mutation));
+ }
+
+ @Override
+ public void sync() throws IOException {
+ if (closed) {
+ throw new IOException("Writer has been closed");
+ }
+ writer.sync();
+ }
+
+ @Override
+ public long getLength() throws IOException {
+ if (closed) {
+ // Attempt to get length from filesystem if stream is closed
+ if (context.getFileSystem().exists(context.getFilePath())) {
+ return context.getFileSystem().getFileStatus(context.getFilePath()).getLen();
+ } else {
+ throw new FileNotFoundException("LogFile not found at path "
+ + context.getFilePath());
+ }
+ }
+ if (writer == null) {
+ return 0; // Not initialized or already closed cleanly
+ }
+ // Return the current position for an open file being written
+ return writer.getPosition();
+ }
+
+ @Override
+ public void close() throws IOException {
+ if (closed) {
+ return;
+ }
+ try {
+ // Close the final block and write the trailer
+ if (writer != null) {
+ writer.close();
+ }
+ } finally {
+ closed = true;
+ LOG.debug("Closed LogFileWriter for path {}", context.getFilePath());
+ }
+ }
+
+ @Override
+ public String toString() {
+ return "LogFileWriter [writerContext=" + context + ", formatWriter=" + writer
+ + ", closed=" + closed + "]";
+ }
+
+}
diff --git a/phoenix-core-server/src/main/java/org/apache/phoenix/replication/log/LogFileWriterContext.java b/phoenix-core-server/src/main/java/org/apache/phoenix/replication/log/LogFileWriterContext.java
new file mode 100644
index 0000000000..756079a1f7
--- /dev/null
+++ b/phoenix-core-server/src/main/java/org/apache/phoenix/replication/log/LogFileWriterContext.java
@@ -0,0 +1,129 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.replication.log;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.io.compress.Compression;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+/**
+ * Context for LogFileWriter. Uses Builder pattern.
+ */
+@edu.umd.cs.findbugs.annotations.SuppressWarnings(value = { "EI_EXPOSE_REP", "EI_EXPOSE_REP2" },
+ justification = "Intentional")
+public class LogFileWriterContext {
+
+ private static final Logger LOG = LoggerFactory.getLogger(LogFileWriterContext.class);
+
+ /** Configuration key for the block size */
+ public static final String LOGFILE_BLOCK_SIZE = "phoenix.replication.logfile.block.size";
+ /** Default block size for replication logs (e.g., 1MB) */
+ public static final long DEFAULT_LOGFILE_BLOCK_SIZE = 1L * 1024 * 1024; // 1 MB
+
+ /** Configuration key for compression type */
+ public static final String LOGFILE_COMPRESSION = "phoenix.replication.logfile.compression";
+ /** Default block size for replication logs (e.g., 1MB) */
+ public static final String DEFAULT_LOGFILE_COMPRESSION = Compression.Algorithm.NONE.name();
+
+ private final Configuration conf;
+ private FileSystem fs;
+ private Path path;
+ private Compression.Algorithm compression;
+ private LogFileCodec codec;
+ private long maxBlockSize;
+
+ public LogFileWriterContext(Configuration conf) {
+ this.conf = conf;
+ try {
+ this.compression =
+ Compression.getCompressionAlgorithmByName(conf.get(LOGFILE_COMPRESSION,
+ DEFAULT_LOGFILE_COMPRESSION));
+ } catch (IllegalArgumentException e) {
+ // "NONE" is not actually a valid compression algorithm name
+ this.compression = Compression.Algorithm.NONE;
+ }
+ this.maxBlockSize = conf.getLong(LOGFILE_BLOCK_SIZE, DEFAULT_LOGFILE_BLOCK_SIZE);
+ if (this.maxBlockSize <= 0) {
+ LOG.warn("Invalid {} configured: {}. Using default: {}", LOGFILE_BLOCK_SIZE,
+ this.maxBlockSize, DEFAULT_LOGFILE_BLOCK_SIZE);
+ this.maxBlockSize = DEFAULT_LOGFILE_BLOCK_SIZE;
+ }
+ // Note: When we have multiple codec types, instantiate the appropriate type based on
+ // configuration;
+ this.codec = new LogFileCodec();
+ }
+
+ public Configuration getConfiguration() {
+ return conf;
+ }
+
+ public FileSystem getFileSystem() {
+ return fs;
+ }
+
+ public Path getFilePath() {
+ return path;
+ }
+
+ public LogFileWriterContext setFilePath(Path path) {
+ this.path = path;
+ return this;
+ }
+
+ public LogFileWriterContext setFileSystem(FileSystem fs) {
+ this.fs = fs;
+ return this;
+ }
+
+ public Compression.Algorithm getCompression() {
+ return compression;
+ }
+
+ public LogFileWriterContext setCompression(Compression.Algorithm compression) {
+ this.compression = compression;
+ return this;
+ }
+
+ public LogFileCodec getCodec() {
+ return codec;
+ }
+
+ public LogFileWriterContext setCodec(LogFileCodec codec) {
+ this.codec = codec;
+ return this;
+ }
+
+ public long getMaxBlockSize() {
+ return maxBlockSize;
+ }
+
+ public LogFileWriterContext setMaxBlockSize(long maxBlockSize) {
+ this.maxBlockSize = maxBlockSize;
+ return this;
+ }
+
+ @Override
+ public String toString() {
+ return "LogFileWriterContext [path=" + path + ", compression=" + compression + ", codec="
+ + codec + ", maxBlockSize=" + maxBlockSize + "]";
+ }
+
+}
diff --git a/phoenix-core-server/src/main/java/org/apache/phoenix/replication/log/SeekableDataInput.java b/phoenix-core-server/src/main/java/org/apache/phoenix/replication/log/SeekableDataInput.java
new file mode 100644
index 0000000000..62f2fa4dca
--- /dev/null
+++ b/phoenix-core-server/src/main/java/org/apache/phoenix/replication/log/SeekableDataInput.java
@@ -0,0 +1,36 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.replication.log;
+
+import java.io.Closeable;
+import java.io.DataInput;
+import java.io.IOException;
+
+import org.apache.hadoop.fs.PositionedReadable;
+import org.apache.hadoop.fs.Seekable;
+
+/**
+ * The parts of the HDFS FSDataInputStream interface contract that we want to generalize.
+ */
+public interface SeekableDataInput extends DataInput, Closeable, Seekable, PositionedReadable {
+
+ int read(byte[] buf) throws IOException;
+
+ int read(byte[] buf, int pos, int len) throws IOException;
+
+}
diff --git a/phoenix-core-server/src/main/java/org/apache/phoenix/replication/log/SyncableDataOutput.java b/phoenix-core-server/src/main/java/org/apache/phoenix/replication/log/SyncableDataOutput.java
new file mode 100644
index 0000000000..84eefdb326
--- /dev/null
+++ b/phoenix-core-server/src/main/java/org/apache/phoenix/replication/log/SyncableDataOutput.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.replication.log;
+
+import java.io.Closeable;
+import java.io.DataOutput;
+import java.io.IOException;
+
+import org.apache.hadoop.fs.Syncable;
+
+/**
+ * Extends DataOutput with a sync method for durability guarantees.
+ * Allows abstracting over different underlying output stream implementations
+ * like FSDataOutputStream or AsyncFSOutput.
+ */
+public interface SyncableDataOutput extends DataOutput, Syncable, Closeable {
+
+ /**
+ * Flushes buffered data and ensures durability based on the underlying
+ * implementation's guarantees (e.g., hsync for HDFS).
+ * @throws IOException if an I/O error occurs during sync.
+ */
+ void sync() throws IOException;
+
+ /**
+ * Returns the current position of the output, including any buffered data.
+ * Note: The exact meaning might differ slightly based on the implementation
+ * (e.g., synchronous vs asynchronous buffering).
+ * @return the current position of the output.
+ * @throws IOException if the position cannot be determined.
+ */
+ long getPos() throws IOException;
+
+}
diff --git a/phoenix-core-server/src/main/java/org/apache/phoenix/replication/log/package-info.java b/phoenix-core-server/src/main/java/org/apache/phoenix/replication/log/package-info.java
new file mode 100644
index 0000000000..12bfdbe70e
--- /dev/null
+++ b/phoenix-core-server/src/main/java/org/apache/phoenix/replication/log/package-info.java
@@ -0,0 +1,53 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Provides classes for managing the custom binary log format used in Phoenix replication. This
+ * package defines the structure of the replication log file, including its header, blocks,
+ * trailer, and individual records representing mutations.
+ * <p>
+ * Key components include:
+ * <ul>
+ * <li>{@link org.apache.phoenix.replication.log.LogFile}: Defines the core interfaces for the
+ * log file structure ({@code Header}, {@code BlockHeader}, {@code Trailer}, {@code Record})
+ * and its I/O operations ({@code Writer}, {@code Reader}, {@code Codec}).</li>
+ * <li>{@link org.apache.phoenix.replication.log.LogFileWriter}: Implementation for writing
+ * replication log files.</li>
+ * <li>{@link org.apache.phoenix.replication.log.LogFileReader}: Implementation for reading
+ * replication log files.</li>
+ * <li>{@link org.apache.phoenix.replication.log.LogFileFormatWriter}: Handles low-level writing
+ * of blocks, including compression and checksums.</li>
+ * <li>{@link org.apache.phoenix.replication.log.LogFileFormatReader}: Handles low-level reading
+ * of blocks, including decompression and checksum validation.</li>
+ * <li>{@link org.apache.phoenix.replication.log.LogFileCodec}: Default implementation for
+ * encoding/decoding {@code Record} objects within blocks.</li>
+ * <li>{@link org.apache.phoenix.replication.log.LogFileRecord}: Concrete implementation of a
+ * log {@code Record}.</li>
+ * <li>{@link org.apache.phoenix.replication.log.LogFileHeader},
+ * {@link org.apache.phoenix.replication.log.LogBlockHeader},
+ * {@link org.apache.phoenix.replication.log.LogFileTrailer}: Concrete implementations for
+ * file/block metadata.</li>
+ * <li>{@link org.apache.phoenix.replication.log.LogFileWriterContext},
+ * {@link org.apache.phoenix.replication.log.LogFileReaderContext}: Context objects holding
+ * configuration and state for writers and readers.</li>
+ * <li>{@link org.apache.phoenix.replication.log.SyncableDataOutput},
+ * {@link org.apache.phoenix.replication.log.SeekableDataInput}: Abstractions for underlying
+ * storage output/input streams.</li>
+ * </ul>
+ */
+package org.apache.phoenix.replication.log;
diff --git a/phoenix-core/pom.xml b/phoenix-core/pom.xml
index 581506f33b..c049ab4331 100644
--- a/phoenix-core/pom.xml
+++ b/phoenix-core/pom.xml
@@ -239,6 +239,11 @@
<artifactId>hbase-mapreduce</artifactId>
<scope>test</scope>
</dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-compression-aircompressor</artifactId>
+ <scope>test</scope>
+ </dependency>
<!-- HBase Adjacent Dependencies -->
<dependency>
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/replication/log/CRC64Test.java b/phoenix-core/src/test/java/org/apache/phoenix/replication/log/CRC64Test.java
new file mode 100644
index 0000000000..2771b2697c
--- /dev/null
+++ b/phoenix-core/src/test/java/org/apache/phoenix/replication/log/CRC64Test.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.replication.log;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotEquals;
+
+import java.nio.charset.StandardCharsets;
+
+import org.apache.phoenix.replication.log.CRC64;
+import org.junit.Test;
+
+public class CRC64Test {
+
+ /**
+ * Test CRC64 calculation against a known value.
+ * The value 0x995dc9bbdf1939faL is the expected CRC64-ECMA result for the ASCII string "123456789".
+ */
+ @Test
+ public void testKnownValue() {
+ CRC64 crc = new CRC64();
+ byte[] input = "123456789".getBytes(StandardCharsets.UTF_8);
+ long expectedCRC = 0x995dc9bbdf1939faL; // Standard CRC64-ECMA value for "123456789"
+ crc.update(input);
+ long actualCRC = crc.getValue();
+ assertEquals("CRC64 for '123456789' does not match known value", expectedCRC, actualCRC);
+ }
+
+ /**
+ * Test CRC64 calculation for an empty input byte array.
+ * The expected result for an empty input with initial -1 and final XOR -1 is 0.
+ */
+ @Test
+ public void testEmptyInput() {
+ CRC64 crc = new CRC64();
+ byte[] emptyInput = new byte[0];
+ long expectedCRC = 0L; // ~(-1) = 0
+ crc.update(emptyInput);
+ long actualCRC = crc.getValue();
+ assertEquals("CRC64 for empty input should be 0", expectedCRC, actualCRC);
+ }
+
+ /**
+ * Test the reset() method to ensure the CRC state is correctly reset.
+ */
+ @Test
+ public void testReset() {
+ CRC64 crc = new CRC64();
+ byte[] input1 = "TestData1".getBytes(StandardCharsets.UTF_8);
+ byte[] input2 = "TestData2".getBytes(StandardCharsets.UTF_8);
+ // Calculate CRC for first input
+ crc.update(input1);
+ long crc1 = crc.getValue();
+ // Reset and calculate CRC for second input
+ crc.reset();
+ crc.update(input2);
+ long crc2 = crc.getValue();
+ // Ensure the two CRC values are different (unless inputs are identical)
+ assertNotEquals("CRCs should be different after reset", crc1, crc2);
+ // Recalculate CRC for first input after reset to ensure it matches the original
+ crc.reset();
+ crc.update(input1);
+ long crc1Again = crc.getValue();
+ assertEquals("CRC after reset should match original CRC", crc1, crc1Again);
+ }
+
+}
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/replication/log/LogFileCodecTest.java b/phoenix-core/src/test/java/org/apache/phoenix/replication/log/LogFileCodecTest.java
new file mode 100644
index 0000000000..49a3754b27
--- /dev/null
+++ b/phoenix-core/src/test/java/org/apache/phoenix/replication/log/LogFileCodecTest.java
@@ -0,0 +1,322 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.replication.log;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Random;
+
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class LogFileCodecTest {
+
+ private static final Logger LOG = LoggerFactory.getLogger(LogFileCodecTest.class);
+
+ private static Random RNG = new Random();
+
+ @Test
+ public void testLogFileCodecSinglePut() throws IOException {
+ singleRecordTest(LogFileTestUtil.newPutRecord("TBLPUT", 10L, "row1", 12345L, 1));
+ }
+
+ @Test
+ public void testLogFileCodecSingleDelete() throws IOException {
+ singleRecordTest(LogFileTestUtil.newDeleteRecord("TBLDEL", 10L, "row1", 12345L, 1));
+ }
+
+ @Test
+ public void testLogFileCodecSingleDeleteColumn() throws IOException {
+ singleRecordTest(LogFileTestUtil.newDeleteColumnRecord("TBLDELCOL", 10L, "row1",
+ 12345L, 1));
+ }
+
+ @Test
+ public void testLogFileCodecSingleDeleteFamily() throws IOException {
+ singleRecordTest(LogFileTestUtil.newDeleteFamilyRecord("TBLDELFAM", 10L, "row1",
+ 12345L, 1));
+ }
+
+ @Test
+ public void testLogFileCodecSingleDeleteFamilyVersion() throws IOException {
+ singleRecordTest(LogFileTestUtil.newDeleteFamilyVersionRecord("TBLDELFAMVER", 10L, "row1",
+ 12345L, 1));
+ }
+
+ private void singleRecordTest(LogFile.Record original) throws IOException {
+ LogFileCodec codec = new LogFileCodec();
+ // Encode
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ DataOutputStream dos = new DataOutputStream(baos);
+ LogFile.Codec.Encoder encoder = codec.getEncoder(dos);
+ encoder.write(original);
+ LOG.debug("Encoded " + original);
+ dos.close();
+ byte[] encodedBytes = baos.toByteArray();
+
+ // Decode
+ ByteArrayInputStream bais = new ByteArrayInputStream(encodedBytes);
+ DataInputStream dis = new DataInputStream(bais);
+ LogFile.Codec.Decoder decoder = codec.getDecoder(dis);
+
+ assertTrue("Should be able to advance decoder", decoder.advance());
+ LogFileRecord decoded = (LogFileRecord) decoder.current();
+ LOG.debug("Decoded " + decoded);
+
+ // Verify length was set
+ assertTrue("Serialized length should be greater than 0",
+ decoded.getSerializedLength() > 0);
+ LogFileTestUtil.assertRecordEquals("Decoded record should match original", original,
+ decoded);
+
+ assertFalse("Should be no more records", decoder.advance());
+ }
+
+ @Test
+ public void testLogFileCodecMultipleRecords() throws IOException {
+ LogFileCodec codec = new LogFileCodec();
+ List<LogFile.Record> originals = Arrays.asList(
+ LogFileTestUtil.newPutRecord("TBL1", 100L, "row1", 12345L, 1),
+ LogFileTestUtil.newPutRecord("TBL2", 101L, "row2", 12346L, 2),
+ LogFileTestUtil.newPutRecord("TBL1", 102L, "row3", 12347L, 0) // No columns
+ );
+
+ // Encode
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ DataOutputStream dos = new DataOutputStream(baos);
+ LogFile.Codec.Encoder encoder = codec.getEncoder(dos);
+ for (LogFile.Record record : originals) {
+ encoder.write(record);
+ LOG.info("Encoded: size={} record={}", record.getSerializedLength(), record);
+ }
+ dos.close();
+ byte[] encodedBytes = baos.toByteArray();
+
+ // Decode
+ ByteArrayInputStream bais = new ByteArrayInputStream(encodedBytes);
+ DataInputStream dis = new DataInputStream(bais);
+ LogFile.Codec.Decoder decoder = codec.getDecoder(dis);
+
+ List<LogFile.Record> decodedRecords = new ArrayList<>();
+ while (decoder.advance()) {
+ LogFile.Record record = decoder.current();
+ LOG.info("Decoded: record={}", record);
+ decodedRecords.add(record);
+ }
+
+ assertEquals("Number of decoded records should match", originals.size(),
+ decodedRecords.size());
+ for (int i = 0; i < originals.size(); i++) {
+ LogFileTestUtil.assertRecordEquals("Record " + i + " should match", originals.get(i),
+ decodedRecords.get(i));
+ assertTrue("Serialized length should be set",
+ decodedRecords.get(i).getSerializedLength() > 0);
+ }
+ }
+
+ @Test
+ public void testLogFileCodecManyRecords() throws IOException {
+ LogFileCodec codec = new LogFileCodec();
+ List<LogFile.Record> originals = new ArrayList<>();
+ for (int i = 0; i < 100_000; i++) {
+ switch (RNG.nextInt(4)) {
+ case 0:
+ originals.add(LogFileTestUtil.newPutRecord("TBLMR", 10L + i, "row" + i,
+ 12345L + i, 1));
+ break;
+ case 1:
+ originals.add(LogFileTestUtil.newDeleteRecord("TBLMR", 10L + i, "row" + i,
+ 12345L + i, 1));
+ break;
+ case 2:
+ originals.add(LogFileTestUtil.newDeleteFamilyRecord("TBLMR", 10L + i,
+ "row" + i, 12345L + i, 1));
+ break;
+ case 3:
+ originals.add(LogFileTestUtil.newDeleteFamilyVersionRecord("TBLMR", 10L + i,
+ "row" + i, 12345L + i, 1));
+ break;
+ }
+ }
+
+ // Encode
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ DataOutputStream dos = new DataOutputStream(baos);
+ LogFile.Codec.Encoder encoder = codec.getEncoder(dos);
+ long start = System.currentTimeMillis();
+ for (LogFile.Record record : originals) {
+ encoder.write(record);
+ }
+ dos.close();
+ long end = System.currentTimeMillis();
+ byte[] encodedBytes = baos.toByteArray();
+
+ LOG.info("Encoded {} records into {} bytes (avg {} bytes/record) in {} ms",
+ originals.size(), encodedBytes.length, encodedBytes.length / originals.size(),
+ end - start);
+
+ // Decode
+ ByteArrayInputStream bais = new ByteArrayInputStream(encodedBytes);
+ DataInputStream dis = new DataInputStream(bais);
+ LogFile.Codec.Decoder decoder = codec.getDecoder(dis);
+ List<LogFile.Record> decodedRecords = new ArrayList<>();
+ start = System.currentTimeMillis();
+ while (decoder.advance()) {
+ LogFile.Record record = decoder.current();
+ decodedRecords.add(record);
+ }
+ end = System.currentTimeMillis();
+
+ LOG.info("Decoded {} records in {} ms", decodedRecords.size(), end - start);
+
+ assertEquals("Number of decoded records should match", originals.size(),
+ decodedRecords.size());
+ for (int i = 0; i < originals.size(); i++) {
+ LogFileTestUtil.assertRecordEquals("Record " + i + " should match", originals.get(i),
+ decodedRecords.get(i));
+ assertTrue("Serialized length should be set",
+ decodedRecords.get(i).getSerializedLength() > 0);
+ }
+ }
+
+ @Test
+ public void testLogFileCodecDecodeFromByteBuffer() throws IOException {
+ LogFileCodec codec = new LogFileCodec();
+ LogFile.Record original = LogFileTestUtil.newPutRecord("TBLBB", 200L, "row1", 54321L, 1);
+
+ // Encode
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ DataOutputStream dos = new DataOutputStream(baos);
+ LogFile.Codec.Encoder encoder = codec.getEncoder(dos);
+ encoder.write(original);
+ dos.close();
+ byte[] encodedBytes = baos.toByteArray();
+
+ // Decode using ByteBuffer
+ ByteBuffer buffer = ByteBuffer.wrap(encodedBytes);
+ LogFile.Codec.Decoder decoder = codec.getDecoder(buffer);
+
+ assertTrue("Should be able to advance decoder from ByteBuffer", decoder.advance());
+ LogFileRecord decoded = (LogFileRecord) decoder.current();
+ LogFileTestUtil.assertRecordEquals("Decoded record from ByteBuffer should match original",
+ original, decoded);
+ assertFalse("Should be no more records in ByteBuffer", decoder.advance());
+ }
+
+ // Edge cases
+
+ @Test
+ public void testCodecWithEmptyTableName() throws IOException {
+ singleRecordTest(LogFileTestUtil.newPutRecord("", 1L, "row", 12345L, 1));
+ }
+
+ @Test
+ public void testCodecWithEmptyFamily() throws IOException {
+ long ts = 12345L;
+ Put put = new Put(Bytes.toBytes("row"));
+ put.setTimestamp(ts);
+ put.addColumn(HConstants.EMPTY_BYTE_ARRAY, Bytes.toBytes("q"), ts, Bytes.toBytes("v"));
+ singleRecordTest(new LogFileRecord().setHBaseTableName("TBLEMPTYFAM").setCommitId(1L)
+ .setMutation(put));
+ }
+
+ @Test
+ public void testCodecWithEmptyQualifier() throws IOException {
+ long ts = 12345L;
+ Put put = new Put(Bytes.toBytes("row"));
+ put.setTimestamp(ts);
+ put.addColumn(Bytes.toBytes("cf"), HConstants.EMPTY_BYTE_ARRAY, ts, Bytes.toBytes("v"));
+ singleRecordTest(new LogFileRecord().setHBaseTableName("TBLEMPTYFAM").setCommitId(1L)
+ .setMutation(put));
+ }
+
+ @Test
+ public void testCodecWithEmptyValue() throws IOException {
+ long ts = 12345L;
+ Put put = new Put(Bytes.toBytes("row"));
+ put.setTimestamp(ts);
+ put.addColumn(Bytes.toBytes("cf"), Bytes.toBytes("q"), ts, HConstants.EMPTY_BYTE_ARRAY);
+ singleRecordTest(new LogFileRecord().setHBaseTableName("TBLEMPTYVAL").setCommitId(1L)
+ .setMutation(put));
+ }
+
+ @Test
+ public void testCodecWithManyValues() throws IOException {
+ // 100 columns
+ singleRecordTest(LogFileTestUtil.newPutRecord("TBLMANYVALS", 1L, "row", 12345L, 100));
+ }
+
+ @Test
+ public void testCodecWithEmptyPut() throws IOException {
+ long ts = 12345L;
+ Put put = new Put(Bytes.toBytes("row"));
+ put.setTimestamp(ts);
+ singleRecordTest(new LogFileRecord().setHBaseTableName("TBLEMPTYPUT").setCommitId(1L)
+ .setMutation(put));
+ }
+
+ @Test
+ public void testCodecWithEmptyDelete() throws IOException {
+ long ts = 12345L;
+ Delete delete = new Delete(Bytes.toBytes("row"));
+ delete.setTimestamp(ts);
+ singleRecordTest(new LogFileRecord().setHBaseTableName("TBLEMPTYDEL").setCommitId(1L)
+ .setMutation(delete));
+ }
+
+ @Test
+ public void testCodecWithLargeRowKey() throws IOException {
+ long ts = 12345L;
+ byte[] largeRowKey = new byte[HConstants.MAX_ROW_LENGTH];
+ RNG.nextBytes(largeRowKey);
+ Put put = new Put(largeRowKey);
+ put.setTimestamp(ts);
+ put.addColumn(Bytes.toBytes("cf"), Bytes.toBytes("q"), ts, Bytes.toBytes("v"));
+ singleRecordTest(new LogFileRecord().setHBaseTableName("TBLLRK").setCommitId(1L)
+ .setMutation(put));
+ }
+
+ @Test
+ public void testCodecWithLargeValue() throws IOException {
+ long ts = 12345L;
+ byte[] largeValue = new byte[1024 * 1024 * 10]; // 10 MB value
+ RNG.nextBytes(largeValue);
+ Put put = new Put(Bytes.toBytes("row"));
+ put.setTimestamp(ts);
+ put.addColumn(Bytes.toBytes("cf"), Bytes.toBytes("q"), ts, largeValue);
+ singleRecordTest(new LogFileRecord().setHBaseTableName("TBLLVAL").setCommitId(1L)
+ .setMutation(put));
+ }
+
+}
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/replication/log/LogFileCompressionTest.java b/phoenix-core/src/test/java/org/apache/phoenix/replication/log/LogFileCompressionTest.java
new file mode 100644
index 0000000000..05c00429ae
--- /dev/null
+++ b/phoenix-core/src/test/java/org/apache/phoenix/replication/log/LogFileCompressionTest.java
@@ -0,0 +1,225 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.replication.log;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.Iterator;
+import java.util.List;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.io.compress.Compression;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.junit.rules.TestName;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+import org.junit.runners.Parameterized.Parameters;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+@RunWith(Parameterized.class)
+public class LogFileCompressionTest {
+
+ private static final Logger LOG = LoggerFactory.getLogger(LogFileCompressionTest.class);
+
+ @ClassRule
+ public static TemporaryFolder testFolder = new TemporaryFolder();
+ @Rule
+ public TestName testName = new TestName();
+
+ private static Configuration conf;
+ private FileSystem localFs;
+ private Path filePath;
+ private Compression.Algorithm compression;
+ private LogFileReaderContext readerContext;
+ private LogFileReader reader;
+ private LogFileWriterContext writerContext;
+ private LogFileWriter writer;
+
+ public LogFileCompressionTest(Compression.Algorithm compression) {
+ this.compression = compression;
+ }
+
+ @Parameters(name = "Compression={0}")
+ public static Collection<Object[]> data() {
+ List<Object[]> params = new ArrayList<>();
+ // NONE
+ params.add(new Object[]{Compression.Algorithm.NONE});
+ // LZ4, LZO, SNAPPY, and ZSTD provided by hbase-compression-aircompressor
+ params.add(new Object[]{Compression.Algorithm.LZ4});
+ params.add(new Object[]{Compression.Algorithm.LZO});
+ params.add(new Object[]{Compression.Algorithm.SNAPPY});
+ params.add(new Object[]{Compression.Algorithm.ZSTD});
+ // GZ provided by the JRE
+ params.add(new Object[]{Compression.Algorithm.GZ});
+ return params;
+ }
+
+ @BeforeClass
+ public static void setUpBeforeClass() {
+ conf = HBaseConfiguration.create();
+ // Use the pure java AirCompressor codecs so we don't need to worry about environment
+ // issues.
+ conf.set("hbase.io.compress.lz4.codec",
+ "org.apache.hadoop.hbase.io.compress.aircompressor.Lz4Codec");
+ Compression.Algorithm.LZ4.reload(conf);
+ conf.set("hbase.io.compress.lzo.codec",
+ "org.apache.hadoop.hbase.io.compress.aircompressor.LzoCodec");
+ Compression.Algorithm.LZO.reload(conf);
+ conf.set("hbase.io.compress.snappy.codec",
+ "org.apache.hadoop.hbase.io.compress.aircompressor.SnappyCodec");
+ Compression.Algorithm.SNAPPY.reload(conf);
+ conf.set("hbase.io.compress.zstd.codec",
+ "org.apache.hadoop.hbase.io.compress.aircompressor.ZstdCodec");
+ Compression.Algorithm.ZSTD.reload(conf);
+ }
+
+ @Before
+ public void setUp() throws IOException {
+ localFs = FileSystem.getLocal(conf);
+ // Use a unique path for each test instance based on compression
+ filePath = new Path(testFolder.newFile("LogCompressionTest_"
+ + testName.getMethodName()).getAbsolutePath());
+ reader = new LogFileReader();
+ writer = new LogFileWriter();
+ }
+
+ @After
+ public void tearDown() throws IOException {
+ if (writer != null) {
+ writer.close();
+ }
+ if (reader != null) {
+ reader.close();
+ }
+ }
+
+ @Test
+ public void testLogFileSingleBlockWithCompression() throws IOException {
+ LOG.info("Testing single block with compression {}", compression.getName());
+ initLogFileWriter(LogFileWriterContext.DEFAULT_LOGFILE_BLOCK_SIZE);
+ List<LogFile.Record> originals = new ArrayList<>();
+ for (int i = 0; i < 100; i++) {
+ LogFile.Record record = LogFileTestUtil.newPutRecord("TBLSBWC", i, "row" + i,
+ 100L + i, 2);
+ originals.add(record);
+ writer.append(record.getHBaseTableName(), record.getCommitId(), record.getMutation());
+ }
+ writer.close();
+ initLogFileReader();
+ readAndVerifyRecords(originals, 1); // Expect exactly one block
+ }
+
+ @Test
+ public void testLogFileMultipleBlocksWithCompression() throws IOException {
+ LOG.info("Testing multiple blocks with compression {}", compression.getName());
+ // Use a small block size to force multiple blocks
+ initLogFileWriter(8 * 1024); // 8k
+ List<LogFile.Record> originals = new ArrayList<>();
+ // Generate enough records to span multiple blocks
+ for (int i = 0; i < 100_000; i++) {
+ LogFile.Record record = LogFileTestUtil.newPutRecord("TBLMBWC", i, "row" + i,
+ 100L + i, 5);
+ originals.add(record);
+ writer.append(record.getHBaseTableName(), record.getCommitId(), record.getMutation());
+ }
+ writer.close();
+ initLogFileReader();
+ assertTrue("File has more than one block", reader.getTrailer().getBlockCount() > 1);
+ readAndVerifyRecords(originals, reader.getTrailer().getBlockCount());
+ }
+
+ @Test
+ public void testLogFileEmptyFile() throws IOException {
+ LOG.info("Testing empty file with compression {}", compression.getName());
+ initLogFileWriter(LogFileWriterContext.DEFAULT_LOGFILE_BLOCK_SIZE);
+ writer.close(); // Creates an empty file (header + trailer)
+ initLogFileReader();
+ assertNull("Next should return null for empty file", reader.next());
+ reader.close();
+ }
+
+ private void readAndVerifyRecords(List<LogFile.Record> originals, long expectedBlockCount)
+ throws IOException {
+ assertTrue("Test file does not exist: " + filePath, localFs.exists(filePath));
+ assertTrue("Test file has zero length: " + filePath,
+ localFs.getFileStatus(filePath).getLen() > 0);
+
+ // Verify Header
+ LogFile.Header header = reader.getHeader();
+ assertNotNull("Header should not be null", header);
+
+ // Read records using iterator
+ List<LogFile.Record> decoded = new ArrayList<>();
+ Iterator<LogFile.Record> iterator = reader.iterator();
+ while (iterator.hasNext()) {
+ decoded.add(iterator.next());
+ }
+
+ // Verify Records
+ assertEquals("Number of decoded records mismatch", originals.size(),
+ decoded.size());
+ for (int i = 0; i < originals.size(); i++) {
+ LogFileTestUtil.assertRecordEquals("Record " + i + " mismatch", originals.get(i),
+ decoded.get(i));
+ }
+
+ // Verify Trailer
+ LogFile.Trailer trailer = reader.getTrailer();
+ assertNotNull("Trailer should not be null", trailer);
+ assertEquals("Trailer record count mismatch", originals.size(),
+ trailer.getRecordCount());
+ assertEquals("Trailer block count mismatch", expectedBlockCount,
+ trailer.getBlockCount());
+
+ // Verify Reader Context Counters
+ assertEquals("Reader context record count mismatch", trailer.getRecordCount(),
+ readerContext.getRecordsRead());
+ assertEquals("Reader context block count mismatch", trailer.getBlockCount(),
+ readerContext.getBlocksRead());
+ assertEquals("Reader context corrupt blocks skipped count should be 0", 0,
+ readerContext.getCorruptBlocksSkipped());
+ }
+
+ private void initLogFileReader() throws IOException {
+ readerContext = new LogFileReaderContext(conf).setFileSystem(localFs)
+ .setFilePath(filePath);
+ reader.init(readerContext);
+ }
+
+ private void initLogFileWriter(long maxBlockSize) throws IOException {
+ writerContext = new LogFileWriterContext(conf).setCompression(compression)
+ .setFileSystem(localFs).setFilePath(filePath).setMaxBlockSize(maxBlockSize);
+ writer.init(writerContext);
+ }
+
+}
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/replication/log/LogFileFormatTest.java b/phoenix-core/src/test/java/org/apache/phoenix/replication/log/LogFileFormatTest.java
new file mode 100644
index 0000000000..875cb6ea1b
--- /dev/null
+++ b/phoenix-core/src/test/java/org/apache/phoenix/replication/log/LogFileFormatTest.java
@@ -0,0 +1,588 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.replication.log;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class LogFileFormatTest {
+
+ private static final Logger LOG = LoggerFactory.getLogger(LogFileFormatTest.class);
+
+ private Configuration conf;
+ private ByteArrayOutputStream writerBaos;
+ private DataOutputStream writerDos;
+ private LogFileReaderContext readerContext;
+ private LogFileFormatReader reader;
+ private LogFileWriterContext writerContext;
+ private LogFileFormatWriter writer;
+
+ @Before
+ public void setUp() {
+ conf = HBaseConfiguration.create();
+ readerContext = new LogFileReaderContext(conf)
+ .setSkipCorruptBlocks(true); // Enable skipping for corruption tests
+ reader = new LogFileFormatReader();
+ writerBaos = new ByteArrayOutputStream();
+ writerDos = new DataOutputStream(writerBaos);
+ writerContext = new LogFileWriterContext(conf);
+ writer = new LogFileFormatWriter();
+ }
+
+ @After
+ public void tearDown() throws IOException {
+ writerDos.close();
+ writerBaos.close();
+ writer.close();
+ reader.close();
+ }
+
+ @Test
+ public void testLogFileFormatSingleBlock() throws IOException {
+ initLogFileWriter();
+ LogFile.Record r1 = LogFileTestUtil.newPutRecord("TBL1", 1L, "row1", 10L, 1);
+ LogFile.Record r2 = LogFileTestUtil.newPutRecord("TBL1", 2L, "row2", 11L, 1);
+
+ writer.append(r1);
+ writer.append(r2);
+ writer.close(); // Writes block and trailer
+
+ byte[] data = writerBaos.toByteArray();
+ initLogFileReader(data);
+
+ assertEquals("Major version mismatch", LogFileHeader.VERSION_MAJOR,
+ reader.getHeader().getMajorVersion());
+ assertEquals("Minor version mismatch", LogFileHeader.VERSION_MINOR,
+ reader.getHeader().getMinorVersion());
+
+ LogFile.Record decoded1 = reader.next();
+ assertNotNull("First record should not be null", decoded1);
+ LogFileTestUtil.assertRecordEquals("First record mismatch", r1, decoded1);
+
+ LogFile.Record decoded2 = reader.next();
+ assertNotNull("Second record should not be null", decoded2);
+ LogFileTestUtil.assertRecordEquals("Second record mismatch", r2, decoded2);
+
+ assertNull("Should be no more records", reader.next());
+
+ LogFile.Trailer trailer = reader.getTrailer();
+ assertNotNull("Trailer should exist", trailer);
+ assertEquals("Trailer record count mismatch", 2, trailer.getRecordCount());
+ assertEquals("Trailer block count mismatch", 1, trailer.getBlockCount());
+ assertTrue("Blocks start offset should be positive", trailer.getBlocksStartOffset() > 0);
+ assertTrue("Trailer start offset should be > blocks start offset",
+ trailer.getTrailerStartOffset() >= trailer.getBlocksStartOffset());
+
+ assertEquals("Reader context record count mismatch", 2, readerContext.getRecordsRead());
+ assertEquals("Reader context block count mismatch", 1, readerContext.getBlocksRead());
+ }
+
+ @Test
+ public void testLogFileFormatMultipleBlocks() throws IOException {
+ initLogFileWriter();
+ List<LogFile.Record> originals = new ArrayList<>();
+ // Write enough records to cause multiple blocks
+ // This has the nice property of writing a large number of blocks compared with the other
+ // tests in this unit.
+ for (int i = 0; i < 100_000; i++) {
+ LogFile.Record r = LogFileTestUtil.newPutRecord("TBLMULTI", i, "row" + i, 10L + i, 2);
+ originals.add(r);
+ writer.append(r);
+ }
+ writer.close();
+
+ byte[] data = writerBaos.toByteArray();
+ initLogFileReader(data);
+
+ List<LogFile.Record> decoded = new ArrayList<>();
+ LogFile.Record r;
+ while ((r = reader.next()) != null) {
+ decoded.add(r);
+ }
+
+ assertEquals("Number of records mismatch", originals.size(), decoded.size());
+ for (int i = 0; i < originals.size(); i++) {
+ LogFileTestUtil.assertRecordEquals("Record " + i + " mismatch", originals.get(i),
+ decoded.get(i));
+ }
+
+ LogFile.Trailer trailer = reader.getTrailer();
+ assertNotNull("Trailer should exist", trailer);
+ assertEquals("Trailer record count mismatch", originals.size(), trailer.getRecordCount());
+ assertTrue("Trailer block count should be > 1", trailer.getBlockCount() > 1);
+ assertEquals("Reader context record count mismatch", originals.size(),
+ readerContext.getRecordsRead());
+ assertEquals("Reader context block count mismatch", trailer.getBlockCount(),
+ readerContext.getBlocksRead());
+ }
+
+ @Test
+ public void testLogFileFormatHeaderTrailerOnly() throws IOException {
+ // Write header and trailer, no blocks
+ initLogFileWriter();
+ writer.close();
+ byte[] data = writerBaos.toByteArray();
+ initLogFileReader(data);
+ assertEquals("Major version mismatch", LogFileHeader.VERSION_MAJOR,
+ reader.getHeader().getMajorVersion());
+ assertNull("Should be no records", reader.next());
+ LogFileTrailer trailer = (LogFileTrailer) reader.getTrailer();
+ assertNotNull("Trailer should exist", trailer);
+ assertEquals("Trailer record count should be 0", 0, trailer.getRecordCount());
+ assertEquals("Trailer block count should be 0", 0, trailer.getBlockCount());
+ // blocksStartOffset might be equal to trailerStartOffset if no blocks written
+ assertEquals("Blocks start offset should equal trailer start offset",
+ trailer.getBlocksStartOffset(), trailer.getTrailerStartOffset());
+ }
+
+ @Test
+ public void testLogFileCorruptionInvalidBlockMagic() throws IOException {
+ initLogFileWriter();
+ List<LogFile.Record> block1Records = writeBlock(writer, "B1", 0, 10);
+ List<LogFile.Record> block2Records = writeBlock(writer, "B2", 10, 10);
+ List<LogFile.Record> block3Records = writeBlock(writer, "B3", 20, 10);
+ writer.close();
+
+ byte[] data = writerBaos.toByteArray();
+
+ int block1End = findBlockEndOffset(data, (int)writer.getBlocksStartOffset());
+ data[(int)block1End] = (byte) 'X'; // Corrupt the first byte of 'PBLK'
+
+ initLogFileReader(data);
+
+ List<LogFile.Record> decoded = readRecords(reader);
+
+ int shouldHave = block1Records.size() + block3Records.size();
+ int have = decoded.size();
+ // Should read block 1 and block 3, skipping block 2
+ assertEquals("Should read records from block 1 and 3", shouldHave, have);
+ // Verify first block records
+ for (int i = 0; i < block1Records.size(); i++) {
+ LogFileTestUtil.assertRecordEquals("Block 1 record " + i + " mismatch",
+ block1Records.get(i), decoded.get(i));
+ }
+ // Verify third block records (offset by block1 size)
+ for (int i = 0; i < block3Records.size(); i++) {
+ LogFileTestUtil.assertRecordEquals("Block 3 record " + i + " mismatch",
+ block3Records.get(i), decoded.get(i + block1Records.size()));
+ }
+
+ assertEquals("Should have skipped 1 corrupt block", 1,
+ readerContext.getCorruptBlocksSkipped());
+ }
+
+ @Test
+ public void testLogFileCorruptionBadBlockChecksum() throws IOException {
+ initLogFileWriter();
+ List<LogFile.Record> block1Records = writeBlock(writer, "B1", 0, 5);
+ List<LogFile.Record> block2Records = writeBlock(writer, "B2", 5, 5);
+ List<LogFile.Record> block3Records = writeBlock(writer, "B3", 10, 5);
+ writer.close();
+
+ byte[] data = writerBaos.toByteArray();
+
+ // Find offset of second block's checksum and corrupt it
+ long block1End = findBlockEndOffset(data, (int)writer.getBlocksStartOffset());
+ long block2End = findBlockEndOffset(data, (int)block1End);
+ int block2ChecksumOffset = (int) (block2End - Bytes.SIZEOF_LONG);
+ data[block2ChecksumOffset] ^= 0xFF; // Flip some bits in the checksum
+
+ initLogFileReader(data);
+
+ List<LogFile.Record> decoded = readRecords(reader);
+
+ // Should read block 1 and block 3, skipping block 2
+ int shouldHave = block1Records.size() + block3Records.size();
+ int have = decoded.size();
+ assertEquals("Should read records from block 1 and 3", shouldHave, have);
+ // Verify first block records
+ for (int i = 0; i < block1Records.size(); i++) {
+ LogFileTestUtil.assertRecordEquals("Block 1 record " + i + " mismatch",
+ block1Records.get(i), decoded.get(i));
+ }
+ // Verify third block records (offset by block1 size)
+ for (int i = 0; i < block3Records.size(); i++) {
+ LogFileTestUtil.assertRecordEquals("Block 3 record " + i + " mismatch",
+ block3Records.get(i), decoded.get(i + block1Records.size()));
+ }
+
+ assertEquals("Should have skipped 1 corrupt block", 1,
+ readerContext.getCorruptBlocksSkipped());
+ }
+
+ @Test
+ public void testLogFileCorruptionBadRecord() throws IOException {
+ initLogFileWriter();
+ List<LogFile.Record> block1Records = writeBlock(writer, "B1", 0, 10);
+ List<LogFile.Record> block2Records = writeBlock(writer, "B2", 10, 10);
+ List<LogFile.Record> block3Records = writeBlock(writer, "B3", 20, 10);
+ writer.close();
+
+ byte[] data = writerBaos.toByteArray();
+
+ // Find offset of second block's first record and corrupt it
+ long block1End = findBlockEndOffset(data, (int)writer.getBlocksStartOffset());
+ long block2Start = block1End + LogBlockHeader.HEADER_SIZE;
+ data[(int)block2Start + 1] ^= 0xFF;
+
+ initLogFileReader(data);
+
+ List<LogFile.Record> decoded = readRecords(reader);
+
+ // Should read block 1 and block 3, skipping block 2
+ int shouldHave = block1Records.size() + block3Records.size();
+ int have = decoded.size();
+ assertEquals("Should read records from block 1 and 3", shouldHave, have);
+ // Verify first block records
+ for (int i = 0; i < block1Records.size(); i++) {
+ LogFileTestUtil.assertRecordEquals("Block 1 record " + i + " mismatch",
+ block1Records.get(i), decoded.get(i));
+ }
+ // Verify third block records (offset by block1 size)
+ for (int i = 0; i < block3Records.size(); i++) {
+ LogFileTestUtil.assertRecordEquals("Block 3 record " + i + " mismatch",
+ block3Records.get(i), decoded.get(i + block1Records.size()));
+ }
+
+ assertEquals("Should have skipped 1 corrupt block", 1,
+ readerContext.getCorruptBlocksSkipped());
+ }
+
+ @Test
+ public void testLogFileCorruptionTruncatedFinalBlock() throws IOException {
+ initLogFileWriter();
+ List<LogFile.Record> block1Records = writeBlock(writer, "B1", 0, 10);
+ List<LogFile.Record> block2Records = writeBlock(writer, "B2", 10, 10);
+ // Don't close the writer, simulate truncation within the last block
+ // Get the current position which is somewhere inside block 2's data
+ long truncationPoint = writer.getPosition() - 10; // Truncate 10 bytes before end
+
+ byte[] data = writerBaos.toByteArray();
+ byte[] truncatedData = Arrays.copyOf(data, (int) truncationPoint);
+
+ initLogFileReader(truncatedData);
+
+ List<LogFile.Record> decoded = new ArrayList<>();
+ LogFile.Record r;
+ try {
+ while ((r = reader.next()) != null) {
+ decoded.add(r);
+ }
+ // Depending on where truncation happened, next() might throw or return null.
+ // If it returns null cleanly, the counts should still be checked.
+ } catch (IOException e) {
+ // Expecting an EOF or similar if truncation happened mid-record/header/checksum
+ }
+
+ // Should have read only block 1 completely. Block 2 read might fail or be partial.
+ assertEquals("Should read records only from block 1", block1Records.size(),
+ decoded.size());
+ for (int i = 0; i < block1Records.size(); i++) {
+ LogFileTestUtil.assertRecordEquals("Block 1 record " + i + " mismatch",
+ block1Records.get(i), decoded.get(i));
+ }
+
+ // Depending on where the truncation happens, the block might be detected as corrupt or
+ // just end early.
+ assertTrue("Corrupt blocks skipped should be 0 or 1",
+ readerContext.getCorruptBlocksSkipped() <= 1);
+ assertEquals("Blocks read count should be 1 (Block 1)", 1, readerContext.getBlocksRead());
+ assertEquals("Records read count mismatch", block1Records.size(),
+ readerContext.getRecordsRead());
+ assertNull("Trailer should not be present", reader.getTrailer());
+ }
+
+ @Test
+ public void testLogFileCorruptionMissingTrailer() throws IOException {
+ initLogFileWriter();
+ List<LogFile.Record> block1Records = writeBlock(writer, "B1", 0, 5);
+ // Don't close the writer, simulate missing trailer
+ long trailerStartOffset = writer.getPosition(); // Position before trailer write
+
+ byte[] data = writerBaos.toByteArray();
+ byte[] truncatedData = Arrays.copyOf(data, (int) trailerStartOffset);
+
+ // Re-initialize reader with truncated data
+ LogFileTestUtil.SeekableByteArrayInputStream input =
+ new LogFileTestUtil.SeekableByteArrayInputStream(truncatedData);
+ readerContext.setFileSize(truncatedData.length);
+ // This init should log a warning but succeed
+ reader.init(readerContext, input);
+
+ List<LogFile.Record> decoded = readRecords(reader);
+
+ assertEquals("Should read all records from block 1", block1Records.size(), decoded.size());
+ for (int i = 0; i < block1Records.size(); i++) {
+ LogFileTestUtil.assertRecordEquals("Block 1 record " + i + " mismatch",
+ block1Records.get(i), decoded.get(i));
+ }
+
+ assertNull("Trailer should be null", reader.getTrailer());
+ assertEquals("Corrupt blocks skipped should be 0", 0,
+ readerContext.getCorruptBlocksSkipped());
+ assertEquals("Blocks read count mismatch", 1, readerContext.getBlocksRead());
+ assertEquals("Records read count mismatch", block1Records.size(),
+ readerContext.getRecordsRead());
+ }
+
+ @Test
+ public void testLogFileCorruptionPartialTrailer() throws IOException {
+ initLogFileWriter();
+ List<LogFile.Record> block1Records = writeBlock(writer, "B1", 0, 10);
+ List<LogFile.Record> block2Records = writeBlock(writer, "B2", 10, 10);
+ writer.close(); // Writes trailer
+ byte[] data = writerBaos.toByteArray();
+
+ // Truncate partially into the trailer
+ int truncationPoint = data.length - (LogFileTrailer.FIXED_TRAILER_SIZE / 2);
+ byte[] truncatedData = Arrays.copyOf(data, truncationPoint);
+
+ // Re-initialize reader with truncated data
+ LogFileTestUtil.SeekableByteArrayInputStream input =
+ new LogFileTestUtil.SeekableByteArrayInputStream(truncatedData);
+ readerContext.setFileSize(truncatedData.length);
+ // Init should log a warning but succeed by ignoring the trailer
+ reader.init(readerContext, input);
+
+ List<LogFile.Record> decoded = readRecords(reader);
+
+ // Should read all records from both blocks
+ int totalRecords = block1Records.size() + block2Records.size();
+ assertEquals("Should read all records from block 1 and 2", totalRecords, decoded.size());
+ for (int i = 0; i < block1Records.size(); i++) {
+ LogFileTestUtil.assertRecordEquals("Block 1 record " + i + " mismatch",
+ block1Records.get(i), decoded.get(i));
+ }
+ for (int i = 0; i < block2Records.size(); i++) {
+ LogFileTestUtil.assertRecordEquals("Block 2 record " + i + " mismatch",
+ block2Records.get(i), decoded.get(i + block1Records.size()));
+ }
+
+ assertNull("Trailer should be null due to corruption/truncation", reader.getTrailer());
+
+ // Because the trailer is corrupt we don't know when to stop trying to find blocks, and the
+ // last "block" will be the start of the truncated trailer, so we count it.
+ assertEquals("Corrupt blocks skipped should be 1", 1,
+ readerContext.getCorruptBlocksSkipped());
+
+ // However we should have read all of the blocks and records.
+ assertEquals("Blocks read count mismatch", 2, readerContext.getBlocksRead());
+ assertEquals("Records read count mismatch", totalRecords, readerContext.getRecordsRead());
+ }
+
+ @Test
+ public void testLogFileCorruptionFirstBlockChecksum() throws IOException {
+ initLogFileWriter();
+ List<LogFile.Record> block1Records = writeBlock(writer, "B1", 0, 5);
+ List<LogFile.Record> block2Records = writeBlock(writer, "B2", 5, 5);
+ List<LogFile.Record> block3Records = writeBlock(writer, "B3", 10, 5);
+ writer.close();
+ byte[] data = writerBaos.toByteArray();
+
+ // Find offset of first block's checksum and corrupt it
+ long block1End = findBlockEndOffset(data, (int) writer.getBlocksStartOffset());
+ int block1ChecksumOffset = (int) (block1End - Bytes.SIZEOF_LONG);
+ data[block1ChecksumOffset] ^= 0xFF; // Flip some bits
+
+ initLogFileReader(data);
+ List<LogFile.Record> decoded = readRecords(reader);
+
+ // Should read blocks 2 and 3, skipping block 1
+ int shouldHave = block2Records.size() + block3Records.size();
+ assertEquals("Should read records from block 2 and 3", shouldHave, decoded.size());
+ for (int i = 0; i < block2Records.size(); i++) {
+ LogFileTestUtil.assertRecordEquals("Block 2 record " + i + " mismatch",
+ block2Records.get(i), decoded.get(i));
+ }
+ for (int i = 0; i < block3Records.size(); i++) {
+ LogFileTestUtil.assertRecordEquals("Block 3 record " + i + " mismatch",
+ block3Records.get(i), decoded.get(i + block2Records.size()));
+ }
+
+ assertEquals("Should have skipped 1 corrupt block (block 1)", 1,
+ readerContext.getCorruptBlocksSkipped());
+ assertEquals("Blocks read count should be 2 (blocks 2 and 3)", 2,
+ readerContext.getBlocksRead());
+ }
+
+ @Test
+ public void testLogFileCorruptionLastBlockChecksum() throws IOException {
+ initLogFileWriter();
+ List<LogFile.Record> block1Records = writeBlock(writer, "B1", 0, 5);
+ List<LogFile.Record> block2Records = writeBlock(writer, "B2", 5, 5);
+ List<LogFile.Record> block3Records = writeBlock(writer, "B3", 10, 5);
+ writer.close(); // Close includes trailer
+ byte[] data = writerBaos.toByteArray();
+
+ // Find offset of last block's checksum and corrupt it
+ long block1End = findBlockEndOffset(data, (int) writer.getBlocksStartOffset());
+ long block2End = findBlockEndOffset(data, (int) block1End);
+ long block3End = findBlockEndOffset(data, (int) block2End); // This should be trailer start
+ int block3ChecksumOffset = (int) (block3End - Bytes.SIZEOF_LONG);
+ data[block3ChecksumOffset] ^= 0xFF; // Flip some bits
+
+ initLogFileReader(data);
+ List<LogFile.Record> decoded = readRecords(reader);
+
+ // Should read blocks 1 and 2, skipping block 3
+ int shouldHave = block1Records.size() + block2Records.size();
+ assertEquals("Should read records from block 1 and 2", shouldHave, decoded.size());
+ for (int i = 0; i < block1Records.size(); i++) {
+ LogFileTestUtil.assertRecordEquals("Block 1 record " + i + " mismatch",
+ block1Records.get(i), decoded.get(i));
+ }
+ for (int i = 0; i < block2Records.size(); i++) {
+ LogFileTestUtil.assertRecordEquals("Block 2 record " + i + " mismatch",
+ block2Records.get(i), decoded.get(i + block1Records.size()));
+ }
+
+ assertEquals("Should have skipped 1 corrupt block (block 3)", 1,
+ readerContext.getCorruptBlocksSkipped());
+ assertEquals("Blocks read count should be 2 (blocks 1 and 2)", 2,
+ readerContext.getBlocksRead());
+ // Trailer should still be readable as it's read first
+ assertNotNull("Trailer should still be readable", reader.getTrailer());
+ }
+
+ @Test
+ public void testLogFileCorruptionLastBlockPayloadNearEnd() throws IOException {
+ initLogFileWriter();
+ List<LogFile.Record> block1Records = writeBlock(writer, "B1", 0, 5);
+ List<LogFile.Record> block2Records = writeBlock(writer, "B2", 5, 5);
+ List<LogFile.Record> block3Records = writeBlock(writer, "B3", 10, 5);
+ writer.close(); // Close includes trailer
+ byte[] data = writerBaos.toByteArray();
+
+ // Find offset near the end of the last block's payload and corrupt it
+ long block1End = findBlockEndOffset(data, (int) writer.getBlocksStartOffset());
+ long block2End = findBlockEndOffset(data, (int) block1End);
+ long block3End = findBlockEndOffset(data, (int) block2End); // This should be trailer start
+ int block3PayloadEndOffset = (int) (block3End - Bytes.SIZEOF_LONG);
+ int corruptionOffset = block3PayloadEndOffset - 5; // Corrupt 5 bytes before checksum
+ if (corruptionOffset > block2End + LogBlockHeader.HEADER_SIZE) { // Ensure we are in payload
+ data[corruptionOffset] ^= 0xFF; // Flip some bits
+ } else {
+ fail("Calculated corruption offset is not within the last block's payload.");
+ }
+
+ initLogFileReader(data);
+ List<LogFile.Record> decoded = readRecords(reader);
+
+ // Should read blocks 1 and 2, skipping block 3 due to checksum mismatch
+ int shouldHave = block1Records.size() + block2Records.size();
+ assertEquals("Should read records from block 1 and 2", shouldHave, decoded.size());
+ for (int i = 0; i < block1Records.size(); i++) {
+ LogFileTestUtil.assertRecordEquals("Block 1 record " + i + " mismatch",
+ block1Records.get(i), decoded.get(i));
+ }
+ for (int i = 0; i < block2Records.size(); i++) {
+ LogFileTestUtil.assertRecordEquals("Block 2 record " + i + " mismatch",
+ block2Records.get(i), decoded.get(i + block1Records.size()));
+ }
+
+ assertEquals("Should have skipped 1 corrupt block (block 3)", 1,
+ readerContext.getCorruptBlocksSkipped());
+ assertEquals("Blocks read count should be 2 (blocks 1 and 2)", 2,
+ readerContext.getBlocksRead());
+ assertNotNull("Trailer should still be readable", reader.getTrailer());
+ }
+
+ private int findBlockEndOffset(byte[] data, int startOffset) throws IOException {
+ ByteArrayInputStream bais = new ByteArrayInputStream(data);
+ DataInputStream dis = new DataInputStream(bais);
+ // Skip to where the block header should start
+ dis.skipBytes(startOffset);
+ LOG.info("Reading header at " + startOffset);
+ LogBlockHeader header = new LogBlockHeader();
+ header.readFields(dis);
+ int payloadSize = header.getCompressedDataSize();
+ int endOffset = startOffset + header.getSerializedHeaderLength() + payloadSize
+ + Bytes.SIZEOF_LONG;
+ LOG.info("Block ending offset is " + endOffset);
+ return endOffset;
+ }
+
+ private List<LogFile.Record> readRecords(LogFileFormatReader reader) throws IOException {
+ List<LogFile.Record> decoded = new ArrayList<>();
+ LogFile.Record r;
+ while ((r = reader.next()) != null) {
+ decoded.add(r);
+ }
+ return decoded;
+ }
+
+ private List<LogFile.Record> writeBlock(LogFileFormatWriter writer, String table,
+ long startCommitId, int numRecords) throws IOException {
+ List<LogFile.Record> records = new ArrayList<>();
+ for (int i = 0; i < numRecords; i++) {
+ LogFile.Record r = LogFileTestUtil.newPutRecord(table, startCommitId + i,
+ "row" + (startCommitId + i), 10L + i, 1);
+ records.add(r);
+ writer.append(r);
+ }
+ writer.closeBlock();
+ LOG.info("Next block start position is " + writer.getPosition());
+ writer.startBlock();
+ return records;
+ }
+
+ private void initLogFileReader(byte[] data) throws IOException {
+ readerContext.setFileSize(data.length);
+ reader.init(readerContext, new LogFileTestUtil.SeekableByteArrayInputStream(data));
+ }
+
+ private void initLogFileWriter() throws IOException {
+ HDFSDataOutput output = new HDFSDataOutput(new FSDataOutputStream(writerDos, null) {
+ @Override
+ public long getPos() {
+ return writerDos.size();
+ }
+ @Override
+ public void hflush() throws IOException {
+ writerDos.flush();
+ }
+ @Override
+ public void hsync() throws IOException {
+ writerDos.flush();
+ }
+ });
+ writer.init(writerContext, output);
+ }
+
+}
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/replication/log/LogFileTestUtil.java b/phoenix-core/src/test/java/org/apache/phoenix/replication/log/LogFileTestUtil.java
new file mode 100644
index 0000000000..842917a088
--- /dev/null
+++ b/phoenix-core/src/test/java/org/apache/phoenix/replication/log/LogFileTestUtil.java
@@ -0,0 +1,434 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.replication.log;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.EOFException;
+import java.io.IOException;
+import java.io.OutputStream;
+
+import org.apache.hadoop.hbase.Cell;
+import org.apache.hadoop.hbase.CellBuilderFactory;
+import org.apache.hadoop.hbase.CellBuilderType;
+import org.apache.hadoop.hbase.client.Delete;
+import org.apache.hadoop.hbase.client.Mutation;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.util.Bytes;
+
+public interface LogFileTestUtil {
+
+ static LogFile.Record newPutRecord(String table, long commitId, String rowKey, long ts,
+ int numCols) {
+ return new LogFileRecord().setMutation(newPut(rowKey, ts, numCols))
+ .setHBaseTableName(table).setCommitId(commitId);
+ }
+
+ static Put newPut(String rowKey, long ts, int numCols) {
+ byte[] qualifier = Bytes.toBytes("q");
+ Put put = new Put(Bytes.toBytes(rowKey));
+ put.setTimestamp(ts);
+ for (int i = 0; i < numCols; i++) {
+ put.addColumn(Bytes.toBytes("col" + i), qualifier, ts,
+ Bytes.toBytes("v" + i + "_" + rowKey));
+ }
+ return put;
+ }
+
+ static LogFile.Record newDeleteRecord(String table, long commitId, String rowKey, long ts,
+ int numCols) {
+ return new LogFileRecord().setMutation(newDelete(rowKey, ts, numCols))
+ .setHBaseTableName(table).setCommitId(commitId);
+ }
+
+ static Delete newDelete(String rowKey, long ts, int numCols) {
+ byte[] qualifier = Bytes.toBytes("q");
+ Delete delete = new Delete(Bytes.toBytes(rowKey));
+ delete.setTimestamp(ts);
+ for (int i = 0; i < numCols; i++) {
+ delete.addColumn(Bytes.toBytes("col" + i), qualifier);
+ }
+ return delete;
+ }
+
+ static LogFile.Record newDeleteColumnRecord(String table, long commitId, String rowKey,
+ long ts, int numCols) throws IOException {
+ byte[] qualifier = Bytes.toBytes("q");
+ Delete delete = new Delete(Bytes.toBytes(rowKey));
+ delete.setTimestamp(ts);
+ for (int i = 0; i < numCols; i++) {
+ byte[] column = Bytes.toBytes("col" + i);
+ delete.add(CellBuilderFactory.create(CellBuilderType.SHALLOW_COPY)
+ .setRow(Bytes.toBytes(rowKey)).setFamily(column).setQualifier(qualifier)
+ .setTimestamp(ts).setType(Cell.Type.DeleteColumn).build());
+ }
+ LogFile.Record record = new LogFileRecord().setMutation(delete).setHBaseTableName(table)
+ .setCommitId(commitId);
+ return record;
+ }
+
+ static LogFile.Record newDeleteFamilyRecord(String table, long commitId, String rowKey,
+ long ts, int numCols) throws IOException {
+ Delete delete = new Delete(Bytes.toBytes(rowKey));
+ delete.setTimestamp(ts);
+ for (int i = 0; i < numCols; i++) {
+ byte[] column = Bytes.toBytes("col" + i);
+ delete.addFamily(column);
+ }
+ LogFile.Record record = new LogFileRecord().setMutation(delete).setHBaseTableName(table)
+ .setCommitId(commitId);
+ return record;
+ }
+
+ static LogFile.Record newDeleteFamilyVersionRecord(String table, long commitId, String rowKey,
+ long ts, int numCols) throws IOException {
+ Delete delete = new Delete(Bytes.toBytes(rowKey));
+ delete.setTimestamp(ts);
+ for (int i = 0; i < numCols; i++) {
+ byte[] column = Bytes.toBytes("col" + i);
+ delete.addFamilyVersion(column, ts);
+ }
+ LogFile.Record record = new LogFileRecord().setMutation(delete).setHBaseTableName(table)
+ .setCommitId(commitId);
+ return record;
+ }
+
+ static void assertRecordEquals(String message, LogFile.Record r1, LogFile.Record r2)
+ throws AssertionError {
+ try {
+ if (!r1.getMutation().toJSON().equals(r2.getMutation().toJSON())
+ || !r1.getHBaseTableName().equals(r2.getHBaseTableName())
+ || r1.getCommitId() != r2.getCommitId()) {
+ throw new AssertionError(message + ": left=" + r1 + ", right=" + r2);
+ }
+ } catch (IOException e) {
+ throw new AssertionError(e.getMessage());
+ }
+ }
+
+ static void assertMutationEquals(String message, Mutation m1, Mutation m2) {
+ try {
+ if (!m1.toJSON().equals(m2.toJSON())) {
+ throw new AssertionError(message + ": left=" + m1 + ", right=" + m2);
+ }
+ } catch (IOException e) {
+ throw new AssertionError(e.getMessage());
+ }
+ }
+
+ static class SeekableByteArrayInputStream extends ByteArrayInputStream
+ implements SeekableDataInput {
+
+ // A view of ourselves as a data input stream
+ private DataInputStream stream;
+
+ public SeekableByteArrayInputStream(byte[] buf) {
+ super(buf);
+ this.stream = new DataInputStream(this);
+ }
+
+ public SeekableByteArrayInputStream(byte[] buf, int offset, int length) {
+ super(buf, offset, length);
+ this.stream = new DataInputStream(this);
+ }
+
+ @Override
+ public void seek(long pos) throws IOException {
+ if (pos < 0) {
+ throw new IOException("Cannot seek to negative position");
+ }
+ if (pos > Integer.MAX_VALUE) {
+ throw new IllegalArgumentException("Cannot seek beyond Integer.MAX_VALUE");
+ }
+ if (pos >= count) {
+ this.pos = count; // Position at the end
+ if (pos > count) {
+ throw new EOFException("Seek position is past the end of the stream");
+ }
+ } else {
+ this.pos = (int) pos;
+ }
+ }
+
+ @Override
+ public long getPos() throws IOException {
+ return pos;
+ }
+
+ @Override
+ public boolean seekToNewSource(long pos) throws IOException {
+ seek(pos);
+ return false;
+ }
+
+ @Override
+ public int read(long position, byte[] buffer, int offset, int length) throws IOException {
+ if (buffer == null) {
+ throw new NullPointerException();
+ }
+ if (offset < 0 || length < 0 || offset + length > buffer.length) {
+ throw new IndexOutOfBoundsException(String.format(
+ "Offset %d or length %d is invalid for buffer of size %d", offset, length,
+ buffer.length));
+ }
+ if (position < 0) {
+ throw new IOException("Position cannot be negative");
+ }
+ if (position > Integer.MAX_VALUE) {
+ throw new IllegalArgumentException("Position cannot exceed Integer.MAX_VALUE");
+ }
+ if (length == 0) {
+ return 0;
+ }
+ int pos = (int) position;
+ if (pos >= count) {
+ return -1;
+ }
+ int avail = count - pos;
+ int n = Math.min(length, avail);
+ System.arraycopy(buf, pos, buffer, offset, n);
+ return n;
+ }
+
+ @Override
+ public void readFully(long position, byte[] buffer, int offset, int length)
+ throws IOException {
+ if (buffer == null) {
+ throw new NullPointerException("Buffer cannot be null");
+ }
+ if (offset < 0 || length < 0 || offset + length > buffer.length) {
+ throw new IndexOutOfBoundsException(String.format(
+ "Offset %d or length %d is invalid for buffer of size %d", offset, length,
+ buffer.length));
+ }
+ if (position < 0) {
+ throw new IOException("Position cannot be negative");
+ }
+ if (position > Integer.MAX_VALUE) {
+ throw new IllegalArgumentException("Position cannot exceed Integer.MAX_VALUE");
+ }
+ if (length == 0) {
+ return;
+ }
+ int pos = (int) position;
+ if (pos >= count) {
+ throw new EOFException("Read position is at or past the end of the stream");
+ }
+ int avail = count - pos;
+ if (length > avail) {
+ throw new EOFException(String.format(
+ "Premature EOF from stream: expected %d bytes, but only %d available", length,
+ avail));
+ }
+ System.arraycopy(buf, pos, buffer, offset, length);
+ }
+
+ @Override
+ public void readFully(long position, byte[] buffer) throws IOException {
+ readFully(position, buffer, 0, buffer.length);
+ }
+
+ @Override
+ public void readFully(byte[] b) throws IOException {
+ stream.readFully(b);
+ }
+
+ @Override
+ public void readFully(byte[] b, int off, int len) throws IOException {
+ stream.readFully(b, off, len);
+ }
+
+ @Override
+ public int skipBytes(int n) throws IOException {
+ return stream.skipBytes(n);
+ }
+
+ @Override
+ public boolean readBoolean() throws IOException {
+ return stream.readBoolean();
+ }
+
+ @Override
+ public byte readByte() throws IOException {
+ return stream.readByte();
+ }
+
+ @Override
+ public int readUnsignedByte() throws IOException {
+ return stream.readUnsignedByte();
+ }
+
+ @Override
+ public short readShort() throws IOException {
+ return stream.readShort();
+ }
+
+ @Override
+ public int readUnsignedShort() throws IOException {
+ return stream.readUnsignedShort();
+ }
+
+ @Override
+ public char readChar() throws IOException {
+ return stream.readChar();
+ }
+
+ @Override
+ public int readInt() throws IOException {
+ return stream.readInt();
+ }
+
+ @Override
+ public long readLong() throws IOException {
+ return stream.readLong();
+ }
+
+ @Override
+ public float readFloat() throws IOException {
+ return stream.readFloat();
+ }
+
+ @Override
+ public double readDouble() throws IOException {
+ return stream.readDouble();
+ }
+
+ @Override
+ @Deprecated
+ public String readLine() throws IOException {
+ return stream.readLine();
+ }
+
+ @Override
+ public String readUTF() throws IOException {
+ return stream.readUTF();
+ }
+
+ }
+
+ static class SyncableByteArrayOutputStream extends OutputStream
+ implements SyncableDataOutput {
+
+ private ByteArrayOutputStream out = new ByteArrayOutputStream();
+ // A view of ourselves as a data output stream
+ private DataOutputStream stream;
+
+ public SyncableByteArrayOutputStream() {
+ this.stream = new DataOutputStream(out);
+ }
+
+ @Override
+ public void writeBoolean(boolean v) throws IOException {
+ stream.writeBoolean(v);;
+ }
+
+ @Override
+ public void writeByte(int v) throws IOException {
+ stream.writeByte(v);
+ }
+
+ @Override
+ public void writeShort(int v) throws IOException {
+ stream.writeShort(v);
+ }
+
+ @Override
+ public void writeChar(int v) throws IOException {
+ stream.writeChar(v);
+ }
+
+ @Override
+ public void writeInt(int v) throws IOException {
+ stream.writeInt(v);
+ }
+
+ @Override
+ public void writeLong(long v) throws IOException {
+ stream.writeLong(v);
+ }
+
+ @Override
+ public void writeFloat(float v) throws IOException {
+ stream.writeFloat(v);
+ }
+
+ @Override
+ public void writeDouble(double v) throws IOException {
+ stream.writeDouble(v);
+ }
+
+ @Override
+ public void writeBytes(String s) throws IOException {
+ stream.writeBytes(s);
+ }
+
+ @Override
+ public void writeChars(String s) throws IOException {
+ stream.writeChars(s);
+ }
+
+ @Override
+ public void writeUTF(String s) throws IOException {
+ stream.writeUTF(s);
+ }
+
+ @Override
+ public long getPos() throws IOException {
+ return out.size();
+ }
+
+ @Override
+ public void write(int b) throws IOException {
+ out.write(b);
+ }
+
+ @Override
+ public void write(byte[] b) throws IOException {
+ out.write(b);
+ }
+
+ @Override
+ public void write(byte[] b, int off, int len) throws IOException {
+ out.write(b, off, len);
+ }
+
+ @Override
+ public void close() throws IOException {
+
+ }
+
+ @Override
+ public void hflush() throws IOException {
+
+ }
+
+ @Override
+ public void hsync() throws IOException {
+
+ }
+
+ @Override
+ public void sync() throws IOException {
+
+ }
+
+ }
+
+}
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/replication/log/LogFileWriterSyncTest.java b/phoenix-core/src/test/java/org/apache/phoenix/replication/log/LogFileWriterSyncTest.java
new file mode 100644
index 0000000000..eeb5d05f60
--- /dev/null
+++ b/phoenix-core/src/test/java/org/apache/phoenix/replication/log/LogFileWriterSyncTest.java
@@ -0,0 +1,194 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.replication.log;
+
+import static org.mockito.ArgumentMatchers.any;
+import static org.mockito.ArgumentMatchers.anyBoolean;
+import static org.mockito.Mockito.inOrder;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import java.io.IOException;
+import java.io.OutputStream;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.client.Mutation;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.InOrder;
+
+public class LogFileWriterSyncTest {
+
+ private Configuration conf;
+ private FileSystem mockFs;
+ private SyncableDataOutput internalOutput;
+ private LogFileWriterContext writerContext;
+ private LogFileWriter writer;
+
+ @Before
+ public void setUp() throws IOException {
+ conf = HBaseConfiguration.create();
+
+ // This is structured so we verify that FSDataOutputStream correctly calls hsync() on its
+ // understream, which in this case will be our SyncableByteArrayOutputStream.
+ // The sync flow is: LogFileWriter -sync-> LogFileFormatWriter -sync-> HDFSDataOutput
+ // -hsync-> FSDataOutputStream -hsync-> internalOutput
+ internalOutput = spy(new LogFileTestUtil.SyncableByteArrayOutputStream());
+ mockFs = mock(FileSystem.class);
+ when(mockFs.create(any(), anyBoolean()))
+ .thenReturn(new FSDataOutputStream((OutputStream) internalOutput,
+ new FileSystem.Statistics("hdfs"), 0));
+ when(internalOutput.getPos()).thenReturn(100L);
+
+ // Create context using mocks
+ writerContext = new LogFileWriterContext(conf).setFileSystem(mockFs);
+
+ // Create the writer instance to be tested
+ writer = new LogFileWriter();
+
+ // Initialize the writer - this will call fs.create() and set up internal writers
+ writer.init(writerContext);
+ }
+
+ @After
+ public void tearDown() throws IOException {
+ if (writer != null) {
+ writer.close();
+ }
+ }
+
+ @Test
+ public void testSyncAfterAppend() throws IOException {
+ InOrder inOrder = inOrder(internalOutput);
+
+ // Append data
+ Mutation m1 = LogFileTestUtil.newPut("row1", 1L, 1);
+ writer.append("TBL", 1L, m1);
+ Mutation m2 = LogFileTestUtil.newPut("row2", 2L, 1);
+ writer.append("TBL", 2L, m2);
+
+ // Sync the writer
+ writer.sync();
+
+ // Verify hsync was called *after* the data related to appends would have been written
+ // We know sync() forces a block close and hsync on the underlying stream.
+ inOrder.verify(internalOutput, times(1)).hsync();
+
+ // Append more data after sync
+ Mutation m3 = LogFileTestUtil.newPut("row3", 12L, 1);
+ writer.append("TBL", 3L, m3);
+
+ // Sync again
+ writer.sync();
+
+ // Verify hsync was called again
+ inOrder.verify(internalOutput, times(1)).hsync();
+ }
+
+ @Test
+ public void testAppendAfterSync() throws IOException {
+ InOrder inOrder = inOrder(internalOutput);
+
+ // Append A, then sync
+ Mutation m1 = LogFileTestUtil.newPut("row1", 1L, 1);
+ writer.append("TBL", 1L, m1);
+ writer.sync();
+
+ // Verify first hsync
+ inOrder.verify(internalOutput, times(1)).hsync();
+
+ // Append B after sync
+ Mutation m2 = LogFileTestUtil.newPut("row2", 2L, 1);
+ writer.append("TBL", 2L, m2);
+
+ // Verify hsync was NOT called immediately after appending B. It might be called later on
+ // close or another sync.
+ inOrder.verify(internalOutput, never()).hsync();
+
+ // Now close (which includes a sync)
+ writer.close();
+
+ // Verify the final hsync happened during close
+ inOrder.verify(internalOutput, times(1)).hsync();
+ }
+
+ @Test
+ public void testMultipleSyncs() throws IOException {
+ InOrder inOrder = inOrder(internalOutput);
+
+ // Append A
+ Mutation m1 = LogFileTestUtil.newPut("row1", 1L, 1);
+ writer.append("TBL", 1L, m1);
+
+ // Sync 1
+ writer.sync();
+ inOrder.verify(internalOutput, times(1)).hsync();
+
+ // Sync 2 (no data appended in between)
+ writer.sync();
+ // The current implementation should do nothing if the block is empty
+ inOrder.verify(internalOutput, never()).hsync();
+
+ // Append B
+ Mutation m2 = LogFileTestUtil.newPut("row2", 2L, 1);
+ writer.append("TBL", 2L, m2);
+
+ // Sync 3
+ writer.sync();
+ inOrder.verify(internalOutput, times(1)).hsync();
+
+ // Now close (which includes a sync)
+ writer.close();
+
+ // A total of three hsyncs
+ verify(internalOutput, times(3)).hsync();
+ }
+
+ @Test
+ public void testSyncEmpty() throws IOException {
+ InOrder inOrder = inOrder(internalOutput);
+
+ // Sync immediately after init
+ writer.sync();
+
+ // The current implementation should do nothing if the block is empty
+ inOrder.verify(internalOutput, never()).hsync();
+
+ // Append a record
+ Mutation m1 = LogFileTestUtil.newPut("row", 1L, 1);
+ writer.append("TBL", 1L, m1);
+
+ // Sync again
+ writer.sync();
+
+ // Verify the second hsync
+ inOrder.verify(internalOutput, times(1)).hsync();
+
+ // A total of one hsync
+ verify(internalOutput, times(1)).hsync();
+ }
+
+}
diff --git a/phoenix-core/src/test/java/org/apache/phoenix/replication/log/LogFileWriterTest.java b/phoenix-core/src/test/java/org/apache/phoenix/replication/log/LogFileWriterTest.java
new file mode 100644
index 0000000000..55c4737223
--- /dev/null
+++ b/phoenix-core/src/test/java/org/apache/phoenix/replication/log/LogFileWriterTest.java
@@ -0,0 +1,152 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.phoenix.replication.log;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.junit.After;
+import org.junit.Before;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.TemporaryFolder;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+public class LogFileWriterTest {
+
+ private static final Logger LOG = LoggerFactory.getLogger(LogFileWriterTest.class);
+
+ @Rule
+ public TemporaryFolder testFolder = new TemporaryFolder();
+
+ private Configuration conf;
+ private FileSystem localFs;
+ private Path filePath;
+ private LogFileReaderContext readerContext;
+ private LogFileReader reader;
+ private LogFileWriterContext writerContext;
+ private LogFileWriter writer;
+
+ @Before
+ public void setUp() throws IOException {
+ conf = HBaseConfiguration.create();
+ localFs = FileSystem.getLocal(conf);
+ filePath = new Path(testFolder.newFile("LogFileWriterTest").toURI());
+ reader = new LogFileReader();
+ writer = new LogFileWriter();
+ }
+
+ @After
+ public void tearDown() throws IOException {
+ writer.close();
+ reader.close();
+ }
+
+ @Test
+ public void testLogFileWriter() throws IOException {
+ initLogFileWriter();
+ LogFile.Record r1 = LogFileTestUtil.newPutRecord("TBLTLFW", 100, "row1", 10L, 1);
+ LogFile.Record r2 = LogFileTestUtil.newDeleteRecord("TBLTLFW", 101, "row2", 11L, 1);
+ writer.append(r1.getHBaseTableName(), r1.getCommitId(), r1.getMutation());
+ LOG.debug("Appended " + r1);
+ writer.sync();
+ writer.append(r2.getHBaseTableName(), r2.getCommitId(), r2.getMutation());
+ LOG.debug("Appended " + r2);
+ writer.close();
+
+ assertTrue("File should exist", localFs.exists(filePath));
+ assertTrue("File length should be > 0", writer.getLength() > 0);
+
+ initLogFileReader();
+ assertEquals("Header major version mismatch", LogFileHeader.VERSION_MAJOR,
+ reader.getHeader().getMajorVersion());
+
+ LogFile.Record decoded1 = reader.next();
+ LOG.debug("Read " + decoded1);
+ LogFileTestUtil.assertRecordEquals("First record mismatch", r1, decoded1);
+
+ LogFile.Record decoded2 = reader.next();
+ LOG.debug("Read " + decoded2);
+ LogFileTestUtil.assertRecordEquals("Second record mismatch", r2, decoded2);
+
+ assertNull("Should be end of file", reader.next());
+ assertNotNull("Trailer should exist", reader.getTrailer());
+ reader.close();
+ }
+
+ @Test
+ public void testLogFileReaderIterator() throws IOException {
+ initLogFileWriter();
+ List<LogFile.Record> originals = new ArrayList<>();
+ for (int i = 0; i < 1000; i++) {
+ LogFile.Record record = LogFileTestUtil.newPutRecord("TBLFRI", 100L + i, "row" + i,
+ 10L + i, 1);
+ originals.add(record);
+ writer.append(record.getHBaseTableName(), record.getCommitId(), record.getMutation());
+ }
+ writer.close();
+
+ initLogFileReader();
+ List<LogFile.Record> decoded = new ArrayList<>();
+ Iterator<LogFile.Record> iterator = reader.iterator();
+ while (iterator.hasNext()) {
+ LogFile.Record r = iterator.next();
+ decoded.add(r);
+ }
+
+ assertEquals("Iterator count mismatch", originals.size(), decoded.size());
+ for (int i = 0; i < originals.size(); i++) {
+ LogFileTestUtil.assertRecordEquals("Iterator record " + i + " mismatch",
+ originals.get(i), decoded.get(i));
+ }
+ reader.close();
+ }
+
+ @Test
+ public void testLogFileReaderEmptyFile() throws IOException {
+ initLogFileWriter();
+ writer.close(); // Creates an empty file (header + trailer)
+ initLogFileReader();
+ assertNull("Next should return null for empty file", reader.next());
+ reader.close();
+ }
+
+ private void initLogFileReader() throws IOException {
+ readerContext = new LogFileReaderContext(conf).setFileSystem(localFs)
+ .setFilePath(filePath);
+ reader.init(readerContext);
+ }
+
+ private void initLogFileWriter() throws IOException {
+ writerContext = new LogFileWriterContext(conf).setFileSystem(localFs)
+ .setFilePath(filePath);
+ writer.init(writerContext);
+ }
+
+}
diff --git a/pom.xml b/pom.xml
index f637d815a8..0489bed424 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1443,6 +1443,11 @@
<artifactId>hbase-endpoint</artifactId>
<version>${hbase.version}</version>
</dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-asyncfs</artifactId>
+ <version>${hbase.version}</version>
+ </dependency>
<dependency>
<groupId>org.apache.hbase</groupId>
<artifactId>hbase-server</artifactId>
@@ -1572,6 +1577,12 @@
</exclusion>
</exclusions>
</dependency>
+ <dependency>
+ <groupId>org.apache.hbase</groupId>
+ <artifactId>hbase-compression-aircompressor</artifactId>
+ <version>${hbase.version}</version>
+ <scope>test</scope>
+ </dependency>
<!-- Transaction dependencies -->
<!-- Omid -->