You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ns...@apache.org on 2011/10/11 04:19:30 UTC
svn commit: r1181555 [1/4] - in /hbase/branches/0.89/src:
main/java/org/apache/hadoop/hbase/io/hfile/
test/java/org/apache/hadoop/hbase/io/hfile/
Author: nspiegelberg
Date: Tue Oct 11 02:19:30 2011
New Revision: 1181555
URL: http://svn.apache.org/viewvc?rev=1181555&view=rev
Log:
HFile format version 2: HFile reader/writer hierarchy, and an updated fixed file trailer
Summary:
With the introduction of HFile format version 2, we still want to support
reading the old format for the purpose of automatic migration. To better ensure
backwards-compatibility, we keep the version 1 writer as well, and utilize
existing unit tests to ensure our version 1 reader/writer implementations are
consistent. The version 1 reader/writer are updated to significantly reuse
version 2's functionality, and the block cache always contains version 2 blocks.
As a result, we get a natural class hierarchy of HFile readers and writers for
the two versions, and break the original monolithic HFile class into a class
hierarchy. HFile block index has been moved out of HFile into its own class, which will be
submitted separately. The new FixedFileTrailer implementation supports both v1
and v2 trailers, and is aware of the fields that are valid in each version.
Test Plan:
Unit tests. Load testing using HBaseTest. New backwards-compatibility unit
tests will be added. Will test in dark launch.
Reviewed By: nspiegelberg
Reviewers: kannan, liyintang, kranganathan, gqchen, aaiyer, nspiegelberg, jgray
Commenters: jgray, kannan
CC: hbase@lists, , mbautin, jgray, kenny, kannan, nspiegelberg
Revert Plan:
The HFile format v2 is a backwards-compatible, but non-reverse-compatible
feature, and has to be tested thoroughly even before being deployed to dark
launch.
Differential Revision: 251875
Added:
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/AbstractHFileReader.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/AbstractHFileWriter.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/FixedFileTrailer.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/HFilePrettyPrinter.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV1.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileReaderV2.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV1.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/HFileWriterV2.java
hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/io/hfile/TestFixedFileTrailer.java
hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/io/hfile/TestHFileWriterV2.java
Modified:
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/HFile.java
Added: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/AbstractHFileReader.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/AbstractHFileReader.java?rev=1181555&view=auto
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/AbstractHFileReader.java (added)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/AbstractHFileReader.java Tue Oct 11 02:19:30 2011
@@ -0,0 +1,364 @@
+/*
+ * Copyright 2011 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.io.hfile;
+
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.io.hfile.HFile.FileInfo;
+import org.apache.hadoop.hbase.io.hfile.HFile.Reader;
+import org.apache.hadoop.io.RawComparator;
+
+/**
+ * Common functionality needed by all versions of {@link HFile} readers.
+ */
+public abstract class AbstractHFileReader implements HFile.Reader {
+
+ private static final Log LOG = LogFactory.getLog(AbstractHFileReader.class);
+
+ /** Filesystem-level block reader for this HFile format version. */
+ protected HFileBlock.FSReader fsBlockReader;
+
+ /** Stream to read from. */
+ protected FSDataInputStream istream;
+
+ /**
+ * True if we should close the input stream when done. We don't close it if we
+ * didn't open it.
+ */
+ protected final boolean closeIStream;
+
+ /** Data block index */
+ protected HFileBlockIndex.BlockIndexReader dataBlockIndexReader;
+
+ /** Meta block index */
+ protected HFileBlockIndex.BlockIndexReader metaBlockIndexReader;
+
+ protected final FixedFileTrailer trailer;
+
+ /** Filled when we read in the trailer. */
+ protected final Compression.Algorithm compressAlgo;
+
+ /** Last key in the file. Filled in when we read in the file info */
+ protected byte [] lastKey = null;
+
+ /** Average key length read from file info */
+ protected int avgKeyLen = -1;
+
+ /** Average value length read from file info */
+ protected int avgValueLen = -1;
+
+ /** Key comparator */
+ protected RawComparator<byte []> comparator;
+
+ /** Size of this file. */
+ protected final long fileSize;
+
+ /** Block cache to use. */
+ protected final BlockCache blockCache;
+
+ public int cacheHits = 0;
+ public int blockLoads = 0;
+ public int metaLoads = 0;
+
+ /** Whether file is from in-memory store */
+ protected boolean inMemory = false;
+
+ /** Whether blocks of file should be evicted on close of file */
+ protected final boolean evictOnClose;
+
+ /** Path of file */
+ protected final Path path;
+
+ /** File name to be used for block names */
+ protected final String name;
+
+ protected FileInfo fileInfo;
+
+ // table qualified cfName for this HFile.
+ // This is used to report stats on a per-table/CF basis
+ public String cfName = "";
+
+ // various metrics that we want to track on a per-cf basis
+ public String fsReadTimeMetric = "";
+ public String compactionReadTimeMetric = "";
+
+ public String fsBlockReadCntMetric = "";
+ public String compactionBlockReadCntMetric = "";
+
+ public String fsBlockReadCacheHitCntMetric = "";
+ public String compactionBlockReadCacheHitCntMetric = "";
+
+ public String fsMetaBlockReadCntMetric = "";
+ public String fsMetaBlockReadCacheHitCntMetric = "";
+
+ protected AbstractHFileReader(Path path, FixedFileTrailer trailer,
+ final FSDataInputStream fsdis, final long fileSize,
+ final boolean closeIStream,
+ final BlockCache blockCache, final boolean inMemory,
+ final boolean evictOnClose) {
+ this.trailer = trailer;
+ this.compressAlgo = trailer.getCompressionCodec();
+ this.blockCache = blockCache;
+ this.fileSize = fileSize;
+ this.istream = fsdis;
+ this.closeIStream = closeIStream;
+ this.inMemory = inMemory;
+ this.evictOnClose = evictOnClose;
+ this.path = path;
+ this.name = path.getName();
+ parsePath(path.toString());
+ }
+
+ @SuppressWarnings("serial")
+ public static class BlockIndexNotLoadedException extends RuntimeException {
+ public BlockIndexNotLoadedException() {
+ super("Block index not loaded");
+ }
+ }
+
+ protected String toStringFirstKey() {
+ return KeyValue.keyToString(getFirstKey());
+ }
+
+ protected String toStringLastKey() {
+ return KeyValue.keyToString(getLastKey());
+ }
+
+ /**
+ * Parse the HFile path to figure out which table and column family
+ * it belongs to. This is used to maintain read statistics on a
+ * per-column-family basis.
+ *
+ * @param path HFile path name
+ */
+ public void parsePath(String path) {
+ String splits[] = path.split("/");
+ if (splits.length < 2) {
+ LOG.warn("Could not determine the table and column family of the " +
+ "HFile path " + path);
+ return;
+ }
+
+ cfName = "cf." + splits[splits.length - 2];
+
+ fsReadTimeMetric = cfName + ".fsRead";
+ compactionReadTimeMetric = cfName + ".compactionRead";
+
+ fsBlockReadCntMetric = cfName + ".fsBlockReadCnt";
+ fsBlockReadCacheHitCntMetric = cfName + ".fsBlockReadCacheHitCnt";
+
+ compactionBlockReadCntMetric = cfName + ".compactionBlockReadCnt";
+ compactionBlockReadCacheHitCntMetric = cfName
+ + ".compactionBlockReadCacheHitCnt";
+
+ fsMetaBlockReadCntMetric = cfName + ".fsMetaBlockReadCnt";
+ fsMetaBlockReadCacheHitCntMetric = cfName + ".fsMetaBlockReadCacheHitCnt";
+ }
+
+ public abstract boolean isFileInfoLoaded();
+
+ @Override
+ public String toString() {
+ return "reader=" + path.toString() +
+ (!isFileInfoLoaded()? "":
+ ", compression=" + compressAlgo.getName() +
+ ", inMemory=" + inMemory +
+ ", firstKey=" + toStringFirstKey() +
+ ", lastKey=" + toStringLastKey()) +
+ ", avgKeyLen=" + avgKeyLen +
+ ", avgValueLen=" + avgValueLen +
+ ", entries=" + trailer.getEntryCount() +
+ ", length=" + fileSize;
+ }
+
+ @Override
+ public long length() {
+ return fileSize;
+ }
+
+ /**
+ * Create a Scanner on this file. No seeks or reads are done on creation. Call
+ * {@link HFileScanner#seekTo(byte[])} to position an start the read. There is
+ * nothing to clean up in a Scanner. Letting go of your references to the
+ * scanner is sufficient. NOTE: Do not use this overload of getScanner for
+ * compactions.
+ *
+ * @param cacheBlocks True if we should cache blocks read in by this scanner.
+ * @param pread Use positional read rather than seek+read if true (pread is
+ * better for random reads, seek+read is better scanning).
+ * @return Scanner on this file.
+ */
+ @Override
+ public HFileScanner getScanner(boolean cacheBlocks, final boolean pread) {
+ return getScanner(cacheBlocks, pread, false);
+ }
+
+ /**
+ * @return the first key in the file. May be null if file has no entries. Note
+ * that this is not the first row key, but rather the byte form of the
+ * first KeyValue.
+ */
+ @Override
+ public byte [] getFirstKey() {
+ if (dataBlockIndexReader == null) {
+ throw new BlockIndexNotLoadedException();
+ }
+ return dataBlockIndexReader.isEmpty() ? null
+ : dataBlockIndexReader.getRootBlockKey(0);
+ }
+
+ /**
+ * TODO left from {@HFile} version 1: move this to StoreFile after Ryan's
+ * patch goes in to eliminate {@link KeyValue} here.
+ *
+ * @return the first row key, or null if the file is empty.
+ */
+ @Override
+ public byte[] getFirstRowKey() {
+ byte[] firstKey = getFirstKey();
+ if (firstKey == null)
+ return null;
+ return KeyValue.createKeyValueFromKey(firstKey).getRow();
+ }
+
+ /**
+ * TODO left from {@HFile} version 1: move this to StoreFile after
+ * Ryan's patch goes in to eliminate {@link KeyValue} here.
+ *
+ * @return the last row key, or null if the file is empty.
+ */
+ @Override
+ public byte[] getLastRowKey() {
+ byte[] lastKey = getLastKey();
+ if (lastKey == null)
+ return null;
+ return KeyValue.createKeyValueFromKey(lastKey).getRow();
+ }
+
+ /** @return number of KV entries in this HFile */
+ @Override
+ public long getEntries() {
+ return trailer.getEntryCount();
+ }
+
+ /** @return comparator */
+ @Override
+ public RawComparator<byte []> getComparator() {
+ return comparator;
+ }
+
+ /** @return compression algorithm */
+ @Override
+ public Compression.Algorithm getCompressionAlgorithm() {
+ return compressAlgo;
+ }
+
+ /**
+ * @return the total heap size of data and meta block indexes in bytes. Does
+ * not take into account non-root blocks of a multilevel data index.
+ */
+ public long indexSize() {
+ return (dataBlockIndexReader != null ? dataBlockIndexReader.heapSize() : 0)
+ + ((metaBlockIndexReader != null) ? metaBlockIndexReader.heapSize()
+ : 0);
+ }
+
+ @Override
+ public String getName() {
+ return name;
+ }
+
+ @Override
+ public HFileBlockIndex.BlockIndexReader getDataBlockIndexReader() {
+ return dataBlockIndexReader;
+ }
+
+ @Override
+ public String getColumnFamilyName() {
+ return cfName;
+ }
+
+ @Override
+ public FixedFileTrailer getTrailer() {
+ return trailer;
+ }
+
+ @Override
+ public FileInfo loadFileInfo() throws IOException {
+ return fileInfo;
+ }
+
+ /**
+ * An exception thrown when an operation requiring a scanner to be seeked
+ * is invoked on a scanner that is not seeked.
+ */
+ @SuppressWarnings("serial")
+ public static class NotSeekedException extends IllegalStateException {
+ public NotSeekedException() {
+ super("Not seeked to a key/value");
+ }
+ }
+
+ protected static abstract class Scanner implements HFileScanner {
+ protected HFile.Reader reader;
+ protected ByteBuffer blockBuffer;
+
+ protected boolean cacheBlocks;
+ protected boolean pread;
+ protected boolean isCompaction;
+
+ protected int currKeyLen;
+ protected int currValueLen;
+
+ protected int blockFetches;
+
+ @Override
+ public Reader getReader() {
+ return reader;
+ }
+
+ @Override
+ public boolean isSeeked(){
+ return blockBuffer != null;
+ }
+
+ @Override
+ public String toString() {
+ return "HFileScanner for reader " + String.valueOf(reader);
+ }
+
+ protected void assertSeeked() {
+ if (!isSeeked())
+ throw new NotSeekedException();
+ }
+ }
+
+ /** For testing */
+ HFileBlock.FSReader getUncachedBlockReader() {
+ return fsBlockReader;
+ }
+
+}
Added: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/AbstractHFileWriter.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/AbstractHFileWriter.java?rev=1181555&view=auto
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/AbstractHFileWriter.java (added)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/AbstractHFileWriter.java Tue Oct 11 02:19:30 2011
@@ -0,0 +1,287 @@
+/*
+ * Copyright 2011 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.io.hfile;
+
+import java.io.DataOutput;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hbase.KeyValue.KeyComparator;
+import org.apache.hadoop.hbase.io.hfile.HFile.FileInfo;
+import org.apache.hadoop.hbase.regionserver.StoreFile;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.io.RawComparator;
+import org.apache.hadoop.io.Writable;
+
+/**
+ * Common functionality needed by all versions of {@link HFile} writers.
+ */
+public abstract class AbstractHFileWriter implements HFile.Writer {
+
+ /** Key previously appended. Becomes the last key in the file. */
+ protected byte[] lastKeyBuffer = null;
+
+ protected int lastKeyOffset = -1;
+ protected int lastKeyLength = -1;
+
+ /** FileSystem stream to write into. */
+ protected FSDataOutputStream outputStream;
+
+ /** True if we opened the <code>outputStream</code> (and so will close it). */
+ protected final boolean closeOutputStream;
+
+ /** A "file info" block: a key-value map of file-wide metadata. */
+ protected FileInfo fileInfo = new HFile.FileInfo();
+
+ /** Number of uncompressed bytes we allow per block. */
+ protected final int blockSize;
+
+ /** Total # of key/value entries, i.e. how many times add() was called. */
+ protected long entryCount = 0;
+
+ /** Used for calculating the average key length. */
+ protected long totalKeyLength = 0;
+
+ /** Used for calculating the average value length. */
+ protected long totalValueLength = 0;
+
+ /** Total uncompressed bytes, maybe calculate a compression ratio later. */
+ protected long totalUncompressedBytes = 0;
+
+ /** Key comparator. Used to ensure we write in order. */
+ protected final RawComparator<byte[]> comparator;
+
+ /** Meta block names. */
+ protected List<byte[]> metaNames = new ArrayList<byte[]>();
+
+ /** {@link Writable}s representing meta block data. */
+ protected List<Writable> metaData = new ArrayList<Writable>();
+
+ /** The compression algorithm used. NONE if no compression. */
+ protected final Compression.Algorithm compressAlgo;
+
+ /** First key in a block. */
+ protected byte[] firstKeyInBlock = null;
+
+ /** May be null if we were passed a stream. */
+ protected final Path path;
+
+ /** Whether to cache key/value data blocks on write */
+ protected final boolean cacheDataBlocksOnWrite;
+
+ /** Whether to cache non-root index blocks on write */
+ protected final boolean cacheIndexBlocksOnWrite;
+
+ /** Block cache to optionally fill on write. */
+ protected BlockCache blockCache;
+
+ /** Configuration used for block cache initialization */
+ private Configuration conf;
+
+ /**
+ * Name for this object used when logging or in toString. Is either
+ * the result of a toString on stream or else toString of passed file Path.
+ */
+ protected final String name;
+
+ public AbstractHFileWriter(Configuration conf,
+ FSDataOutputStream outputStream, Path path, int blockSize,
+ Compression.Algorithm compressAlgo, KeyComparator comparator) {
+ this.outputStream = outputStream;
+ this.path = path;
+ this.name = path != null ? path.getName() : outputStream.toString();
+ this.blockSize = blockSize;
+ this.compressAlgo = compressAlgo == null
+ ? HFile.DEFAULT_COMPRESSION_ALGORITHM : compressAlgo;
+ this.comparator = comparator != null ? comparator
+ : Bytes.BYTES_RAWCOMPARATOR;
+
+ closeOutputStream = path != null;
+
+ cacheDataBlocksOnWrite = conf.getBoolean(HFile.CACHE_BLOCKS_ON_WRITE_KEY,
+ false);
+ cacheIndexBlocksOnWrite = HFileBlockIndex.shouldCacheOnWrite(conf);
+
+ this.conf = conf;
+
+ if (cacheDataBlocksOnWrite || cacheIndexBlocksOnWrite)
+ initBlockCache();
+ }
+
+ /**
+ * Add last bits of metadata to file info before it is written out.
+ */
+ protected void finishFileInfo() throws IOException {
+ if (lastKeyBuffer != null) {
+ // Make a copy. The copy is stuffed into HMapWritable. Needs a clean
+ // byte buffer. Won't take a tuple.
+ fileInfo.append(FileInfo.LASTKEY, Arrays.copyOfRange(lastKeyBuffer,
+ lastKeyOffset, lastKeyOffset + lastKeyLength), false);
+ }
+
+ // Average key length.
+ int avgKeyLen =
+ entryCount == 0 ? 0 : (int) (totalKeyLength / entryCount);
+ fileInfo.append(FileInfo.AVG_KEY_LEN, Bytes.toBytes(avgKeyLen), false);
+
+ // Average value length.
+ int avgValueLen =
+ entryCount == 0 ? 0 : (int) (totalValueLength / entryCount);
+ fileInfo.append(FileInfo.AVG_VALUE_LEN, Bytes.toBytes(avgValueLen), false);
+ }
+
+ /**
+ * Add to the file info. All added key/value pairs can be obtained using
+ * {@link HFile.Reader#loadFileInfo()}.
+ *
+ * @param k Key
+ * @param v Value
+ * @throws IOException in case the key or the value are invalid
+ */
+ @Override
+ public void appendFileInfo(final byte[] k, final byte[] v)
+ throws IOException {
+ fileInfo.append(k, v, true);
+ }
+
+ /**
+ * Sets the file info offset in the trailer, finishes up populating fields in
+ * the file info, and writes the file info into the given data output. The
+ * reason the data output is not always {@link #outputStream} is that we store
+ * file info as a block in version 2.
+ *
+ * @param trailer fixed file trailer
+ * @param out the data output to write the file info to
+ * @throws IOException
+ */
+ protected final void writeFileInfo(FixedFileTrailer trailer, DataOutput out)
+ throws IOException {
+ trailer.setFileInfoOffset(outputStream.getPos());
+ finishFileInfo();
+ fileInfo.write(out);
+ }
+
+ /**
+ * Checks that the given key does not violate the key order.
+ *
+ * @param key Key to check.
+ * @return true if the key is duplicate
+ * @throws IOException if the key or the key order is wrong
+ */
+ protected boolean checkKey(final byte[] key, final int offset,
+ final int length) throws IOException {
+ boolean isDuplicateKey = false;
+
+ if (key == null || length <= 0) {
+ throw new IOException("Key cannot be null or empty");
+ }
+ if (length > HFile.MAXIMUM_KEY_LENGTH) {
+ throw new IOException("Key length " + length + " > "
+ + HFile.MAXIMUM_KEY_LENGTH);
+ }
+ if (lastKeyBuffer != null) {
+ int keyComp = comparator.compare(lastKeyBuffer, lastKeyOffset,
+ lastKeyLength, key, offset, length);
+ if (keyComp > 0) {
+ throw new IOException("Added a key not lexically larger than"
+ + " previous key="
+ + Bytes.toStringBinary(key, offset, length)
+ + ", lastkey="
+ + Bytes.toStringBinary(lastKeyBuffer, lastKeyOffset,
+ lastKeyLength));
+ } else if (keyComp == 0) {
+ isDuplicateKey = true;
+ }
+ }
+ return isDuplicateKey;
+ }
+
+ /** Checks the given value for validity. */
+ protected void checkValue(final byte[] value, final int offset,
+ final int length) throws IOException {
+ if (value == null) {
+ throw new IOException("Value cannot be null");
+ }
+ }
+
+ /**
+ * @return Path or null if we were passed a stream rather than a Path.
+ */
+ @Override
+ public Path getPath() {
+ return path;
+ }
+
+ @Override
+ public String toString() {
+ return "writer=" + (path != null ? path.toString() : null) + ", name="
+ + name + ", compression=" + compressAlgo.getName();
+ }
+
+ /**
+ * Sets remaining trailer fields, writes the trailer to disk, and optionally
+ * closes the output stream.
+ */
+ protected void finishClose(FixedFileTrailer trailer) throws IOException {
+ trailer.setMetaIndexCount(metaNames.size());
+ trailer.setTotalUncompressedBytes(totalUncompressedBytes);
+ trailer.setEntryCount(entryCount);
+ trailer.setCompressionCodec(compressAlgo);
+
+ trailer.serialize(outputStream);
+
+ if (closeOutputStream) {
+ outputStream.close();
+ outputStream = null;
+ }
+ }
+
+ public static Compression.Algorithm compressionByName(String algoName) {
+ if (algoName == null)
+ return HFile.DEFAULT_COMPRESSION_ALGORITHM;
+ return Compression.getCompressionAlgorithmByName(algoName);
+ }
+
+ /** A helper method to create HFile output streams in constructors */
+ protected static FSDataOutputStream createOutputStream(Configuration conf,
+ FileSystem fs, Path path, int bytesPerChecksum) throws IOException {
+ return fs.create(path, FsPermission.getDefault(), true,
+ fs.getConf().getInt("io.file.buffer.size", 4096),
+ fs.getDefaultReplication(), fs.getDefaultBlockSize(), bytesPerChecksum,
+ null);
+ }
+
+ /** Initializes the block cache to use for cache-on-write */
+ protected void initBlockCache() {
+ if (blockCache == null) {
+ blockCache = StoreFile.getBlockCache(conf);
+ conf = null; // This is all we need configuration for.
+ }
+ }
+
+}
Added: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/FixedFileTrailer.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/FixedFileTrailer.java?rev=1181555&view=auto
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/FixedFileTrailer.java (added)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/io/hfile/FixedFileTrailer.java Tue Oct 11 02:19:30 2011
@@ -0,0 +1,479 @@
+/*
+ * Copyright 2011 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.io.hfile;
+
+import java.io.ByteArrayInputStream;
+import java.io.ByteArrayOutputStream;
+import java.io.DataInputStream;
+import java.io.DataOutput;
+import java.io.DataOutputStream;
+import java.io.IOException;
+import java.nio.ByteBuffer;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.io.RawComparator;
+
+import static org.apache.hadoop.hbase.io.hfile.HFile.MIN_FORMAT_VERSION;
+import static org.apache.hadoop.hbase.io.hfile.HFile.MAX_FORMAT_VERSION;
+
+import com.google.common.io.NullOutputStream;
+
+/**
+ * The {@link HFile} has a fixed trailer which contains offsets to other
+ * variable parts of the file. Also includes basic metadata on this file. The
+ * trailer size is fixed within a given {@link HFile} format version only, but
+ * we always store the version number as the last four-byte integer of the file.
+ */
+public class FixedFileTrailer {
+
+ private static final Log LOG = LogFactory.getLog(FixedFileTrailer.class);
+
+ /**
+ * We store the comparator class name as a fixed-length field in the trailer.
+ */
+ private static final int MAX_COMPARATOR_NAME_LENGTH = 128;
+
+ /**
+ * Offset to the fileinfo data, a small block of vitals. Necessary in v1 but
+ * only potentially useful for pretty-printing in v2.
+ */
+ private long fileInfoOffset;
+
+ /**
+ * In version 1, the offset to the data block index. Starting from version 2,
+ * the meaning of this field is the offset to the section of the file that
+ * should be loaded at the time the file is being opened, and as of the time
+ * of writing, this happens to be the offset of the file info section.
+ */
+ private long loadOnOpenDataOffset;
+
+ /** The number of entries in the root data index. */
+ private int dataIndexCount;
+
+ /** Total uncompressed size of all blocks of the data index */
+ private long uncompressedDataIndexSize;
+
+ /** The number of entries in the meta index */
+ private int metaIndexCount;
+
+ /** The total uncompressed size of keys/values stored in the file. */
+ private long totalUncompressedBytes;
+
+ /**
+ * The number of key/value pairs in the file. This field was int in version 1,
+ * but is now long.
+ */
+ private long entryCount;
+
+ /** The compression codec used for all blocks. */
+ private Compression.Algorithm compressionCodec = Compression.Algorithm.NONE;
+
+ /**
+ * The number of levels in the potentially multi-level data index. Used from
+ * version 2 onwards.
+ */
+ private int numDataIndexLevels;
+
+ /** The offset of the first data block. */
+ private long firstDataBlockOffset;
+
+ /**
+ * It is guaranteed that no key/value data blocks start after this offset in
+ * the file.
+ */
+ private long lastDataBlockOffset;
+
+ /** Raw key comparator class name in version 2 */
+ private String comparatorClassName = RawComparator.class.getName();
+
+ /** The {@link HFile} format version. */
+ private final int version;
+
+ FixedFileTrailer(int version) {
+ this.version = version;
+ HFile.checkFormatVersion(version);
+ }
+
+ private static int[] computeTrailerSizeByVersion() {
+ int versionToSize[] = new int[HFile.MAX_FORMAT_VERSION + 1];
+ for (int version = MIN_FORMAT_VERSION;
+ version <= MAX_FORMAT_VERSION;
+ ++version) {
+ FixedFileTrailer fft = new FixedFileTrailer(version);
+ DataOutputStream dos = new DataOutputStream(new NullOutputStream());
+ try {
+ fft.serialize(dos);
+ } catch (IOException ex) {
+ // The above has no reason to fail.
+ throw new RuntimeException(ex);
+ }
+ versionToSize[version] = dos.size();
+ }
+ return versionToSize;
+ }
+
+ private static int getMaxTrailerSize() {
+ int maxSize = 0;
+ for (int version = MIN_FORMAT_VERSION;
+ version <= MAX_FORMAT_VERSION;
+ ++version)
+ maxSize = Math.max(getTrailerSize(version), maxSize);
+ return maxSize;
+ }
+
+ private static final int TRAILER_SIZE[] = computeTrailerSizeByVersion();
+ private static final int MAX_TRAILER_SIZE = getMaxTrailerSize();
+
+ static int getTrailerSize(int version) {
+ return TRAILER_SIZE[version];
+ }
+
+ public int getTrailerSize() {
+ return getTrailerSize(version);
+ }
+
+ /**
+ * Write the trailer to a data stream. We support writing version 1 for
+ * testing and for determining version 1 trailer size. It is also easy to see
+ * what fields changed in version 2.
+ *
+ * @param outputStream
+ * @throws IOException
+ */
+ void serialize(DataOutputStream outputStream) throws IOException {
+ HFile.checkFormatVersion(version);
+
+ ByteArrayOutputStream baos = new ByteArrayOutputStream();
+ DataOutput baosDos = new DataOutputStream(baos);
+
+ BlockType.TRAILER.write(baosDos);
+ baosDos.writeLong(fileInfoOffset);
+ baosDos.writeLong(loadOnOpenDataOffset);
+ baosDos.writeInt(dataIndexCount);
+
+ if (version == 1) {
+ // This used to be metaIndexOffset, but it was not used in version 1.
+ baosDos.writeLong(0);
+ } else {
+ baosDos.writeLong(uncompressedDataIndexSize);
+ }
+
+ baosDos.writeInt(metaIndexCount);
+ baosDos.writeLong(totalUncompressedBytes);
+ if (version == 1) {
+ baosDos.writeInt((int) Math.min(Integer.MAX_VALUE, entryCount));
+ } else {
+ // This field is long from version 2 onwards.
+ baosDos.writeLong(entryCount);
+ }
+ baosDos.writeInt(compressionCodec.ordinal());
+
+ if (version > 1) {
+ baosDos.writeInt(numDataIndexLevels);
+ baosDos.writeLong(firstDataBlockOffset);
+ baosDos.writeLong(lastDataBlockOffset);
+ Bytes.writeStringFixedSize(baosDos, comparatorClassName,
+ MAX_COMPARATOR_NAME_LENGTH);
+ }
+ baosDos.writeInt(version);
+
+ outputStream.write(baos.toByteArray());
+ }
+
+ /**
+ * Deserialize the fixed file trailer from the given stream. The version needs
+ * to already be specified. Make sure this is consistent with
+ * {@link #serialize(DataOutputStream)}.
+ *
+ * @param inputStream
+ * @param version
+ * @throws IOException
+ */
+ void deserialize(DataInputStream inputStream) throws IOException {
+ HFile.checkFormatVersion(version);
+
+ BlockType.TRAILER.readAndCheck(inputStream);
+
+ fileInfoOffset = inputStream.readLong();
+ loadOnOpenDataOffset = inputStream.readLong();
+ dataIndexCount = inputStream.readInt();
+
+ if (version == 1) {
+ inputStream.readLong(); // Read and skip metaIndexOffset.
+ } else {
+ uncompressedDataIndexSize = inputStream.readLong();
+ }
+ metaIndexCount = inputStream.readInt();
+
+ totalUncompressedBytes = inputStream.readLong();
+ entryCount = version == 1 ? inputStream.readInt() : inputStream.readLong();
+ compressionCodec = Compression.Algorithm.values()[inputStream.readInt()];
+ if (version > 1) {
+ numDataIndexLevels = inputStream.readInt();
+ firstDataBlockOffset = inputStream.readLong();
+ lastDataBlockOffset = inputStream.readLong();
+ comparatorClassName =
+ Bytes.readStringFixedSize(inputStream, MAX_COMPARATOR_NAME_LENGTH);
+ }
+
+ int versionRead = inputStream.readInt();
+ if (versionRead != version) {
+ throw new IOException("Version provided=" + version + ", read="
+ + versionRead);
+ }
+ }
+
+ private void append(StringBuilder sb, String s) {
+ if (sb.length() > 0)
+ sb.append(", ");
+ sb.append(s);
+ }
+
+ @Override
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ append(sb, "fileinfoOffset=" + fileInfoOffset);
+ append(sb, "loadOnOpenDataOffset=" + loadOnOpenDataOffset);
+ append(sb, "dataIndexCount=" + dataIndexCount);
+ append(sb, "metaIndexCount=" + metaIndexCount);
+ append(sb, "totalUncomressedBytes=" + totalUncompressedBytes);
+ append(sb, "entryCount=" + entryCount);
+ append(sb, "compressionCodec=" + compressionCodec);
+ if (version == 2) {
+ append(sb, "uncompressedDataIndexSize=" + uncompressedDataIndexSize);
+ append(sb, "numDataIndexLevels=" + numDataIndexLevels);
+ append(sb, "firstDataBlockOffset=" + firstDataBlockOffset);
+ append(sb, "lastDataBlockOffset=" + lastDataBlockOffset);
+ append(sb, "comparatorClassName=" + comparatorClassName);
+ }
+ append(sb, "version=" + version);
+
+ return sb.toString();
+ }
+
+ /**
+ * Reads a file trailer from the given file.
+ *
+ * @param istream the input stream with the ability to seek. Does not have to
+ * be buffered, as only one read operation is made.
+ * @param fileSize the file size. Can be obtained using
+ * {@link org.apache.hadoop.fs.FileSystem#getFileStatus(
+ * org.apache.hadoop.fs.Path)}.
+ * @return the fixed file trailer read
+ * @throws IOException if failed to read from the underlying stream, or the
+ * trailer is corrupted, or the version of the trailer is
+ * unsupported
+ */
+ public static FixedFileTrailer readFromStream(FSDataInputStream istream,
+ long fileSize) throws IOException {
+ int bufferSize = MAX_TRAILER_SIZE;
+ long seekPoint = fileSize - bufferSize;
+ if (seekPoint < 0) {
+ // It is hard to imagine such a small HFile.
+ seekPoint = 0;
+ bufferSize = (int) fileSize;
+ }
+
+ istream.seek(seekPoint);
+ ByteBuffer buf = ByteBuffer.allocate(bufferSize);
+ istream.readFully(buf.array(), buf.arrayOffset(),
+ buf.arrayOffset() + buf.limit());
+
+ // Read the version from the last int of the file.
+ buf.position(buf.limit() - Bytes.SIZEOF_INT);
+ int version = buf.getInt();
+
+ try {
+ HFile.checkFormatVersion(version);
+ } catch (IllegalArgumentException iae) {
+ // In this context, an invalid version might indicate a corrupt HFile.
+ throw new IOException(iae);
+ }
+
+ int trailerSize = getTrailerSize(version);
+
+ FixedFileTrailer fft = new FixedFileTrailer(version);
+ fft.deserialize(new DataInputStream(new ByteArrayInputStream(buf.array(),
+ buf.arrayOffset() + bufferSize - trailerSize, trailerSize)));
+ return fft;
+ }
+
+ public void expectVersion(int expected) {
+ if (version != expected) {
+ throw new IllegalArgumentException("Invalid HFile version: " + version
+ + " (expected: " + expected + ")");
+ }
+ }
+
+ public void expectAtLeastVersion(int lowerBound) {
+ if (version < lowerBound) {
+ throw new IllegalArgumentException("Invalid HFile version: " + version
+ + " (expected: " + lowerBound + " or higher).");
+ }
+ }
+
+ public long getFileInfoOffset() {
+ return fileInfoOffset;
+ }
+
+ public void setFileInfoOffset(long fileInfoOffset) {
+ this.fileInfoOffset = fileInfoOffset;
+ }
+
+ public long getLoadOnOpenDataOffset() {
+ return loadOnOpenDataOffset;
+ }
+
+ public void setLoadOnOpenOffset(long loadOnOpenDataOffset) {
+ this.loadOnOpenDataOffset = loadOnOpenDataOffset;
+ }
+
+ public int getDataIndexCount() {
+ return dataIndexCount;
+ }
+
+ public void setDataIndexCount(int dataIndexCount) {
+ this.dataIndexCount = dataIndexCount;
+ }
+
+ public int getMetaIndexCount() {
+ return metaIndexCount;
+ }
+
+ public void setMetaIndexCount(int metaIndexCount) {
+ this.metaIndexCount = metaIndexCount;
+ }
+
+ public long getTotalUncompressedBytes() {
+ return totalUncompressedBytes;
+ }
+
+ public void setTotalUncompressedBytes(long totalUncompressedBytes) {
+ this.totalUncompressedBytes = totalUncompressedBytes;
+ }
+
+ public long getEntryCount() {
+ return entryCount;
+ }
+
+ public void setEntryCount(long newEntryCount) {
+ if (version == 1) {
+ int intEntryCount = (int) Math.min(Integer.MAX_VALUE, newEntryCount);
+ if (intEntryCount != newEntryCount) {
+ LOG.info("Warning: entry count is " + newEntryCount + " but writing "
+ + intEntryCount + " into the version " + version + " trailer");
+ }
+ entryCount = intEntryCount;
+ return;
+ }
+ entryCount = newEntryCount;
+ }
+
+ public Compression.Algorithm getCompressionCodec() {
+ return compressionCodec;
+ }
+
+ public void setCompressionCodec(Compression.Algorithm compressionCodec) {
+ this.compressionCodec = compressionCodec;
+ }
+
+ public int getNumDataIndexLevels() {
+ expectAtLeastVersion(2);
+ return numDataIndexLevels;
+ }
+
+ public void setNumDataIndexLevels(int numDataIndexLevels) {
+ expectAtLeastVersion(2);
+ this.numDataIndexLevels = numDataIndexLevels;
+ }
+
+ public long getLastDataBlockOffset() {
+ expectAtLeastVersion(2);
+ return lastDataBlockOffset;
+ }
+
+ public void setLastDataBlockOffset(long lastDataBlockOffset) {
+ expectAtLeastVersion(2);
+ this.lastDataBlockOffset = lastDataBlockOffset;
+ }
+
+ public long getFirstDataBlockOffset() {
+ expectAtLeastVersion(2);
+ return firstDataBlockOffset;
+ }
+
+ public void setFirstDataBlockOffset(long firstDataBlockOffset) {
+ expectAtLeastVersion(2);
+ this.firstDataBlockOffset = firstDataBlockOffset;
+ }
+
+ public int getVersion() {
+ return version;
+ }
+
+ @SuppressWarnings("rawtypes")
+ public void setComparatorClass(Class<? extends RawComparator> klass) {
+ expectAtLeastVersion(2);
+ comparatorClassName = klass.getName();
+ }
+
+ @SuppressWarnings("unchecked")
+ private static Class<? extends RawComparator<byte[]>> getComparatorClass(
+ String comparatorClassName) throws IOException {
+ try {
+ return (Class<? extends RawComparator<byte[]>>)
+ Class.forName(comparatorClassName);
+ } catch (ClassNotFoundException ex) {
+ throw new IOException(ex);
+ }
+ }
+
+ public static RawComparator<byte[]> createComparator(
+ String comparatorClassName) throws IOException {
+ try {
+ return getComparatorClass(comparatorClassName).newInstance();
+ } catch (InstantiationException e) {
+ throw new IOException(e);
+ } catch (IllegalAccessException e) {
+ throw new IOException(e);
+ }
+ }
+
+ RawComparator<byte[]> createComparator() throws IOException {
+ expectAtLeastVersion(2);
+ return createComparator(comparatorClassName);
+ }
+
+ public long getUncompressedDataIndexSize() {
+ if (version == 1)
+ return 0;
+ return uncompressedDataIndexSize;
+ }
+
+ public void setUncompressedDataIndexSize(
+ long uncompressedDataIndexSize) {
+ expectAtLeastVersion(2);
+ this.uncompressedDataIndexSize = uncompressedDataIndexSize;
+ }
+
+}