You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@zookeeper.apache.org by iv...@apache.org on 2012/02/09 18:14:30 UTC
svn commit: r1242404 - in /zookeeper/bookkeeper/trunk: ./
bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/
bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/
bookkeeper-server/src/main/java/org/apache/bookkeeper/util/ bookkeeper-...
Author: ivank
Date: Thu Feb 9 17:14:29 2012
New Revision: 1242404
URL: http://svn.apache.org/viewvc?rev=1242404&view=rev
Log:
BOOKKEEPER-137: Do not create Ledger index files until absolutely necessary. (ivank)
Modified:
zookeeper/bookkeeper/trunk/CHANGES.txt
zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/Bookie.java
zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/FileInfo.java
zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/JournalChannel.java
zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerCache.java
zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerDescriptor.java
zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieServer.java
zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/util/LocalBookKeeper.java
zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/BookieJournalTest.java
zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/BookieLayoutVersionTest.java
zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/BookieRecoveryTest.java
zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/BaseTestCase.java
Modified: zookeeper/bookkeeper/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/CHANGES.txt?rev=1242404&r1=1242403&r2=1242404&view=diff
==============================================================================
--- zookeeper/bookkeeper/trunk/CHANGES.txt (original)
+++ zookeeper/bookkeeper/trunk/CHANGES.txt Thu Feb 9 17:14:29 2012
@@ -49,6 +49,8 @@ Trunk (unreleased changes)
BOOKKEEPER-165: Add versioning support for journal files (ivank)
+ BOOKKEEPER-137: Do not create Ledger index files until absolutely necessary. (ivank)
+
hedwig-server/
BOOKKEEPER-77: Add a console client for hedwig (Sijie Guo via ivank)
Modified: zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/Bookie.java
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/Bookie.java?rev=1242404&r1=1242403&r2=1242404&view=diff
==============================================================================
--- zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/Bookie.java (original)
+++ zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/Bookie.java Thu Feb 9 17:14:29 2012
@@ -87,9 +87,12 @@ public class Bookie extends Thread {
* when you make a change to the format of any of the files in
* this directory or to the general layout of the directory.
*/
- static final int CURRENT_DIRECTORY_LAYOUT_VERSION = 1;
+ static final int MIN_COMPAT_DIRECTORY_LAYOUT_VERSION = 1;
+ static final int CURRENT_DIRECTORY_LAYOUT_VERSION = 2;
static final String VERSION_FILENAME = "VERSION";
-
+
+ static final long METAENTRY_ID_LEDGER_KEY = -0x1000;
+
// ZK registration path for this bookie
static final String BOOKIE_REGISTRATION_PATH = "/ledgers/available/";
@@ -250,7 +253,7 @@ public class Bookie extends Thread {
}
public Bookie(ServerConfiguration conf)
- throws IOException, KeeperException, InterruptedException {
+ throws IOException, KeeperException, InterruptedException, BookieException {
super("Bookie-" + conf.getBookiePort());
this.conf = conf;
this.journalDirectory = conf.getJournalDir();
@@ -336,12 +339,28 @@ public class Bookie extends Thread {
if (LOG.isDebugEnabled()) {
LOG.debug("Relay journal - ledger id : " + ledgerId);
}
- LedgerDescriptor handle = getHandle(ledgerId, false);
- try {
- recBuff.rewind();
- handle.addEntry(recBuff);
- } finally {
- putHandle(handle);
+ LedgerDescriptor handle = getHandle(ledgerId);
+ long entryId = recBuff.getLong();
+ if (entryId == METAENTRY_ID_LEDGER_KEY) {
+ if (recLog.getFormatVersion() >= 3) {
+ int masterKeyLen = recBuff.getInt();
+ byte[] masterKey = new byte[masterKeyLen];
+ recBuff.get(masterKey);
+
+ handle.checkAccess(masterKey);
+ putHandle(handle);
+ } else {
+ throw new IOException("Invalid journal. Contains journalKey "
+ + " but layout version (" + recLog.getFormatVersion()
+ + ") is too old to hold this");
+ }
+ } else {
+ try {
+ recBuff.rewind();
+ handle.addEntry(recBuff);
+ } finally {
+ putHandle(handle);
+ }
}
}
recLog.close();
@@ -540,7 +559,7 @@ public class Bookie extends Thread {
* @throws IOException if layout version if is outside usable range
* or if there is a problem reading the version file
*/
- private void checkDirectoryLayoutVersion(File dir)
+ private void checkDirectoryLayoutVersion(File dir)
throws IOException {
if (!dir.isDirectory()) {
throw new IOException("Directory("+dir+") isn't a directory");
@@ -566,8 +585,10 @@ public class Bookie extends Thread {
try {
String layoutVersionStr = br.readLine();
int layoutVersion = Integer.parseInt(layoutVersionStr);
- if (layoutVersion != CURRENT_DIRECTORY_LAYOUT_VERSION) {
- String errmsg = "Directory has an invalid version, expected "
+ if (layoutVersion < MIN_COMPAT_DIRECTORY_LAYOUT_VERSION
+ || layoutVersion > CURRENT_DIRECTORY_LAYOUT_VERSION) {
+ String errmsg = "Directory has an invalid version, expected between "
+ + MIN_COMPAT_DIRECTORY_LAYOUT_VERSION + " and "
+ CURRENT_DIRECTORY_LAYOUT_VERSION + ", found " + layoutVersion;
LOG.error(errmsg);
throw new IOException(errmsg);
@@ -620,73 +641,38 @@ public class Bookie extends Thread {
}
}
- private LedgerDescriptor getHandle(long ledgerId, boolean readonly, byte[] masterKey) throws IOException {
+ private LedgerDescriptor getHandle(long ledgerId, boolean readonly, byte[] masterKey)
+ throws IOException, BookieException {
LedgerDescriptor handle = null;
synchronized (ledgers) {
handle = ledgers.get(ledgerId);
if (handle == null) {
- FileInfo fi = null;
- try {
- // get file info will throw NoLedgerException
- fi = ledgerCache.getFileInfo(ledgerId, !readonly);
-
- // if an existed ledger index file, we can get its master key
- // if an new created ledger index file, we will get a null master key
- byte[] existingMasterKey = fi.readMasterKey();
- ByteBuffer masterKeyToSet = ByteBuffer.wrap(masterKey);
- if (existingMasterKey == null) {
- // no master key set before
- fi.writeMasterKey(masterKey);
- } else if (!masterKeyToSet.equals(ByteBuffer.wrap(existingMasterKey))) {
- throw new IOException("Wrong master key for ledger " + ledgerId);
- }
- handle = createHandle(ledgerId, readonly);
- ledgers.put(ledgerId, handle);
- handle.setMasterKey(masterKeyToSet);
- } finally {
- if (fi != null) {
- fi.release();
- }
+ if (readonly) {
+ throw new NoLedgerException(ledgerId);
}
+ handle = createHandle(ledgerId);
+ ledgers.put(ledgerId, handle);
}
+ handle.checkAccess(masterKey);
handle.incRef();
}
return handle;
}
- private LedgerDescriptor getHandle(long ledgerId, boolean readonly) throws IOException {
+ private LedgerDescriptor getHandle(long ledgerId) throws IOException {
LedgerDescriptor handle = null;
synchronized (ledgers) {
handle = ledgers.get(ledgerId);
if (handle == null) {
- FileInfo fi = null;
- try {
- // get file info will throw NoLedgerException
- fi = ledgerCache.getFileInfo(ledgerId, !readonly);
-
- // if an existed ledger index file, we can get its master key
- // if an new created ledger index file, we will get a null master key
- byte[] existingMasterKey = fi.readMasterKey();
- if (existingMasterKey == null) {
- throw new IOException("Weird! No master key found in ledger " + ledgerId);
- }
-
- handle = createHandle(ledgerId, readonly);
- ledgers.put(ledgerId, handle);
- handle.setMasterKey(ByteBuffer.wrap(existingMasterKey));
- } finally {
- if (fi != null) {
- fi.release();
- }
- }
+ handle = createHandle(ledgerId);
+ ledgers.put(ledgerId, handle);
}
handle.incRef();
}
return handle;
}
-
- private LedgerDescriptor createHandle(long ledgerId, boolean readOnly) throws IOException {
+ private LedgerDescriptor createHandle(long ledgerId) throws IOException {
return new LedgerDescriptor(ledgerId, entryLogger, ledgerCache);
}
@@ -920,13 +906,28 @@ public class Bookie extends Thread {
private LedgerDescriptor getLedgerForEntry(ByteBuffer entry, byte[] masterKey)
throws IOException, BookieException {
long ledgerId = entry.getLong();
- LedgerDescriptor handle = getHandle(ledgerId, false, masterKey);
-
- if(!handle.cmpMasterKey(ByteBuffer.wrap(masterKey))) {
- putHandle(handle);
- throw BookieException.create(BookieException.Code.UnauthorizedAccessException);
+ LedgerDescriptor l = getHandle(ledgerId, false, masterKey);
+ if (!l.isMasterKeyPersisted()) {
+ // new handle, we should add the key to journal ensure we can rebuild
+ ByteBuffer bb = ByteBuffer.allocate(8 + 8 + 4 + masterKey.length);
+ bb.putLong(ledgerId);
+ bb.putLong(METAENTRY_ID_LEDGER_KEY);
+ bb.putInt(masterKey.length);
+ bb.put(masterKey);
+ bb.flip();
+
+ queue.add(new QueueEntry(bb,
+ ledgerId, METAENTRY_ID_LEDGER_KEY,
+ new WriteCallback() {
+ public void writeComplete(int rc, long ledgerId,
+ long entryId, InetSocketAddress addr,
+ Object ctx) {
+ // do nothing
+ }
+ }, null));
+ l.setMasterKeyPersisted();
}
- return handle;
+ return l;
}
/**
@@ -991,14 +992,14 @@ public class Bookie extends Thread {
* never be unfenced. Fencing a fenced ledger has no effect.
*/
public void fenceLedger(long ledgerId) throws IOException {
- LedgerDescriptor handle = getHandle(ledgerId, true);
+ LedgerDescriptor handle = getHandle(ledgerId);
synchronized (handle) {
handle.setFenced();
}
}
public ByteBuffer readEntry(long ledgerId, long entryId) throws IOException {
- LedgerDescriptor handle = getHandle(ledgerId, true);
+ LedgerDescriptor handle = getHandle(ledgerId);
try {
if (LOG.isTraceEnabled()) {
LOG.trace("Reading " + entryId + "@" + ledgerId);
Modified: zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/FileInfo.java
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/FileInfo.java?rev=1242404&r1=1242403&r2=1242404&view=diff
==============================================================================
--- zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/FileInfo.java (original)
+++ zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/FileInfo.java Thu Feb 9 17:14:29 2012
@@ -40,7 +40,7 @@ import org.slf4j.LoggerFactory;
* <b>Header</b> is formated as below:
* <pre><magic bytes><len of master key><master key></pre>
* <ul>
- * <li>magic bytes: 8 bytes, 'BKLE\0\0\0\0'
+ * <li>magic bytes: 4 bytes, 'BKLE', version: 4 bytes
* <li>len of master key: indicates length of master key. -1 means no master key stored in header.
* <li>master key: master key
* </ul>
@@ -54,78 +54,86 @@ class FileInfo {
private FileChannel fc;
private final File lf;
+ byte[] masterKey;
+
/**
* The fingerprint of a ledger index file
*/
- private byte header[] = "BKLE\0\0\0\0".getBytes();
+ static final public int signature = ByteBuffer.wrap("BKLE".getBytes()).getInt();
+ static final public int headerVersion = 0;
+
static final long START_OF_DATA = 1024;
private long size;
private int useCount;
private boolean isClosed;
- public FileInfo(File lf) throws IOException {
+
+ public FileInfo(File lf, byte[] masterKey) throws IOException {
this.lf = lf;
- fc = new RandomAccessFile(lf, "rws").getChannel();
- size = fc.size();
- if (size == 0) {
- fc.write(ByteBuffer.wrap(header));
- // write NO_MASTER_KEY, which means there is no master key
- ByteBuffer buf = ByteBuffer.allocate(4);
- buf.putInt(NO_MASTER_KEY);
- buf.flip();
- fc.write(buf);
- }
+
+ this.masterKey = masterKey;
}
- /**
- * Write master key to index file header
- *
- * @param masterKey master key to store
- * @return void
- * @throws IOException
- */
- synchronized public void writeMasterKey(byte[] masterKey) throws IOException {
- // write master key
- if (masterKey == null ||
- masterKey.length + 4 + header.length > START_OF_DATA) {
- throw new IOException("master key is more than " + (START_OF_DATA - 4 - header.length));
- }
+ synchronized public void readHeader() throws IOException {
+ if (lf.exists()) {
+ if (fc != null) {
+ return;
+ }
+
+ fc = new RandomAccessFile(lf, "rw").getChannel();
+ size = fc.size();
- int len = masterKey.length;
- ByteBuffer lenBuf = ByteBuffer.allocate(4);
- lenBuf.putInt(len);
- lenBuf.flip();
- fc.position(header.length);
- fc.write(lenBuf);
- fc.write(ByteBuffer.wrap(masterKey));
+ ByteBuffer bb = ByteBuffer.allocate(1024);
+ while(bb.hasRemaining()) {
+ fc.read(bb);
+ }
+ bb.flip();
+ if (bb.getInt() != signature) {
+ throw new IOException("Missing ledger signature");
+ }
+ int version = bb.getInt();
+ if (version != headerVersion) {
+ throw new IOException("Incompatible ledger version " + version);
+ }
+ int length = bb.getInt();
+ if (length < 0 || length > bb.remaining()) {
+ throw new IOException("Length " + length + " is invalid");
+ }
+ masterKey = new byte[length];
+ bb.get(masterKey);
+ } else {
+ throw new IOException("Ledger index file does not exist");
+ }
}
- /**
- * Read master key
- *
- * @return master key. null means no master key stored in index header
- * @throws IOException
- */
- synchronized public byte[] readMasterKey() throws IOException {
- ByteBuffer lenBuf = ByteBuffer.allocate(4);
- int total = readAbsolute(lenBuf, header.length);
- if (total != 4) {
- throw new IOException("Short read during reading master key length");
- }
- lenBuf.rewind();
- int len = lenBuf.getInt();
- if (len == NO_MASTER_KEY) {
- return null;
- }
-
- byte[] masterKey = new byte[len];
- total = readAbsolute(ByteBuffer.wrap(masterKey), header.length + 4);
- if (total != len) {
- throw new IOException("Short read during reading master key");
+ synchronized private void checkOpen(boolean create) throws IOException {
+ if (fc != null) {
+ return;
+ }
+ boolean exists = lf.exists();
+ if (masterKey == null && !exists) {
+ throw new IOException(lf + " not found");
+ }
+ ByteBuffer bb = ByteBuffer.allocate(1024);
+ if (!exists) {
+ if (create) {
+ fc = new RandomAccessFile(lf, "rw").getChannel();
+ size = fc.size();
+ if (size == 0) {
+ bb.putInt(signature);
+ bb.putInt(headerVersion);
+ bb.putInt(masterKey.length);
+ bb.put(masterKey);
+ bb.rewind();
+ fc.write(bb);
+ }
+ }
+ } else {
+ readHeader();
}
- return masterKey;
}
- synchronized public long size() {
+ synchronized public long size() throws IOException {
+ checkOpen(false);
long rc = size-START_OF_DATA;
if (rc < 0) {
rc = 0;
@@ -138,6 +146,7 @@ class FileInfo {
}
private int readAbsolute(ByteBuffer bb, long start) throws IOException {
+ checkOpen(false);
int total = 0;
while(bb.remaining() > 0) {
int rc = fc.read(bb, start);
@@ -153,12 +162,13 @@ class FileInfo {
synchronized public void close() throws IOException {
isClosed = true;
- if (useCount == 0) {
+ if (useCount == 0 && fc != null) {
fc.close();
}
}
synchronized public long write(ByteBuffer[] buffs, long position) throws IOException {
+ checkOpen(true);
long total = 0;
try {
fc.position(position+START_OF_DATA);
@@ -170,6 +180,7 @@ class FileInfo {
total += rc;
}
} finally {
+ fc.force(true);
long newsize = position+START_OF_DATA+total;
if (newsize > size) {
size = newsize;
@@ -178,13 +189,18 @@ class FileInfo {
return total;
}
+ synchronized public byte[] getMasterKey() throws IOException {
+ checkOpen(false);
+ return masterKey;
+ }
+
synchronized public void use() {
useCount++;
}
synchronized public void release() {
useCount--;
- if (isClosed && useCount == 0) {
+ if (isClosed && useCount == 0 && fc != null) {
try {
fc.close();
} catch (IOException e) {
@@ -193,12 +209,7 @@ class FileInfo {
}
}
- /**
- * Getter to a handle on the actual ledger index file.
- * This is used when we are deleting a ledger and want to physically remove the index file.
- */
- File getFile() {
- return lf;
+ public boolean delete() {
+ return lf.delete();
}
-
}
Modified: zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/JournalChannel.java
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/JournalChannel.java?rev=1242404&r1=1242403&r2=1242404&view=diff
==============================================================================
--- zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/JournalChannel.java (original)
+++ zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/JournalChannel.java Thu Feb 9 17:14:29 2012
@@ -50,7 +50,7 @@ class JournalChannel {
int HEADER_SIZE = 8; // 4byte magic word, 4 byte version
int MIN_COMPAT_JOURNAL_FORMAT_VERSION = 1;
- int CURRENT_JOURNAL_FORMAT_VERSION = 2;
+ int CURRENT_JOURNAL_FORMAT_VERSION = 3;
public final static long preAllocSize = 4*1024*1024;
public final static ByteBuffer zeros = ByteBuffer.allocate(512);
Modified: zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerCache.java
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerCache.java?rev=1242404&r1=1242403&r2=1242404&view=diff
==============================================================================
--- zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerCache.java (original)
+++ zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerCache.java Thu Feb 9 17:14:29 2012
@@ -226,7 +226,7 @@ public class LedgerCache {
return dirs[rand.nextInt(dirs.length)];
}
- FileInfo getFileInfo(Long ledger, boolean create) throws IOException {
+ FileInfo getFileInfo(Long ledger, byte masterKey[]) throws IOException {
synchronized(fileInfoCache) {
FileInfo fi = fileInfoCache.get(ledger);
if (fi == null) {
@@ -240,7 +240,7 @@ public class LedgerCache {
lf = null;
}
if (lf == null) {
- if (!create) {
+ if (masterKey == null) {
throw new Bookie.NoLedgerException(ledger);
}
File dir = pickDirs(ledgerDirectories);
@@ -256,7 +256,7 @@ public class LedgerCache {
if (openLedgers.size() > openFileLimit) {
fileInfoCache.remove(openLedgers.removeFirst()).close();
}
- fi = new FileInfo(lf);
+ fi = new FileInfo(lf, masterKey);
fileInfoCache.put(ledger, fi);
openLedgers.add(ledger);
}
@@ -272,7 +272,7 @@ public class LedgerCache {
}
FileInfo fi = null;
try {
- fi = getFileInfo(lep.getLedger(), true);
+ fi = getFileInfo(lep.getLedger(), null);
long pos = lep.getFirstEntry()*8;
if (pos >= fi.size()) {
lep.zeroPage();
@@ -338,7 +338,7 @@ public class LedgerCache {
}
});
ArrayList<Integer> versions = new ArrayList<Integer>(entries.size());
- fi = getFileInfo(l, true);
+ fi = getFileInfo(l, null);
int start = 0;
long lastOffset = -1;
for(int i = 0; i < entries.size(); i++) {
@@ -427,6 +427,7 @@ public class LedgerCache {
LedgerEntryPage lep = new LedgerEntryPage(pageSize, entriesPerPage);
lep.setLedger(ledger);
lep.setFirstEntry(entry);
+
// note, this will not block since it is a new page
lep.usePage();
pageCount++;
@@ -536,8 +537,8 @@ public class LedgerCache {
if (LOG.isDebugEnabled())
LOG.debug("Deleting ledgerId: " + ledgerId);
// Delete the ledger's index file and close the FileInfo
- FileInfo fi = getFileInfo(ledgerId, false);
- fi.getFile().delete();
+ FileInfo fi = getFileInfo(ledgerId, null);
+ fi.delete();
fi.close();
// Remove it from the active ledger manager
Modified: zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerDescriptor.java
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerDescriptor.java?rev=1242404&r1=1242403&r2=1242404&view=diff
==============================================================================
--- zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerDescriptor.java (original)
+++ zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/bookie/LedgerDescriptor.java Thu Feb 9 17:14:29 2012
@@ -23,6 +23,7 @@ package org.apache.bookkeeper.bookie;
import java.io.IOException;
import java.nio.ByteBuffer;
+import java.util.Arrays;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -43,17 +44,46 @@ public class LedgerDescriptor {
this.ledgerCache = ledgerCache;
}
- private ByteBuffer masterKey = null;
+ private byte[] masterKey = null;
volatile private boolean fenced = false;
+ private boolean masterKeyPersisted = false;
- void setMasterKey(ByteBuffer masterKey) {
- this.masterKey = masterKey;
+ synchronized boolean isMasterKeyPersisted() {
+ if (masterKeyPersisted) {
+ return true;
+ }
+
+ try {
+ FileInfo fi = ledgerCache.getFileInfo(ledgerId, masterKey);
+ fi.readHeader();
+ masterKeyPersisted = true;
+ return true;
+ } catch (IOException ioe) {
+ return false;
+ }
}
- boolean cmpMasterKey(ByteBuffer masterKey) {
- return this.masterKey.equals(masterKey);
+ void setMasterKeyPersisted() {
+ masterKeyPersisted = true;
}
+ void checkAccess(byte masterKey[]) throws BookieException, IOException {
+ if (this.masterKey == null) {
+ FileInfo fi = ledgerCache.getFileInfo(ledgerId, masterKey);
+ try {
+ if (fi == null) {
+ throw new IOException(ledgerId + " does not exist");
+ }
+ this.masterKey = fi.getMasterKey();
+ } finally {
+ fi.release();
+ }
+ }
+ if (!Arrays.equals(this.masterKey, masterKey)) {
+ throw BookieException.create(BookieException.Code.UnauthorizedAccessException);
+ }
+ }
+
private long ledgerId;
public long getLedgerId() {
return ledgerId;
@@ -109,7 +139,7 @@ public class LedgerDescriptor {
long lastEntry = ledgerCache.getLastEntry(ledgerId);
FileInfo fi = null;
try {
- fi = ledgerCache.getFileInfo(ledgerId, false);
+ fi = ledgerCache.getFileInfo(ledgerId, null);
long size = fi.size();
// we may not have the last entry in the cache
if (size > lastEntry*8) {
Modified: zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieServer.java
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieServer.java?rev=1242404&r1=1242403&r2=1242404&view=diff
==============================================================================
--- zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieServer.java (original)
+++ zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/BookieServer.java Thu Feb 9 17:14:29 2012
@@ -67,7 +67,7 @@ public class BookieServer implements NIO
protected BookieServerBean jmxBkServerBean;
public BookieServer(ServerConfiguration conf)
- throws IOException, KeeperException, InterruptedException {
+ throws IOException, KeeperException, InterruptedException, BookieException {
this.conf = conf;
this.bookie = new Bookie(conf);
@@ -261,7 +261,7 @@ public class BookieServer implements NIO
* @throws InterruptedException
*/
public static void main(String[] args)
- throws IOException, KeeperException, InterruptedException {
+ throws IOException, KeeperException, InterruptedException, BookieException {
ServerConfiguration conf = null;
try {
conf = parseArgs(args);
Modified: zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/util/LocalBookKeeper.java
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/util/LocalBookKeeper.java?rev=1242404&r1=1242403&r2=1242404&view=diff
==============================================================================
--- zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/util/LocalBookKeeper.java (original)
+++ zookeeper/bookkeeper/trunk/bookkeeper-server/src/main/java/org/apache/bookkeeper/util/LocalBookKeeper.java Thu Feb 9 17:14:29 2012
@@ -27,6 +27,7 @@ import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.net.Socket;
+import org.apache.bookkeeper.bookie.BookieException;
import org.apache.bookkeeper.conf.ServerConfiguration;
import org.apache.bookkeeper.proto.BookieServer;
import org.slf4j.Logger;
@@ -123,7 +124,7 @@ public class LocalBookKeeper {
}
}
private void runBookies(ServerConfiguration baseConf)
- throws IOException, KeeperException, InterruptedException {
+ throws IOException, KeeperException, InterruptedException, BookieException {
LOG.info("Starting Bookie(s)");
// Create Bookie Servers (B1, B2, B3)
@@ -150,7 +151,7 @@ public class LocalBookKeeper {
}
public static void main(String[] args)
- throws IOException, KeeperException, InterruptedException {
+ throws IOException, KeeperException, InterruptedException, BookieException {
if(args.length < 1) {
usage();
System.exit(-1);
Modified: zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/BookieJournalTest.java
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/BookieJournalTest.java?rev=1242404&r1=1242403&r2=1242404&view=diff
==============================================================================
--- zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/BookieJournalTest.java (original)
+++ zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/BookieJournalTest.java Thu Feb 9 17:14:29 2012
@@ -59,8 +59,9 @@ public class BookieJournalTest {
throws Exception {
File fn = new File(indexDir, LedgerCache.getLedgerName(ledgerId));
fn.getParentFile().mkdirs();
- FileInfo fi = new FileInfo(fn);
- fi.writeMasterKey(masterKey);
+ FileInfo fi = new FileInfo(fn, masterKey);
+ // force creation of index file
+ fi.write(new ByteBuffer[]{ ByteBuffer.allocate(0) }, 0);
fi.close();
}
Modified: zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/BookieLayoutVersionTest.java
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/BookieLayoutVersionTest.java?rev=1242404&r1=1242403&r2=1242404&view=diff
==============================================================================
--- zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/BookieLayoutVersionTest.java (original)
+++ zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/bookie/BookieLayoutVersionTest.java Thu Feb 9 17:14:29 2012
@@ -126,7 +126,7 @@ public class BookieLayoutVersionTest ext
try {
Bookie b = new Bookie(newServerConfiguration(
BOOKIE_PORT, HOSTPORT, newDirectoryWithoutVersion(),
- new File[] { newDirectory(Bookie.CURRENT_DIRECTORY_LAYOUT_VERSION - 1) }));
+ new File[] { newDirectory(Bookie.MIN_COMPAT_DIRECTORY_LAYOUT_VERSION - 1) }));
fail("Shouldn't reach here");
} catch (IOException ioe) {
assertTrue("Invalid exception",
@@ -136,7 +136,7 @@ public class BookieLayoutVersionTest ext
// test with bad data dir
try {
Bookie b = new Bookie(newServerConfiguration(
- BOOKIE_PORT, HOSTPORT, newDirectory(Bookie.CURRENT_DIRECTORY_LAYOUT_VERSION - 1),
+ BOOKIE_PORT, HOSTPORT, newDirectory(Bookie.MIN_COMPAT_DIRECTORY_LAYOUT_VERSION - 1),
new File[] { newDirectoryWithoutVersion() }));
fail("Shouldn't reach here");
} catch (IOException ioe) {
@@ -147,8 +147,8 @@ public class BookieLayoutVersionTest ext
// test with both bad
try {
Bookie b = new Bookie(newServerConfiguration(
- BOOKIE_PORT, HOSTPORT, newDirectory(Bookie.CURRENT_DIRECTORY_LAYOUT_VERSION - 1),
- new File[] { newDirectory(Bookie.CURRENT_DIRECTORY_LAYOUT_VERSION - 1) }));
+ BOOKIE_PORT, HOSTPORT, newDirectory(Bookie.MIN_COMPAT_DIRECTORY_LAYOUT_VERSION - 1),
+ new File[] { newDirectory(Bookie.MIN_COMPAT_DIRECTORY_LAYOUT_VERSION - 1) }));
fail("Shouldn't reach here");
} catch (IOException ioe) {
assertTrue("Invalid exception",
@@ -162,11 +162,11 @@ public class BookieLayoutVersionTest ext
try {
Bookie b = new Bookie(newServerConfiguration(
BOOKIE_PORT, HOSTPORT, newDirectory(Bookie.CURRENT_DIRECTORY_LAYOUT_VERSION),
- new File[] { newDirectory(Bookie.CURRENT_DIRECTORY_LAYOUT_VERSION - 1),
+ new File[] { newDirectory(Bookie.MIN_COMPAT_DIRECTORY_LAYOUT_VERSION - 1),
newDirectory(Bookie.CURRENT_DIRECTORY_LAYOUT_VERSION),
newDirectory(Bookie.CURRENT_DIRECTORY_LAYOUT_VERSION + 1),
newDirectory(Bookie.CURRENT_DIRECTORY_LAYOUT_VERSION),
- newDirectory(Bookie.CURRENT_DIRECTORY_LAYOUT_VERSION - 1),}));
+ newDirectory(Bookie.MIN_COMPAT_DIRECTORY_LAYOUT_VERSION - 1),}));
fail("Shouldn't reach here");
} catch (IOException ioe) {
assertTrue("Invalid exception",
Modified: zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/BookieRecoveryTest.java
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/BookieRecoveryTest.java?rev=1242404&r1=1242403&r2=1242404&view=diff
==============================================================================
--- zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/BookieRecoveryTest.java (original)
+++ zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/BookieRecoveryTest.java Thu Feb 9 17:14:29 2012
@@ -50,6 +50,7 @@ import org.apache.bookkeeper.client.Asyn
import org.apache.bookkeeper.client.BookKeeper.DigestType;
import org.apache.bookkeeper.proto.BookieServer;
import org.apache.bookkeeper.client.BookKeeperAdmin;
+import org.apache.bookkeeper.bookie.BookieException;
import org.apache.zookeeper.data.Stat;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
Modified: zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/BaseTestCase.java
URL: http://svn.apache.org/viewvc/zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/BaseTestCase.java?rev=1242404&r1=1242403&r2=1242404&view=diff
==============================================================================
--- zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/BaseTestCase.java (original)
+++ zookeeper/bookkeeper/trunk/bookkeeper-server/src/test/java/org/apache/bookkeeper/test/BaseTestCase.java Thu Feb 9 17:14:29 2012
@@ -36,6 +36,7 @@ import org.apache.bookkeeper.client.Book
import org.apache.bookkeeper.conf.ClientConfiguration;
import org.apache.bookkeeper.conf.ServerConfiguration;
import org.apache.bookkeeper.proto.BookieServer;
+import org.apache.bookkeeper.bookie.BookieException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.zookeeper.KeeperException;
@@ -199,7 +200,7 @@ public abstract class BaseTestCase exten
* @throws IOException
*/
protected void restartBookies()
- throws InterruptedException, IOException, KeeperException {
+ throws InterruptedException, IOException, KeeperException, BookieException {
restartBookies(null);
}
@@ -207,7 +208,7 @@ public abstract class BaseTestCase exten
* Restart bookie servers add new configuration settings
*/
protected void restartBookies(ServerConfiguration newConf)
- throws InterruptedException, IOException, KeeperException {
+ throws InterruptedException, IOException, KeeperException, BookieException {
// shut down bookie server
for (BookieServer server : bs) {
server.shutdown();
@@ -235,7 +236,7 @@ public abstract class BaseTestCase exten
* @throws IOException
*/
protected void startNewBookie(int port)
- throws IOException, InterruptedException, KeeperException {
+ throws IOException, InterruptedException, KeeperException, BookieException {
File f = File.createTempFile("bookie", "test");
tmpDirs.add(f);
f.delete();
@@ -254,7 +255,7 @@ public abstract class BaseTestCase exten
*
*/
private BookieServer startBookie(ServerConfiguration conf)
- throws IOException, InterruptedException, KeeperException {
+ throws IOException, InterruptedException, KeeperException, BookieException {
BookieServer server = new BookieServer(conf);
server.start();