You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@bookkeeper.apache.org by si...@apache.org on 2016/02/23 19:16:11 UTC
bookkeeper git commit: BOOKKEEPER-879: Record ledger creation time
Repository: bookkeeper
Updated Branches:
refs/heads/master 9c937f5d8 -> bac3ed360
BOOKKEEPER-879: Record ledger creation time
Author: eolivelli <eo...@gmail.com>
Reviewers: Sijie Guo <si...@apache.org>, Flavio Junqueira <fp...@apache.org>
Closes #2 from eolivelli/master and squashes the following commits:
509ad4f [eolivelli] BOOKKEEPER-879: Record ledger creation time
227d50c [eolivelli] BOOKKEEPER-879: Record ledger creation time
94cdee6 [eolivelli] BOOKKEEPER-879: Record ledger creation time
2a8e700 [eolivelli] BOOKKEEPER-879: Record ledger creation time
c43f0b4 [eolivelli] BOOKKEEPER-879: Record ledger creation time
e33f2e0 [eolivelli] BOOKKEEPER-879: Record ledger creation time
Project: http://git-wip-us.apache.org/repos/asf/bookkeeper/repo
Commit: http://git-wip-us.apache.org/repos/asf/bookkeeper/commit/bac3ed36
Tree: http://git-wip-us.apache.org/repos/asf/bookkeeper/tree/bac3ed36
Diff: http://git-wip-us.apache.org/repos/asf/bookkeeper/diff/bac3ed36
Branch: refs/heads/master
Commit: bac3ed3603f9dc9864beb610bea215c186ef4fab
Parents: 9c937f5
Author: eolivelli <eo...@gmail.com>
Authored: Tue Feb 23 10:16:06 2016 -0800
Committer: Sijie Guo <si...@apache.org>
Committed: Tue Feb 23 10:16:06 2016 -0800
----------------------------------------------------------------------
.../bookkeeper/client/LedgerMetadata.java | 26 +++++-
.../meta/AbstractZkLedgerManager.java | 9 +-
.../bookkeeper/meta/MSLedgerManagerFactory.java | 3 +-
.../apache/bookkeeper/proto/DataFormats.java | 86 ++++++++++++++++----
.../src/main/proto/DataFormats.proto | 2 +
.../bookkeeper/client/ListLedgersTest.java | 25 ++++++
6 files changed, 131 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/bookkeeper/blob/bac3ed36/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerMetadata.java
----------------------------------------------------------------------
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerMetadata.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerMetadata.java
index a20f34a..a58adba 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerMetadata.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/client/LedgerMetadata.java
@@ -37,6 +37,7 @@ import java.util.SortedMap;
import java.util.TreeMap;
import static com.google.common.base.Charsets.UTF_8;
+import com.google.common.base.Optional;
/**
* This class encapsulates all the ledger metadata that is persistently stored
@@ -66,6 +67,7 @@ public class LedgerMetadata {
private int ackQuorumSize;
private long length;
private long lastEntryId;
+ private long ctime;
private LedgerMetadataFormat.State state;
private SortedMap<Long, ArrayList<BookieSocketAddress>> ensembles =
@@ -82,6 +84,7 @@ public class LedgerMetadata {
this.ensembleSize = ensembleSize;
this.writeQuorumSize = writeQuorumSize;
this.ackQuorumSize = ackQuorumSize;
+ this.ctime = System.currentTimeMillis();
/*
* It is set in PendingReadOp.readEntryComplete, and
@@ -104,6 +107,7 @@ public class LedgerMetadata {
LedgerMetadata(LedgerMetadata other) {
this.ensembleSize = other.ensembleSize;
this.writeQuorumSize = other.writeQuorumSize;
+ this.ctime = other.ctime;
this.ackQuorumSize = other.ackQuorumSize;
this.length = other.length;
this.lastEntryId = other.lastEntryId;
@@ -150,6 +154,14 @@ public class LedgerMetadata {
return writeQuorumSize;
}
+ /**
+ * Get the creation timestamp of the ledger
+ * @return
+ */
+ public long getCtime() {
+ return ctime;
+ }
+
public int getAckQuorumSize() {
return ackQuorumSize;
}
@@ -255,7 +267,7 @@ public class LedgerMetadata {
LedgerMetadataFormat.Builder builder = LedgerMetadataFormat.newBuilder();
builder.setQuorumSize(writeQuorumSize).setAckQuorumSize(ackQuorumSize)
.setEnsembleSize(ensembleSize).setLength(length)
- .setState(state).setLastEntryId(lastEntryId);
+ .setState(state).setLastEntryId(lastEntryId).setCtime(ctime);
if (hasPassword) {
builder.setDigestType(digestType).setPassword(ByteString.copyFrom(password));
@@ -308,11 +320,13 @@ public class LedgerMetadata {
* byte array to parse
* @param version
* version of the ledger metadata
+ * @param msCtime
+ * metadata store creation time, used for legacy ledgers
* @return LedgerConfig
* @throws IOException
* if the given byte[] cannot be parsed
*/
- public static LedgerMetadata parseConfig(byte[] bytes, Version version) throws IOException {
+ public static LedgerMetadata parseConfig(byte[] bytes, Version version, Optional<Long> msCtime) throws IOException {
LedgerMetadata lc = new LedgerMetadata();
lc.version = version;
@@ -351,7 +365,12 @@ public class LedgerMetadata {
LedgerMetadataFormat.Builder builder = LedgerMetadataFormat.newBuilder();
TextFormat.merge(reader, builder);
LedgerMetadataFormat data = builder.build();
- lc.writeQuorumSize = data.getQuorumSize();
+ lc.writeQuorumSize = data.getQuorumSize();
+ if (data.hasCtime()) {
+ lc.ctime = data.getCtime();
+ } else if (msCtime.isPresent()) {
+ lc.ctime = msCtime.get();
+ }
if (data.hasAckQuorumSize()) {
lc.ackQuorumSize = data.getAckQuorumSize();
} else {
@@ -463,6 +482,7 @@ public class LedgerMetadata {
if (metadataFormatVersion != newMeta.metadataFormatVersion ||
ensembleSize != newMeta.ensembleSize ||
writeQuorumSize != newMeta.writeQuorumSize ||
+ ctime != newMeta.ctime ||
ackQuorumSize != newMeta.ackQuorumSize ||
length != newMeta.length ||
state != newMeta.state ||
http://git-wip-us.apache.org/repos/asf/bookkeeper/blob/bac3ed36/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/AbstractZkLedgerManager.java
----------------------------------------------------------------------
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/AbstractZkLedgerManager.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/AbstractZkLedgerManager.java
index 1d7c591..fbf38ad 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/AbstractZkLedgerManager.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/AbstractZkLedgerManager.java
@@ -17,6 +17,7 @@
*/
package org.apache.bookkeeper.meta;
+import com.google.common.base.Optional;
import java.io.IOException;
import java.util.HashSet;
import java.util.List;
@@ -355,10 +356,14 @@ abstract class AbstractZkLedgerManager implements LedgerManager, Watcher {
readCb.operationComplete(BKException.Code.ZKException, null);
return;
}
-
+ if (stat == null) {
+ LOG.error("Could not parse ledger metadata for ledger: " + ledgerId+". Stat object is null");
+ readCb.operationComplete(BKException.Code.ZKException, null);
+ return;
+ }
LedgerMetadata metadata;
try {
- metadata = LedgerMetadata.parseConfig(data, new ZkVersion(stat.getVersion()));
+ metadata = LedgerMetadata.parseConfig(data, new ZkVersion(stat.getVersion()), Optional.of(stat.getCtime()));
} catch (IOException e) {
LOG.error("Could not parse ledger metadata for ledger: " + ledgerId, e);
readCb.operationComplete(BKException.Code.ZKException, null);
http://git-wip-us.apache.org/repos/asf/bookkeeper/blob/bac3ed36/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/MSLedgerManagerFactory.java
----------------------------------------------------------------------
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/MSLedgerManagerFactory.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/MSLedgerManagerFactory.java
index 9f7ef38..8a82d41 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/MSLedgerManagerFactory.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/meta/MSLedgerManagerFactory.java
@@ -17,6 +17,7 @@
*/
package org.apache.bookkeeper.meta;
+import com.google.common.base.Optional;
import static org.apache.bookkeeper.metastore.MetastoreTable.ALL_FIELDS;
import static org.apache.bookkeeper.metastore.MetastoreTable.NON_FIELDS;
@@ -408,7 +409,7 @@ public class MSLedgerManagerFactory extends LedgerManagerFactory {
LedgerMetadata metadata;
try {
metadata = LedgerMetadata
- .parseConfig(value.getValue().getField(META_FIELD), value.getVersion());
+ .parseConfig(value.getValue().getField(META_FIELD), value.getVersion(), Optional.<Long>absent());
} catch (IOException e) {
LOG.error("Could not parse ledger metadata for ledger " + ledgerId + " : ", e);
readCb.operationComplete(BKException.Code.MetaStoreException, null);
http://git-wip-us.apache.org/repos/asf/bookkeeper/blob/bac3ed36/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/DataFormats.java
----------------------------------------------------------------------
diff --git a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/DataFormats.java b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/DataFormats.java
index 65d5444..4552144 100644
--- a/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/DataFormats.java
+++ b/bookkeeper-server/src/main/java/org/apache/bookkeeper/proto/DataFormats.java
@@ -52,6 +52,10 @@ public final class DataFormats {
// optional int32 ackQuorumSize = 9;
boolean hasAckQuorumSize();
int getAckQuorumSize();
+
+ // optional int64 ctime = 10;
+ boolean hasCtime();
+ long getCtime();
}
public static final class LedgerMetadataFormat extends
com.google.protobuf.GeneratedMessage
@@ -781,6 +785,16 @@ public final class DataFormats {
return ackQuorumSize_;
}
+ // optional int64 ctime = 10;
+ public static final int CTIME_FIELD_NUMBER = 10;
+ private long ctime_;
+ public boolean hasCtime() {
+ return ((bitField0_ & 0x00000100) == 0x00000100);
+ }
+ public long getCtime() {
+ return ctime_;
+ }
+
private void initFields() {
quorumSize_ = 0;
ensembleSize_ = 0;
@@ -791,6 +805,7 @@ public final class DataFormats {
digestType_ = org.apache.bookkeeper.proto.DataFormats.LedgerMetadataFormat.DigestType.CRC32;
password_ = com.google.protobuf.ByteString.EMPTY;
ackQuorumSize_ = 0;
+ ctime_ = 0L;
}
private byte memoizedIsInitialized = -1;
public final boolean isInitialized() {
@@ -853,6 +868,9 @@ public final class DataFormats {
if (((bitField0_ & 0x00000080) == 0x00000080)) {
output.writeInt32(9, ackQuorumSize_);
}
+ if (((bitField0_ & 0x00000100) == 0x00000100)) {
+ output.writeInt64(10, ctime_);
+ }
getUnknownFields().writeTo(output);
}
@@ -898,6 +916,10 @@ public final class DataFormats {
size += com.google.protobuf.CodedOutputStream
.computeInt32Size(9, ackQuorumSize_);
}
+ if (((bitField0_ & 0x00000100) == 0x00000100)) {
+ size += com.google.protobuf.CodedOutputStream
+ .computeInt64Size(10, ctime_);
+ }
size += getUnknownFields().getSerializedSize();
memoizedSerializedSize = size;
return size;
@@ -1045,6 +1067,8 @@ public final class DataFormats {
bitField0_ = (bitField0_ & ~0x00000080);
ackQuorumSize_ = 0;
bitField0_ = (bitField0_ & ~0x00000100);
+ ctime_ = 0L;
+ bitField0_ = (bitField0_ & ~0x00000200);
return this;
}
@@ -1124,6 +1148,10 @@ public final class DataFormats {
to_bitField0_ |= 0x00000080;
}
result.ackQuorumSize_ = ackQuorumSize_;
+ if (((from_bitField0_ & 0x00000200) == 0x00000200)) {
+ to_bitField0_ |= 0x00000100;
+ }
+ result.ctime_ = ctime_;
result.bitField0_ = to_bitField0_;
onBuilt();
return result;
@@ -1190,6 +1218,9 @@ public final class DataFormats {
if (other.hasAckQuorumSize()) {
setAckQuorumSize(other.getAckQuorumSize());
}
+ if (other.hasCtime()) {
+ setCtime(other.getCtime());
+ }
this.mergeUnknownFields(other.getUnknownFields());
return this;
}
@@ -1301,6 +1332,11 @@ public final class DataFormats {
ackQuorumSize_ = input.readInt32();
break;
}
+ case 80: {
+ bitField0_ |= 0x00000200;
+ ctime_ = input.readInt64();
+ break;
+ }
}
}
}
@@ -1670,6 +1706,27 @@ public final class DataFormats {
return this;
}
+ // optional int64 ctime = 10;
+ private long ctime_ ;
+ public boolean hasCtime() {
+ return ((bitField0_ & 0x00000200) == 0x00000200);
+ }
+ public long getCtime() {
+ return ctime_;
+ }
+ public Builder setCtime(long value) {
+ bitField0_ |= 0x00000200;
+ ctime_ = value;
+ onChanged();
+ return this;
+ }
+ public Builder clearCtime() {
+ bitField0_ = (bitField0_ & ~0x00000200);
+ ctime_ = 0L;
+ onChanged();
+ return this;
+ }
+
// @@protoc_insertion_point(builder_scope:LedgerMetadataFormat)
}
@@ -3993,7 +4050,7 @@ public final class DataFormats {
descriptor;
static {
java.lang.String[] descriptorData = {
- "\n src/main/proto/DataFormats.proto\"\262\003\n\024L" +
+ "\n src/main/proto/DataFormats.proto\"\301\003\n\024L" +
"edgerMetadataFormat\022\022\n\nquorumSize\030\001 \002(\005\022" +
"\024\n\014ensembleSize\030\002 \002(\005\022\016\n\006length\030\003 \002(\003\022\023\n" +
"\013lastEntryId\030\004 \001(\003\0220\n\005state\030\005 \002(\0162\033.Ledg" +
@@ -4001,18 +4058,19 @@ public final class DataFormats {
"\006 \003(\0132\035.LedgerMetadataFormat.Segment\0224\n\n" +
"digestType\030\007 \001(\0162 .LedgerMetadataFormat." +
"DigestType\022\020\n\010password\030\010 \001(\014\022\025\n\rackQuoru" +
- "mSize\030\t \001(\005\0327\n\007Segment\022\026\n\016ensembleMember" +
- "\030\001 \003(\t\022\024\n\014firstEntryId\030\002 \002(\003\".\n\005State\022\010\n",
- "\004OPEN\020\001\022\017\n\013IN_RECOVERY\020\002\022\n\n\006CLOSED\020\003\"!\n\n" +
- "DigestType\022\t\n\005CRC32\020\001\022\010\n\004HMAC\020\002\"@\n\037Ledge" +
- "rRereplicationLayoutFormat\022\014\n\004type\030\001 \002(\t" +
- "\022\017\n\007version\030\002 \002(\005\".\n\033UnderreplicatedLedg" +
- "erFormat\022\017\n\007replica\030\001 \003(\t\"^\n\014CookieForma" +
- "t\022\022\n\nbookieHost\030\001 \002(\t\022\022\n\njournalDir\030\002 \002(" +
- "\t\022\022\n\nledgerDirs\030\003 \002(\t\022\022\n\ninstanceId\030\004 \001(" +
- "\t\"\"\n\016LockDataFormat\022\020\n\010bookieId\030\001 \001(\t\"%\n" +
- "\021AuditorVoteFormat\022\020\n\010bookieId\030\001 \001(\tB\037\n\033" +
- "org.apache.bookkeeper.protoH\001"
+ "mSize\030\t \001(\005\022\r\n\005ctime\030\n \001(\003\0327\n\007Segment\022\026\n" +
+ "\016ensembleMember\030\001 \003(\t\022\024\n\014firstEntryId\030\002 ",
+ "\002(\003\".\n\005State\022\010\n\004OPEN\020\001\022\017\n\013IN_RECOVERY\020\002\022" +
+ "\n\n\006CLOSED\020\003\"!\n\nDigestType\022\t\n\005CRC32\020\001\022\010\n\004" +
+ "HMAC\020\002\"@\n\037LedgerRereplicationLayoutForma" +
+ "t\022\014\n\004type\030\001 \002(\t\022\017\n\007version\030\002 \002(\005\".\n\033Unde" +
+ "rreplicatedLedgerFormat\022\017\n\007replica\030\001 \003(\t" +
+ "\"^\n\014CookieFormat\022\022\n\nbookieHost\030\001 \002(\t\022\022\n\n" +
+ "journalDir\030\002 \002(\t\022\022\n\nledgerDirs\030\003 \002(\t\022\022\n\n" +
+ "instanceId\030\004 \001(\t\"\"\n\016LockDataFormat\022\020\n\010bo" +
+ "okieId\030\001 \001(\t\"%\n\021AuditorVoteFormat\022\020\n\010boo" +
+ "kieId\030\001 \001(\tB\037\n\033org.apache.bookkeeper.pro",
+ "toH\001"
};
com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner assigner =
new com.google.protobuf.Descriptors.FileDescriptor.InternalDescriptorAssigner() {
@@ -4024,7 +4082,7 @@ public final class DataFormats {
internal_static_LedgerMetadataFormat_fieldAccessorTable = new
com.google.protobuf.GeneratedMessage.FieldAccessorTable(
internal_static_LedgerMetadataFormat_descriptor,
- new java.lang.String[] { "QuorumSize", "EnsembleSize", "Length", "LastEntryId", "State", "Segment", "DigestType", "Password", "AckQuorumSize", },
+ new java.lang.String[] { "QuorumSize", "EnsembleSize", "Length", "LastEntryId", "State", "Segment", "DigestType", "Password", "AckQuorumSize", "Ctime", },
org.apache.bookkeeper.proto.DataFormats.LedgerMetadataFormat.class,
org.apache.bookkeeper.proto.DataFormats.LedgerMetadataFormat.Builder.class);
internal_static_LedgerMetadataFormat_Segment_descriptor =
http://git-wip-us.apache.org/repos/asf/bookkeeper/blob/bac3ed36/bookkeeper-server/src/main/proto/DataFormats.proto
----------------------------------------------------------------------
diff --git a/bookkeeper-server/src/main/proto/DataFormats.proto b/bookkeeper-server/src/main/proto/DataFormats.proto
index 67623ac..6d97c3a 100644
--- a/bookkeeper-server/src/main/proto/DataFormats.proto
+++ b/bookkeeper-server/src/main/proto/DataFormats.proto
@@ -48,6 +48,8 @@ message LedgerMetadataFormat {
optional bytes password = 8;
optional int32 ackQuorumSize = 9;
+
+ optional int64 ctime = 10;
}
message LedgerRereplicationLayoutFormat {
http://git-wip-us.apache.org/repos/asf/bookkeeper/blob/bac3ed36/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/ListLedgersTest.java
----------------------------------------------------------------------
diff --git a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/ListLedgersTest.java b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/ListLedgersTest.java
index d34c064..70ff6e0 100644
--- a/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/ListLedgersTest.java
+++ b/bookkeeper-server/src/test/java/org/apache/bookkeeper/client/ListLedgersTest.java
@@ -108,4 +108,29 @@ public class ListLedgersTest extends BaseTestCase {
Assert.fail("Remove is not supported, we shouln't have reached this point");
}
+
+ @Test(timeout = 60000)
+ public void testCtimeRecorded()
+ throws Exception {
+
+ ClientConfiguration conf = new ClientConfiguration()
+ .setZkServers(zkUtil.getZooKeeperConnectString());
+
+ BookKeeper bkc = new BookKeeper(conf);
+
+ bkc.createLedger(digestType, "testPasswd".
+ getBytes()).close();
+
+ BookKeeperAdmin admin = new BookKeeperAdmin(zkUtil.
+ getZooKeeperConnectString());
+ Iterable<Long> iterable = admin.listLedgers();
+
+ for (Long lId : iterable) {
+ LedgerHandle ledger = bkc.openLedger(lId, digestType, "testPasswd".getBytes());
+ LedgerMetadata metaData = ledger.getLedgerMetadata();
+ Assert.assertTrue("ctime was not recorded", metaData.getCtime() > 0);
+ }
+
+ }
+
}