You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@zookeeper.apache.org by th...@apache.org on 2013/06/28 23:22:36 UTC
svn commit: r1497929 [2/2] - in /zookeeper/trunk: ./
src/java/main/org/apache/zookeeper/server/
src/java/main/org/apache/zookeeper/server/persistence/
src/java/main/org/apache/zookeeper/server/quorum/
src/java/test/org/apache/zookeeper/server/quorum/ s...
Added: zookeeper/trunk/src/java/test/org/apache/zookeeper/test/GetProposalFromTxnTest.java
URL: http://svn.apache.org/viewvc/zookeeper/trunk/src/java/test/org/apache/zookeeper/test/GetProposalFromTxnTest.java?rev=1497929&view=auto
==============================================================================
--- zookeeper/trunk/src/java/test/org/apache/zookeeper/test/GetProposalFromTxnTest.java (added)
+++ zookeeper/trunk/src/java/test/org/apache/zookeeper/test/GetProposalFromTxnTest.java Fri Jun 28 21:22:35 2013
@@ -0,0 +1,144 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.zookeeper.test;
+
+import java.io.File;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.Iterator;
+
+import org.apache.jute.Record;
+import org.apache.log4j.Logger;
+import org.apache.zookeeper.CreateMode;
+import org.apache.zookeeper.PortAssignment;
+import org.apache.zookeeper.WatchedEvent;
+import org.apache.zookeeper.Watcher;
+import org.apache.zookeeper.ZKTestCase;
+import org.apache.zookeeper.ZooDefs.Ids;
+import org.apache.zookeeper.ZooDefs.OpCode;
+import org.apache.zookeeper.ZooKeeper;
+import org.apache.zookeeper.data.Stat;
+import org.apache.zookeeper.server.ServerCnxnFactory;
+import org.apache.zookeeper.server.SyncRequestProcessor;
+import org.apache.zookeeper.server.ZKDatabase;
+import org.apache.zookeeper.server.ZooKeeperServer;
+import org.apache.zookeeper.server.quorum.Leader.Proposal;
+import org.apache.zookeeper.server.util.SerializeUtils;
+import org.apache.zookeeper.txn.TxnHeader;
+import org.junit.Assert;
+import org.junit.Test;
+
+/**
+ * Test loading committed proposal from txnlog. Learner uses these proposals to
+ * catch-up with leader
+ */
+public class GetProposalFromTxnTest extends ZKTestCase implements Watcher {
+ private static final Logger LOG = Logger
+ .getLogger(GetProposalFromTxnTest.class);
+ private static String HOSTPORT = "127.0.0.1:" + PortAssignment.unique();
+ private static final int CONNECTION_TIMEOUT = 3000;
+
+ private static final int MSG_COUNT = 2000;
+
+ /**
+ * Test loading proposal from txnlog
+ *
+ * @throws Exception
+ * an exception might be thrown here
+ */
+ @Test
+ public void testGetProposalFromTxn() throws Exception {
+ File tmpDir = ClientBase.createTmpDir();
+ ClientBase.setupTestEnv();
+ ZooKeeperServer zks = new ZooKeeperServer(tmpDir, tmpDir, 3000);
+ SyncRequestProcessor.setSnapCount(100);
+ final int PORT = Integer.parseInt(HOSTPORT.split(":")[1]);
+ ServerCnxnFactory f = ServerCnxnFactory.createFactory(PORT, -1);
+ f.startup(zks);
+ Assert.assertTrue("waiting for server being up ",
+ ClientBase.waitForServerUp(HOSTPORT, CONNECTION_TIMEOUT));
+ ZooKeeper zk = new ZooKeeper(HOSTPORT, CONNECTION_TIMEOUT, this);
+
+ // Generate transaction so we will have some txnlog
+ Long[] zxids = new Long[MSG_COUNT];
+ try {
+ String data = "data";
+ byte[] bytes = data.getBytes();
+ for (int i = 0; i < MSG_COUNT; i++) {
+ Stat stat = new Stat();
+ zk.create("/invalidsnap-" + i, bytes, Ids.OPEN_ACL_UNSAFE,
+ CreateMode.PERSISTENT);
+ zk.getData("/invalidsnap-" + i, null, stat);
+ zxids[i] = stat.getCzxid();
+ }
+
+ } finally {
+ zk.close();
+ }
+
+ // shutdown and start zookeeper again
+ f.shutdown();
+ Assert.assertTrue("waiting for server to shutdown",
+ ClientBase.waitForServerDown(HOSTPORT, CONNECTION_TIMEOUT));
+ zks = new ZooKeeperServer(tmpDir, tmpDir, 3000);
+ zks.startdata();
+
+ ZKDatabase db = zks.getZKDatabase();
+
+ // Set sizeLimit to be very high number, so we can pull all transactions
+ // from txnlog
+ Iterator<Proposal> itr = db.getProposalsFromTxnLog(zxids[0], 10000000);
+
+ int createCount = 0;
+ ArrayList<Long> retrievedZxids = new ArrayList<Long>(MSG_COUNT);
+
+ // Get zxid of create requests
+ while (itr.hasNext()) {
+ Proposal proposal = itr.next();
+ TxnHeader hdr = new TxnHeader();
+ Record rec = SerializeUtils.deserializeTxn(
+ proposal.packet.getData(), hdr);
+ if (hdr.getType() == OpCode.create) {
+ retrievedZxids.add(hdr.getZxid());
+ createCount++;
+ }
+ }
+
+ // All zxid should match what we created
+ Assert.assertTrue("Zxids missmatches",
+ Arrays.equals(zxids, retrievedZxids.toArray(new Long[0])));
+
+ // There should be 2000 create requests
+ Assert.assertTrue("create proposal count == " + MSG_COUNT,
+ (createCount == MSG_COUNT));
+
+ // We are requesting half the number of transaction from the snapshot
+ // this should exceed threshold (ZKDatabase.snapshotSizeFactor)
+ db.setSnapshotSizeFactor(0.33);
+ long sizeLimit = db.calculateTxnLogSizeLimit();
+
+ itr = db.getProposalsFromTxnLog(zxids[MSG_COUNT / 2], sizeLimit);
+ Assert.assertFalse("Expect empty proposal", (itr.hasNext()));
+ }
+
+ public void process(WatchedEvent event) {
+ // do nothing
+ }
+
+}
Propchange: zookeeper/trunk/src/java/test/org/apache/zookeeper/test/GetProposalFromTxnTest.java
------------------------------------------------------------------------------
svn:eol-style = native
Modified: zookeeper/trunk/src/java/test/org/apache/zookeeper/test/LoadFromLogTest.java
URL: http://svn.apache.org/viewvc/zookeeper/trunk/src/java/test/org/apache/zookeeper/test/LoadFromLogTest.java?rev=1497929&r1=1497928&r2=1497929&view=diff
==============================================================================
--- zookeeper/trunk/src/java/test/org/apache/zookeeper/test/LoadFromLogTest.java (original)
+++ zookeeper/trunk/src/java/test/org/apache/zookeeper/test/LoadFromLogTest.java Fri Jun 28 21:22:35 2013
@@ -46,6 +46,8 @@ import org.apache.zookeeper.server.ZooKe
import org.apache.zookeeper.server.persistence.FileHeader;
import org.apache.zookeeper.server.persistence.FileTxnLog;
import org.apache.zookeeper.server.persistence.FileTxnSnapLog;
+import org.apache.zookeeper.server.persistence.Util;
+import org.apache.zookeeper.server.persistence.FileTxnLog.FileTxnIterator;
import org.apache.zookeeper.server.persistence.TxnLog.TxnIterator;
import org.apache.zookeeper.txn.CreateTxn;
import org.apache.zookeeper.txn.DeleteTxn;
@@ -104,6 +106,14 @@ public class LoadFromLogTest extends ZKT
FileTxnLog txnLog = new FileTxnLog(logDir);
TxnIterator itr = txnLog.read(0);
+
+ // Check that storage space return some value
+ FileTxnIterator fileItr = (FileTxnIterator) itr;
+ long storageSize = fileItr.getStorageSize();
+ LOG.info("Txnlog size: " + storageSize + " bytes");
+ Assert.assertTrue("Storage size is greater than zero ",
+ (storageSize > 0));
+
long expectedZxid = 0;
long lastZxid = 0;
TxnHeader hdr;
@@ -118,8 +128,79 @@ public class LoadFromLogTest extends ZKT
Assert.assertTrue("processed all transactions. " + expectedZxid + " == " + TOTAL_TRANSACTIONS, (expectedZxid == TOTAL_TRANSACTIONS));
}
+ /**
+ * test that we fail to load txnlog of a request zxid that is older
+ * than what exist on disk
+ * @throws Exception an exception might be thrown here
+ */
+ @Test
+ public void testLoadFailure() throws Exception {
+ // setup a single server cluster
+ File tmpDir = ClientBase.createTmpDir();
+ ClientBase.setupTestEnv();
+ ZooKeeperServer zks = new ZooKeeperServer(tmpDir, tmpDir, 3000);
+ // So we have at least 4 logs
+ SyncRequestProcessor.setSnapCount(50);
+ final int PORT = Integer.parseInt(HOSTPORT.split(":")[1]);
+ ServerCnxnFactory f = ServerCnxnFactory.createFactory(PORT, -1);
+ f.startup(zks);
+ Assert.assertTrue("waiting for server being up ",
+ ClientBase.waitForServerUp(HOSTPORT,CONNECTION_TIMEOUT));
+ ZooKeeper zk = new ZooKeeper(HOSTPORT, CONNECTION_TIMEOUT, this);
+ // generate some transactions that will get logged
+ try {
+ for (int i = 0; i< NUM_MESSAGES; i++) {
+ zk.create("/data-", new byte[0], Ids.OPEN_ACL_UNSAFE,
+ CreateMode.PERSISTENT_SEQUENTIAL);
+ }
+ } finally {
+ zk.close();
+ }
+ f.shutdown();
+ Assert.assertTrue("waiting for server to shutdown",
+ ClientBase.waitForServerDown(HOSTPORT, CONNECTION_TIMEOUT));
+ File logDir = new File(tmpDir, FileTxnSnapLog.version + FileTxnSnapLog.VERSION);
+ File[] logFiles = FileTxnLog.getLogFiles(logDir.listFiles(), 0);
+ // Verify that we have at least 4 txnlog
+ Assert.assertTrue(logFiles.length > 4);
+ // Delete the first log file, so we will fail to read it back from disk
+ Assert.assertTrue("delete the first log file", logFiles[0].delete());
+
+ // Find zxid for the second log
+ long secondStartZxid = Util.getZxidFromName(logFiles[1].getName(), "log");
+
+ FileTxnLog txnLog = new FileTxnLog(logDir);
+ TxnIterator itr = txnLog.read(1, false);
+
+ // Oldest log is already remove, so this should point to the start of
+ // of zxid on the second log
+ Assert.assertEquals(secondStartZxid, itr.getHeader().getZxid());
+
+ itr = txnLog.read(secondStartZxid, false);
+ Assert.assertEquals(secondStartZxid, itr.getHeader().getZxid());
+ Assert.assertTrue(itr.next());
+
+ // Trying to get a second txn on second txnlog give us the
+ // the start of second log, since the first one is removed
+ long nextZxid = itr.getHeader().getZxid();
+
+ itr = txnLog.read(nextZxid, false);
+ Assert.assertEquals(secondStartZxid, itr.getHeader().getZxid());
+
+ // Trying to get a first txn on the third give us the
+ // the start of second log, since the first one is removed
+ long thirdStartZxid = Util.getZxidFromName(logFiles[2].getName(), "log");
+ itr = txnLog.read(thirdStartZxid, false);
+ Assert.assertEquals(secondStartZxid, itr.getHeader().getZxid());
+ Assert.assertTrue(itr.next());
+
+ nextZxid = itr.getHeader().getZxid();
+ itr = txnLog.read(nextZxid, false);
+ Assert.assertEquals(secondStartZxid, itr.getHeader().getZxid());
+
+ }
public void process(WatchedEvent event) {
switch (event.getType()) {