You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pulsar.apache.org by GitBox <gi...@apache.org> on 2018/12/19 14:35:01 UTC
[GitHub] ivankelly closed pull request #2808: [WIP] debugging test failure
in CI - do not submit
ivankelly closed pull request #2808: [WIP] debugging test failure in CI - do not submit
URL: https://github.com/apache/pulsar/pull/2808
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/buildtools/src/main/java/org/apache/pulsar/tests/RetryAnalyzer.java b/buildtools/src/main/java/org/apache/pulsar/tests/RetryAnalyzer.java
index d94e1e25a5..b9f041b61f 100644
--- a/buildtools/src/main/java/org/apache/pulsar/tests/RetryAnalyzer.java
+++ b/buildtools/src/main/java/org/apache/pulsar/tests/RetryAnalyzer.java
@@ -26,11 +26,11 @@
private int count = 0;
// Only try again once
- private static final int MAX_RETRIES = Integer.valueOf(System.getProperty("testRetryCount", "1"));
+ private static final int MAX_RETRIES = Integer.valueOf(System.getProperty("testRetryCount", "0"));
@Override
public boolean retry(ITestResult result) {
- return count++ < MAX_RETRIES;
+ return false;//count++ < MAX_RETRIES;
}
}
diff --git a/buildtools/src/main/resources/log4j2.xml b/buildtools/src/main/resources/log4j2.xml
index 2fdc2d05ae..f61eb80ce1 100644
--- a/buildtools/src/main/resources/log4j2.xml
+++ b/buildtools/src/main/resources/log4j2.xml
@@ -29,8 +29,14 @@
<Root level="warn">
<AppenderRef ref="Console" />
</Root>
- <Logger name="org.eclipse.jetty" level="info"/>
+ <Logger name="org.eclipse.jetty.io" level="debug"/>
+ <Logger name="javax.servlet" level="debug"/>
+ <Logger name="org.eclipse.jetty.client" level="debug"/>
+ <Logger name="org.eclipse.jetty.server" level="debug"/>
+ <Logger name="org.eclipse.jetty.servlet" level="debug"/>
+ <Logger name="org.eclipse.jetty.proxy" level="debug"/>
<Logger name="org.apache.pulsar" level="info"/>
+ <Logger name="org.apache.pulsar.proxy" level="debug"/>
<Logger name="org.apache.bookkeeper" level="info"/>
<Logger name="org.apache.kafka" level="info"/>
</Loggers>
diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/client/BookKeeperTestClient.java b/managed-ledger/src/test/java/org/apache/bookkeeper/client/BookKeeperTestClient.java
deleted file mode 100644
index 184fc7f4c0..0000000000
--- a/managed-ledger/src/test/java/org/apache/bookkeeper/client/BookKeeperTestClient.java
+++ /dev/null
@@ -1,51 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.bookkeeper.client;
-
-import java.io.IOException;
-import org.apache.bookkeeper.conf.ClientConfiguration;
-import org.apache.zookeeper.KeeperException;
-import org.apache.zookeeper.ZooKeeper;
-
-/**
- * Test BookKeeperClient which allows access to members we don't wish to expose in the public API.
- */
-public class BookKeeperTestClient extends BookKeeper {
- public BookKeeperTestClient(ClientConfiguration conf) throws IOException, InterruptedException, BKException {
- super(conf);
- }
-
- public ZooKeeper getZkHandle() {
- return super.getZkHandle();
- }
-
- public ClientConfiguration getConf() {
- return super.getConf();
- }
-
- /**
- * Force a read to zookeeper to get list of bookies.
- *
- * @throws InterruptedException
- * @throws KeeperException
- */
- public void readBookiesBlocking() throws InterruptedException, BKException {
- bookieWatcher.initialBlockingBookieRead();
- }
-}
diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/SimpleBookKeeperTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/SimpleBookKeeperTest.java
deleted file mode 100644
index c1b4894c73..0000000000
--- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/SimpleBookKeeperTest.java
+++ /dev/null
@@ -1,65 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.bookkeeper.mledger;
-
-import com.google.common.base.Charsets;
-import java.nio.charset.Charset;
-import java.util.Enumeration;
-import org.apache.bookkeeper.client.BookKeeper.DigestType;
-import org.apache.bookkeeper.client.LedgerEntry;
-import org.apache.bookkeeper.client.LedgerHandle;
-import org.apache.bookkeeper.test.MockedBookKeeperTestCase;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.annotations.Test;
-
-public class SimpleBookKeeperTest extends MockedBookKeeperTestCase {
-
- private static final String SECRET = "secret";
- private static final Charset Encoding = Charsets.UTF_8;
-
- @Test
- public void simpleTest() throws Exception {
-
- LedgerHandle ledger = bkc.createLedger(DigestType.MAC, SECRET.getBytes());
- long ledgerId = ledger.getId();
- log.info("Writing to ledger: {}", ledgerId);
-
- for (int i = 0; i < 10; i++) {
- String content = "entry-" + i;
- ledger.addEntry(content.getBytes(Encoding));
- }
-
- ledger.close();
-
- ledger = bkc.openLedger(ledgerId, DigestType.MAC, SECRET.getBytes());
-
- Enumeration<LedgerEntry> entries = ledger.readEntries(0, 9);
- while (entries.hasMoreElements()) {
- LedgerEntry entry = entries.nextElement();
- String content = new String(entry.getEntry(), Encoding);
- log.info("Entry {} lenght={} content='{}'", entry.getEntryId(), entry.getLength(), content);
- }
-
- ledger.close();
- }
-
- private static Logger log = LoggerFactory.getLogger(SimpleBookKeeperTest.class);
-
-}
diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/EntryCacheManagerTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/EntryCacheManagerTest.java
deleted file mode 100644
index 47c515d137..0000000000
--- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/EntryCacheManagerTest.java
+++ /dev/null
@@ -1,272 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.bookkeeper.mledger.impl;
-
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-import static org.testng.Assert.assertEquals;
-import static org.testng.Assert.assertTrue;
-
-import java.util.List;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.bookkeeper.common.util.OrderedScheduler;
-import org.apache.bookkeeper.mledger.Entry;
-import org.apache.bookkeeper.mledger.ManagedLedgerFactoryConfig;
-import org.apache.bookkeeper.test.MockedBookKeeperTestCase;
-import org.testng.annotations.BeforeClass;
-import org.testng.annotations.Test;
-
-@Test
-public class EntryCacheManagerTest extends MockedBookKeeperTestCase {
-
- ManagedLedgerImpl ml1;
- ManagedLedgerImpl ml2;
-
- @BeforeClass
- void setup() throws Exception {
- OrderedScheduler executor = OrderedScheduler.newSchedulerBuilder().numThreads(1).build();
-
- ml1 = mock(ManagedLedgerImpl.class);
- when(ml1.getScheduledExecutor()).thenReturn(executor);
- when(ml1.getName()).thenReturn("cache1");
-
- ml2 = mock(ManagedLedgerImpl.class);
- when(ml2.getScheduledExecutor()).thenReturn(executor);
- when(ml2.getName()).thenReturn("cache2");
- }
-
- @Test
- void simple() throws Exception {
- ManagedLedgerFactoryConfig config = new ManagedLedgerFactoryConfig();
- config.setMaxCacheSize(10);
- config.setCacheEvictionWatermark(0.8);
-
- factory = new ManagedLedgerFactoryImpl(bkc, bkc.getZkHandle(), config);
-
- EntryCacheManager cacheManager = factory.getEntryCacheManager();
- EntryCache cache1 = cacheManager.getEntryCache(ml1);
- EntryCache cache2 = cacheManager.getEntryCache(ml2);
-
- cache1.insert(EntryImpl.create(1, 1, new byte[4]));
- cache1.insert(EntryImpl.create(1, 0, new byte[3]));
-
- assertEquals(cache1.getSize(), 7);
- assertEquals(cacheManager.getSize(), 7);
-
- cacheManager.mlFactoryMBean.refreshStats(1, TimeUnit.SECONDS);
- assertEquals(cacheManager.mlFactoryMBean.getCacheMaxSize(), 10);
- assertEquals(cacheManager.mlFactoryMBean.getCacheUsedSize(), 7);
- assertEquals(cacheManager.mlFactoryMBean.getCacheHitsRate(), 0.0);
- assertEquals(cacheManager.mlFactoryMBean.getCacheMissesRate(), 0.0);
- assertEquals(cacheManager.mlFactoryMBean.getCacheHitsThroughput(), 0.0);
- assertEquals(cacheManager.mlFactoryMBean.getNumberOfCacheEvictions(), 0);
-
- cache2.insert(EntryImpl.create(2, 0, new byte[1]));
- cache2.insert(EntryImpl.create(2, 1, new byte[1]));
- cache2.insert(EntryImpl.create(2, 2, new byte[1]));
-
- assertEquals(cache2.getSize(), 3);
- assertEquals(cacheManager.getSize(), 10);
-
- // Next insert should trigger a cache eviction to force the size to 8
- // The algorithm should evict entries from cache1
- cache2.insert(EntryImpl.create(2, 3, new byte[1]));
-
- // Wait for eviction to be completed in background
- Thread.sleep(100);
- assertEquals(cacheManager.getSize(), 7);
- assertEquals(cache1.getSize(), 4);
- assertEquals(cache2.getSize(), 3);
-
- cacheManager.removeEntryCache("cache1");
- assertEquals(cacheManager.getSize(), 3);
- assertEquals(cache2.getSize(), 3);
-
- // Should remove 2 entries
- cache2.invalidateEntries(new PositionImpl(2, 1));
- assertEquals(cacheManager.getSize(), 1);
- assertEquals(cache2.getSize(), 1);
-
- cacheManager.mlFactoryMBean.refreshStats(1, TimeUnit.SECONDS);
-
- assertEquals(cacheManager.mlFactoryMBean.getCacheMaxSize(), 10);
- assertEquals(cacheManager.mlFactoryMBean.getCacheUsedSize(), 1);
- assertEquals(cacheManager.mlFactoryMBean.getCacheHitsRate(), 0.0);
- assertEquals(cacheManager.mlFactoryMBean.getCacheMissesRate(), 0.0);
- assertEquals(cacheManager.mlFactoryMBean.getCacheHitsThroughput(), 0.0);
- assertEquals(cacheManager.mlFactoryMBean.getNumberOfCacheEvictions(), 1);
- }
-
- @Test
- void doubleInsert() throws Exception {
- ManagedLedgerFactoryConfig config = new ManagedLedgerFactoryConfig();
- config.setMaxCacheSize(10);
- config.setCacheEvictionWatermark(0.8);
-
- factory = new ManagedLedgerFactoryImpl(bkc, bkc.getZkHandle(), config);
-
- EntryCacheManager cacheManager = factory.getEntryCacheManager();
- EntryCache cache1 = cacheManager.getEntryCache(ml1);
-
- assertEquals(cache1.insert(EntryImpl.create(1, 1, new byte[4])), true);
- assertEquals(cache1.insert(EntryImpl.create(1, 0, new byte[3])), true);
-
- assertEquals(cache1.getSize(), 7);
- assertEquals(cacheManager.getSize(), 7);
-
- assertEquals(cache1.insert(EntryImpl.create(1, 0, new byte[5])), false);
-
- assertEquals(cache1.getSize(), 7);
- assertEquals(cacheManager.getSize(), 7);
- }
-
- @Test
- void cacheDisabled() throws Exception {
- ManagedLedgerFactoryConfig config = new ManagedLedgerFactoryConfig();
- config.setMaxCacheSize(0);
- config.setCacheEvictionWatermark(0.8);
-
- factory = new ManagedLedgerFactoryImpl(bkc, bkc.getZkHandle(), config);
-
- EntryCacheManager cacheManager = factory.getEntryCacheManager();
- EntryCache cache1 = cacheManager.getEntryCache(ml1);
- EntryCache cache2 = cacheManager.getEntryCache(ml2);
-
- assertTrue(cache1 instanceof EntryCacheManager.EntryCacheDisabled);
- assertTrue(cache2 instanceof EntryCacheManager.EntryCacheDisabled);
-
- cache1.insert(EntryImpl.create(1, 1, new byte[4]));
- cache1.insert(EntryImpl.create(1, 0, new byte[3]));
-
- assertEquals(cache1.getSize(), 0);
- assertEquals(cacheManager.getSize(), 0);
-
- cacheManager.mlFactoryMBean.refreshStats(1, TimeUnit.SECONDS);
- assertEquals(cacheManager.mlFactoryMBean.getCacheMaxSize(), 0);
- assertEquals(cacheManager.mlFactoryMBean.getCacheUsedSize(), 0);
- assertEquals(cacheManager.mlFactoryMBean.getCacheHitsRate(), 0.0);
- assertEquals(cacheManager.mlFactoryMBean.getCacheMissesRate(), 0.0);
- assertEquals(cacheManager.mlFactoryMBean.getCacheHitsThroughput(), 0.0);
- assertEquals(cacheManager.mlFactoryMBean.getNumberOfCacheEvictions(), 0);
-
- cache2.insert(EntryImpl.create(2, 0, new byte[1]));
- cache2.insert(EntryImpl.create(2, 1, new byte[1]));
- cache2.insert(EntryImpl.create(2, 2, new byte[1]));
-
- assertEquals(cache2.getSize(), 0);
- assertEquals(cacheManager.getSize(), 0);
- }
-
- @Test
- void verifyNoCacheIfNoConsumer() throws Exception {
- ManagedLedgerFactoryConfig config = new ManagedLedgerFactoryConfig();
- config.setMaxCacheSize(7 * 10);
- config.setCacheEvictionWatermark(0.8);
-
- factory = new ManagedLedgerFactoryImpl(bkc, bkc.getZkHandle(), config);
-
- EntryCacheManager cacheManager = factory.getEntryCacheManager();
- ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("ledger");
- EntryCache cache1 = ledger.entryCache;
-
- for (int i = 0; i < 10; i++) {
- ledger.addEntry(("entry-" + i).getBytes());
- }
-
- assertEquals(cache1.getSize(), 0);
- assertEquals(cacheManager.getSize(), 0);
-
- cacheManager.mlFactoryMBean.refreshStats(1, TimeUnit.SECONDS);
- assertEquals(cacheManager.mlFactoryMBean.getCacheMaxSize(), 7 * 10);
- assertEquals(cacheManager.mlFactoryMBean.getCacheUsedSize(), 0);
- assertEquals(cacheManager.mlFactoryMBean.getCacheHitsRate(), 0.0);
- assertEquals(cacheManager.mlFactoryMBean.getCacheMissesRate(), 0.0);
- assertEquals(cacheManager.mlFactoryMBean.getCacheHitsThroughput(), 0.0);
- assertEquals(cacheManager.mlFactoryMBean.getNumberOfCacheEvictions(), 0);
- }
-
- @Test
- void verifyHitsMisses() throws Exception {
- ManagedLedgerFactoryConfig config = new ManagedLedgerFactoryConfig();
- config.setMaxCacheSize(7 * 10);
- config.setCacheEvictionWatermark(0.8);
-
- factory = new ManagedLedgerFactoryImpl(bkc, bkc.getZkHandle(), config);
-
- EntryCacheManager cacheManager = factory.getEntryCacheManager();
- ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("ledger");
-
- ManagedCursorImpl c1 = (ManagedCursorImpl) ledger.openCursor("c1");
- ManagedCursorImpl c2 = (ManagedCursorImpl) ledger.openCursor("c2");
-
- for (int i = 0; i < 10; i++) {
- ledger.addEntry(("entry-" + i).getBytes());
- }
-
- cacheManager.mlFactoryMBean.refreshStats(1, TimeUnit.SECONDS);
- assertEquals(cacheManager.mlFactoryMBean.getCacheUsedSize(), 70);
- assertEquals(cacheManager.mlFactoryMBean.getCacheHitsRate(), 0.0);
- assertEquals(cacheManager.mlFactoryMBean.getCacheMissesRate(), 0.0);
- assertEquals(cacheManager.mlFactoryMBean.getCacheHitsThroughput(), 0.0);
- assertEquals(cacheManager.mlFactoryMBean.getNumberOfCacheEvictions(), 0);
-
- List<Entry> entries = c1.readEntries(10);
- assertEquals(entries.size(), 10);
- entries.forEach(e -> e.release());
-
- cacheManager.mlFactoryMBean.refreshStats(1, TimeUnit.SECONDS);
- assertEquals(cacheManager.mlFactoryMBean.getCacheUsedSize(), 70);
- assertEquals(cacheManager.mlFactoryMBean.getCacheHitsRate(), 10.0);
- assertEquals(cacheManager.mlFactoryMBean.getCacheMissesRate(), 0.0);
- assertEquals(cacheManager.mlFactoryMBean.getCacheHitsThroughput(), 70.0);
- assertEquals(cacheManager.mlFactoryMBean.getNumberOfCacheEvictions(), 0);
-
- ledger.deactivateCursor(c1);
-
- cacheManager.mlFactoryMBean.refreshStats(1, TimeUnit.SECONDS);
- assertEquals(cacheManager.mlFactoryMBean.getCacheUsedSize(), 70);
- assertEquals(cacheManager.mlFactoryMBean.getCacheHitsRate(), 0.0);
- assertEquals(cacheManager.mlFactoryMBean.getCacheMissesRate(), 0.0);
- assertEquals(cacheManager.mlFactoryMBean.getCacheHitsThroughput(), 0.0);
- assertEquals(cacheManager.mlFactoryMBean.getNumberOfCacheEvictions(), 0);
-
- entries = c2.readEntries(10);
- assertEquals(entries.size(), 10);
-
- cacheManager.mlFactoryMBean.refreshStats(1, TimeUnit.SECONDS);
- assertEquals(cacheManager.mlFactoryMBean.getCacheUsedSize(), 70);
- assertEquals(cacheManager.mlFactoryMBean.getCacheHitsRate(), 10.0);
- assertEquals(cacheManager.mlFactoryMBean.getCacheMissesRate(), 0.0);
- assertEquals(cacheManager.mlFactoryMBean.getCacheHitsThroughput(), 70.0);
- assertEquals(cacheManager.mlFactoryMBean.getNumberOfCacheEvictions(), 0);
-
- PositionImpl pos = (PositionImpl) entries.get(entries.size() - 1).getPosition();
- c2.setReadPosition(pos);
- ledger.discardEntriesFromCache(c2, pos);
- entries.forEach(e -> e.release());
-
- cacheManager.mlFactoryMBean.refreshStats(1, TimeUnit.SECONDS);
- assertEquals(cacheManager.mlFactoryMBean.getCacheUsedSize(), 0);
- assertEquals(cacheManager.mlFactoryMBean.getCacheHitsRate(), 0.0);
- assertEquals(cacheManager.mlFactoryMBean.getCacheMissesRate(), 0.0);
- assertEquals(cacheManager.mlFactoryMBean.getCacheHitsThroughput(), 0.0);
- assertEquals(cacheManager.mlFactoryMBean.getNumberOfCacheEvictions(), 0);
- }
-}
diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/EntryCacheTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/EntryCacheTest.java
deleted file mode 100644
index c48ea24d88..0000000000
--- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/EntryCacheTest.java
+++ /dev/null
@@ -1,262 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.bookkeeper.mledger.impl;
-
-import static org.mockito.Matchers.any;
-import static org.mockito.Matchers.anyLong;
-import static org.mockito.Mockito.*;
-import static org.testng.Assert.assertEquals;
-
-import io.netty.buffer.Unpooled;
-import java.lang.reflect.Method;
-import java.util.List;
-import java.util.Vector;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.CountDownLatch;
-import org.apache.bookkeeper.client.BKException.BKNoSuchLedgerExistsException;
-import org.apache.bookkeeper.client.api.BKException;
-import org.apache.bookkeeper.client.api.LedgerEntries;
-import org.apache.bookkeeper.client.api.LedgerEntry;
-import org.apache.bookkeeper.client.api.ReadHandle;
-import org.apache.bookkeeper.mledger.AsyncCallbacks.ReadEntriesCallback;
-import org.apache.bookkeeper.mledger.Entry;
-import org.apache.bookkeeper.mledger.ManagedLedgerException;
-import org.apache.bookkeeper.test.MockedBookKeeperTestCase;
-import org.mockito.Mockito;
-import org.mockito.invocation.InvocationOnMock;
-import org.mockito.stubbing.Answer;
-import org.testng.Assert;
-import org.testng.annotations.BeforeMethod;
-import org.testng.annotations.Test;
-
-@Test
-public class EntryCacheTest extends MockedBookKeeperTestCase {
-
- private ManagedLedgerImpl ml;
-
- @BeforeMethod
- public void setUp(Method method) throws Exception {
- super.setUp(method);
- ml = mock(ManagedLedgerImpl.class);
- when(ml.getName()).thenReturn("name");
- when(ml.getExecutor()).thenReturn(executor);
- when(ml.getMBean()).thenReturn(new ManagedLedgerMBeanImpl(ml));
- }
-
- @Test(timeOut = 5000)
- void testRead() throws Exception {
- ReadHandle lh = getLedgerHandle();
- when(lh.getId()).thenReturn((long) 0);
-
- EntryCacheManager cacheManager = factory.getEntryCacheManager();
- EntryCache entryCache = cacheManager.getEntryCache(ml);
-
- byte[] data = new byte[10];
- for (int i = 0; i < 10; i++) {
- entryCache.insert(EntryImpl.create(0, i, data));
- }
-
- final CountDownLatch counter = new CountDownLatch(1);
-
- entryCache.asyncReadEntry(lh, 0, 9, false, new ReadEntriesCallback() {
- public void readEntriesComplete(List<Entry> entries, Object ctx) {
- assertEquals(entries.size(), 10);
- entries.forEach(e -> e.release());
- counter.countDown();
- }
-
- public void readEntriesFailed(ManagedLedgerException exception, Object ctx) {
- Assert.fail("should not have failed");
- }
- }, null);
- counter.await();
-
- // Verify no entries were read from bookkeeper
- verify(lh, never()).readAsync(anyLong(), anyLong());
- }
-
- @Test(timeOut = 5000)
- void testReadMissingBefore() throws Exception {
- ReadHandle lh = getLedgerHandle();
- when(lh.getId()).thenReturn((long) 0);
-
- EntryCacheManager cacheManager = factory.getEntryCacheManager();
- EntryCache entryCache = cacheManager.getEntryCache(ml);
-
- byte[] data = new byte[10];
- for (int i = 3; i < 10; i++) {
- entryCache.insert(EntryImpl.create(0, i, data));
- }
-
- final CountDownLatch counter = new CountDownLatch(1);
-
- entryCache.asyncReadEntry(lh, 0, 9, false, new ReadEntriesCallback() {
- public void readEntriesComplete(List<Entry> entries, Object ctx) {
- assertEquals(entries.size(), 10);
- counter.countDown();
- }
-
- public void readEntriesFailed(ManagedLedgerException exception, Object ctx) {
- Assert.fail("should not have failed");
- }
- }, null);
- counter.await();
- }
-
- @Test(timeOut = 5000)
- void testReadMissingAfter() throws Exception {
- ReadHandle lh = getLedgerHandle();
- when(lh.getId()).thenReturn((long) 0);
-
- EntryCacheManager cacheManager = factory.getEntryCacheManager();
- EntryCache entryCache = cacheManager.getEntryCache(ml);
-
- byte[] data = new byte[10];
- for (int i = 0; i < 8; i++) {
- entryCache.insert(EntryImpl.create(0, i, data));
- }
-
- final CountDownLatch counter = new CountDownLatch(1);
-
- entryCache.asyncReadEntry(lh, 0, 9, false, new ReadEntriesCallback() {
- public void readEntriesComplete(List<Entry> entries, Object ctx) {
- assertEquals(entries.size(), 10);
- counter.countDown();
- }
-
- public void readEntriesFailed(ManagedLedgerException exception, Object ctx) {
- Assert.fail("should not have failed");
- }
- }, null);
- counter.await();
- }
-
- @Test(timeOut = 5000)
- void testReadMissingMiddle() throws Exception {
- ReadHandle lh = getLedgerHandle();
- when(lh.getId()).thenReturn((long) 0);
-
- EntryCacheManager cacheManager = factory.getEntryCacheManager();
- EntryCache entryCache = cacheManager.getEntryCache(ml);
-
- byte[] data = new byte[10];
- entryCache.insert(EntryImpl.create(0, 0, data));
- entryCache.insert(EntryImpl.create(0, 1, data));
- entryCache.insert(EntryImpl.create(0, 8, data));
- entryCache.insert(EntryImpl.create(0, 9, data));
-
- final CountDownLatch counter = new CountDownLatch(1);
-
- entryCache.asyncReadEntry(lh, 0, 9, false, new ReadEntriesCallback() {
- public void readEntriesComplete(List<Entry> entries, Object ctx) {
- assertEquals(entries.size(), 10);
- counter.countDown();
- }
-
- public void readEntriesFailed(ManagedLedgerException exception, Object ctx) {
- Assert.fail("should not have failed");
- }
- }, null);
- counter.await();
- }
-
- @Test(timeOut = 5000)
- void testReadMissingMultiple() throws Exception {
- ReadHandle lh = getLedgerHandle();
- when(lh.getId()).thenReturn((long) 0);
-
- EntryCacheManager cacheManager = factory.getEntryCacheManager();
- EntryCache entryCache = cacheManager.getEntryCache(ml);
-
- byte[] data = new byte[10];
- entryCache.insert(EntryImpl.create(0, 0, data));
- entryCache.insert(EntryImpl.create(0, 2, data));
- entryCache.insert(EntryImpl.create(0, 5, data));
- entryCache.insert(EntryImpl.create(0, 8, data));
-
- final CountDownLatch counter = new CountDownLatch(1);
-
- entryCache.asyncReadEntry(lh, 0, 9, false, new ReadEntriesCallback() {
- public void readEntriesComplete(List<Entry> entries, Object ctx) {
- assertEquals(entries.size(), 10);
- counter.countDown();
- }
-
- public void readEntriesFailed(ManagedLedgerException exception, Object ctx) {
- Assert.fail("should not have failed");
- }
- }, null);
- counter.await();
- }
-
- @Test(timeOut = 5000)
- void testReadWithError() throws Exception {
- final ReadHandle lh = getLedgerHandle();
- when(lh.getId()).thenReturn((long) 0);
-
- doAnswer((invocation) -> {
- CompletableFuture<LedgerEntries> future = new CompletableFuture<>();
- future.completeExceptionally(new BKNoSuchLedgerExistsException());
- return future;
- }).when(lh).readAsync(anyLong(), anyLong());
-
- EntryCacheManager cacheManager = factory.getEntryCacheManager();
- EntryCache entryCache = cacheManager.getEntryCache(ml);
-
- byte[] data = new byte[10];
- entryCache.insert(EntryImpl.create(0, 2, data));
-
- final CountDownLatch counter = new CountDownLatch(1);
-
- entryCache.asyncReadEntry(lh, 0, 9, false, new ReadEntriesCallback() {
- public void readEntriesComplete(List<Entry> entries, Object ctx) {
- Assert.fail("should not complete");
- }
-
- public void readEntriesFailed(ManagedLedgerException exception, Object ctx) {
- counter.countDown();
- }
- }, null);
- counter.await();
- }
-
- private static ReadHandle getLedgerHandle() {
- final ReadHandle lh = mock(ReadHandle.class);
- final LedgerEntry ledgerEntry = mock(LedgerEntry.class, Mockito.CALLS_REAL_METHODS);
- doReturn(Unpooled.wrappedBuffer(new byte[10])).when(ledgerEntry).getEntryBuffer();
- doReturn((long) 10).when(ledgerEntry).getLength();
-
- doAnswer((invocation) -> {
- Object[] args = invocation.getArguments();
- long firstEntry = (Long) args[0];
- long lastEntry = (Long) args[1];
-
- Vector<LedgerEntry> entries = new Vector<LedgerEntry>();
- for (int i = 0; i <= (lastEntry - firstEntry); i++) {
- entries.add(ledgerEntry);
- }
- LedgerEntries ledgerEntries = mock(LedgerEntries.class);
- doAnswer((invocation2) -> entries.iterator()).when(ledgerEntries).iterator();
- return CompletableFuture.completedFuture(ledgerEntries);
- }).when(lh).readAsync(anyLong(), anyLong());
-
- return lh;
- }
-
-}
diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorConcurrencyTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorConcurrencyTest.java
deleted file mode 100644
index 62f4045e8c..0000000000
--- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorConcurrencyTest.java
+++ /dev/null
@@ -1,437 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.bookkeeper.mledger.impl;
-
-import static org.apache.bookkeeper.mledger.util.SafeRun.safeRun;
-import static org.testng.Assert.*;
-
-import com.google.common.collect.Lists;
-import java.util.List;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.CyclicBarrier;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicReference;
-import org.apache.bookkeeper.mledger.AsyncCallbacks;
-import org.apache.bookkeeper.mledger.AsyncCallbacks.DeleteCallback;
-import org.apache.bookkeeper.mledger.AsyncCallbacks.ReadEntryCallback;
-import org.apache.bookkeeper.mledger.Entry;
-import org.apache.bookkeeper.mledger.ManagedCursor;
-import org.apache.bookkeeper.mledger.ManagedCursor.IndividualDeletedEntries;
-import org.apache.bookkeeper.mledger.ManagedLedger;
-import org.apache.bookkeeper.mledger.ManagedLedgerConfig;
-import org.apache.bookkeeper.mledger.ManagedLedgerException;
-import org.apache.bookkeeper.mledger.Position;
-import org.apache.bookkeeper.test.MockedBookKeeperTestCase;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.annotations.Test;
-
-public class ManagedCursorConcurrencyTest extends MockedBookKeeperTestCase {
-
- private static final Logger log = LoggerFactory.getLogger(ManagedCursorConcurrencyTest.class);
-
- private final AsyncCallbacks.DeleteCallback deleteCallback = new AsyncCallbacks.DeleteCallback() {
- @Override
- public void deleteComplete(Object ctx) {
- log.info("Deleted message at {}", ctx);
- }
-
- @Override
- public void deleteFailed(ManagedLedgerException exception, Object ctx) {
- log.error("Failed to delete message at {}", ctx, exception);
- }
- };
-
- @Test
- public void testMarkDeleteAndRead() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger", new ManagedLedgerConfig().setMaxEntriesPerLedger(2));
-
- final ManagedCursor cursor = ledger.openCursor("c1");
-
- final List<Position> addedEntries = Lists.newArrayList();
-
- for (int i = 0; i < 1000; i++) {
- Position pos = ledger.addEntry("entry".getBytes());
- addedEntries.add(pos);
- }
-
- final CyclicBarrier barrier = new CyclicBarrier(2);
- final CountDownLatch counter = new CountDownLatch(2);
- final AtomicBoolean gotException = new AtomicBoolean(false);
-
- Thread deleter = new Thread() {
- public void run() {
- try {
- barrier.await();
-
- for (Position position : addedEntries) {
- cursor.markDelete(position);
- }
-
- } catch (Exception e) {
- e.printStackTrace();
- gotException.set(true);
- } finally {
- counter.countDown();
- }
- }
- };
-
- Thread reader = new Thread() {
- public void run() {
- try {
- barrier.await();
-
- for (int i = 0; i < 1000; i++) {
- cursor.readEntries(1).forEach(e -> e.release());
- }
-
- } catch (Exception e) {
- e.printStackTrace();
- gotException.set(true);
- } finally {
- counter.countDown();
- }
- }
- };
-
- deleter.start();
- reader.start();
-
- counter.await();
-
- assertEquals(gotException.get(), false);
- }
-
- @Test
- public void testCloseAndRead() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger_test_close_and_read",
- new ManagedLedgerConfig().setMaxEntriesPerLedger(2));
-
- final ManagedCursor cursor = ledger.openCursor("c1");
- final CompletableFuture<String> closeFuture = new CompletableFuture<>();
- final String CLOSED = "closed";
-
- final List<Position> addedEntries = Lists.newArrayList();
-
- for (int i = 0; i < 1000; i++) {
- Position pos = ledger.addEntry("entry".getBytes());
- addedEntries.add(pos);
- }
-
- final CyclicBarrier barrier = new CyclicBarrier(2);
- final CountDownLatch counter = new CountDownLatch(2);
- final AtomicBoolean gotException = new AtomicBoolean(false);
-
- Thread deleter = new Thread() {
- public void run() {
- try {
- barrier.await();
-
- for (Position position : addedEntries) {
- cursor.markDelete(position);
- Thread.sleep(1);
- }
- } catch (ManagedLedgerException e) {
- if (!e.getMessage().equals("Cursor was already closed")) {
- gotException.set(true);
- }
-
- } catch (Exception e) {
- e.printStackTrace();
- gotException.set(true);
- } finally {
- counter.countDown();
- }
- }
- };
-
- Thread reader = new Thread() {
- public void run() {
- try {
- barrier.await();
-
- for (int i = 0; i < 1000; i++) {
- cursor.readEntries(1).forEach(e -> e.release());
- // Thread.sleep(2,200);
- Thread.sleep(2, 195);
- }
- cursor.asyncClose(new AsyncCallbacks.CloseCallback() {
- @Override
- public void closeComplete(Object ctx) {
- log.info("Successfully closed cursor ledger");
- closeFuture.complete(CLOSED);
- }
-
- @Override
- public void closeFailed(ManagedLedgerException exception, Object ctx) {
- log.error("Error closing cursor: ", exception);
- closeFuture.completeExceptionally(new Exception(exception));
- }
- }, null);
-
- } catch (Exception e) {
- e.printStackTrace();
- gotException.set(true);
- } finally {
- counter.countDown();
- }
- }
- };
-
- deleter.start();
- reader.start();
-
- counter.await();
-
- assertEquals(gotException.get(), false);
- assertEquals(closeFuture.get(), CLOSED);
- }
-
- @Test(timeOut = 30000)
- public void testAckAndClose() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger_test_ack_and_close",
- new ManagedLedgerConfig().setMaxEntriesPerLedger(2));
-
- final ManagedCursor cursor = ledger.openCursor("c1");
-
- final List<Position> addedEntries = Lists.newArrayList();
-
- for (int i = 0; i < 1000; i++) {
- Position pos = ledger.addEntry("entry".getBytes());
- addedEntries.add(pos);
- }
-
- final CyclicBarrier barrier = new CyclicBarrier(2);
- final CountDownLatch counter = new CountDownLatch(2);
- final AtomicBoolean gotException = new AtomicBoolean(false);
-
- // Deleter thread
- cachedExecutor.execute(() -> {
- try {
- barrier.await();
-
- for (Position position : addedEntries) {
- cursor.asyncDelete(position, deleteCallback, position);
- }
- } catch (Exception e) {
- e.printStackTrace();
- gotException.set(true);
- } finally {
- counter.countDown();
- }
- });
-
- // Reader thread
- cachedExecutor.execute(() -> {
- try {
- barrier.await();
-
- for (int i = 0; i < 1000; i++) {
- cursor.readEntries(1).forEach(e -> e.release());
- }
- } catch (Exception e) {
- e.printStackTrace();
- gotException.set(true);
- } finally {
- counter.countDown();
- }
- });
-
- counter.await();
-
- assertEquals(gotException.get(), false);
- }
-
- @Test(timeOut = 30000)
- public void testConcurrentIndividualDeletes() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger", new ManagedLedgerConfig().setMaxEntriesPerLedger(100));
-
- final ManagedCursor cursor = ledger.openCursor("c1");
-
- final int N = 1000;
- final List<Position> addedEntries = Lists.newArrayListWithExpectedSize(N);
-
- for (int i = 0; i < N; i++) {
- Position pos = ledger.addEntry("entry".getBytes());
- addedEntries.add(pos);
- }
-
- final int Threads = 10;
- final CyclicBarrier barrier = new CyclicBarrier(Threads);
- final CountDownLatch counter = new CountDownLatch(Threads);
- final AtomicBoolean gotException = new AtomicBoolean(false);
-
- for (int thread = 0; thread < Threads; thread++) {
- final int myThread = thread;
- cachedExecutor.execute(() -> {
- try {
- barrier.await();
-
- for (int i = 0; i < N; i++) {
- int threadId = i % Threads;
- if (threadId == myThread) {
- cursor.delete(addedEntries.get(i));
- }
- }
-
- } catch (Exception e) {
- e.printStackTrace();
- gotException.set(true);
- } finally {
- counter.countDown();
- }
- });
- }
-
- counter.await();
-
- assertEquals(gotException.get(), false);
- assertEquals(cursor.getMarkDeletedPosition(), addedEntries.get(addedEntries.size() - 1));
- }
-
- @Test(timeOut = 30000)
- public void testConcurrentReadOfSameEntry() throws Exception {
- ManagedLedger ledger = factory.open("testConcurrentReadOfSameEntry", new ManagedLedgerConfig());
- final int numCursors = 5;
- final List<ManagedCursor> cursors = Lists.newArrayList();
- for (int i = 0; i < numCursors; i++) {
- final ManagedCursor cursor = ledger.openCursor("c" + i);
- cursors.add(cursor);
- }
-
- final int N = 100;
- for (int i = 0; i < N; i++) {
- ledger.addEntry(("entry" + i).getBytes());
- }
- long currentLedger = ((PositionImpl) cursors.get(0).getMarkDeletedPosition()).getLedgerId();
-
- // empty the cache
- ((ManagedLedgerImpl) ledger).entryCache.invalidateAllEntries(currentLedger);
-
- final CyclicBarrier barrier = new CyclicBarrier(numCursors);
- final CountDownLatch counter = new CountDownLatch(numCursors);
-
- AtomicReference<String> result = new AtomicReference<String>();
-
- for (int i = 0; i < numCursors; i++) {
- final int cursorIndex = i;
- final ManagedCursor cursor = cursors.get(cursorIndex);
- cachedExecutor.execute(() -> {
- try {
- barrier.await();
- for (int j = 0; j < N; j++) {
- String expected = "entry" + j;
- String data = new String(cursor.readEntries(1).get(0).getDataAndRelease());
- if ((!expected.equals(data)) && result.get() == null) {
- result.set("Mismatched entry in cursor " + (cursorIndex + 1) + " at position " + (j + 1)
- + "--- Expected: " + expected + ", Actual: " + data);
- }
- }
- } catch (Exception e) {
- e.printStackTrace();
- } finally {
- counter.countDown();
- }
- });
- }
-
- counter.await();
- assertNull(result.get());
- }
-
- @Test(timeOut = 30000)
- public void testConcurrentIndividualDeletesWithGetNthEntry() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger",
- new ManagedLedgerConfig().setMaxEntriesPerLedger(100).setThrottleMarkDelete(0.5));
-
- final ManagedCursor cursor = ledger.openCursor("c1");
-
- final int N = 1000;
- final List<Position> addedEntries = Lists.newArrayListWithExpectedSize(N);
-
- for (int i = 0; i < N; i++) {
- Position pos = ledger.addEntry("entry".getBytes());
- addedEntries.add(pos);
- }
-
- final int deleteEntries = 100;
- final CountDownLatch counter = new CountDownLatch(deleteEntries);
- final AtomicBoolean gotException = new AtomicBoolean(false);
- final AtomicInteger iteration = new AtomicInteger(0);
-
- for (int i = 0; i < deleteEntries; i++) {
- executor.submit(safeRun(() -> {
- try {
- cursor.asyncDelete(addedEntries.get(iteration.getAndIncrement()), new DeleteCallback() {
- @Override
- public void deleteComplete(Object ctx) {
- // Ok
- }
-
- @Override
- public void deleteFailed(ManagedLedgerException exception, Object ctx) {
- exception.printStackTrace();
- gotException.set(true);
- }
- }, null);
- } catch (Exception e) {
- e.printStackTrace();
- gotException.set(true);
- } finally {
- counter.countDown();
- }
- }));
- }
-
- counter.await();
-
- final int readEntries = N - deleteEntries;
- final CountDownLatch readCounter = new CountDownLatch(readEntries);
- final AtomicInteger successReadEntries = new AtomicInteger(0);
- for (int i = 1; i <= readEntries; i++) {
- try {
- cursor.asyncGetNthEntry(i, IndividualDeletedEntries.Exclude, new ReadEntryCallback() {
- @Override
- public void readEntryComplete(Entry entry, Object ctx) {
- successReadEntries.getAndIncrement();
- entry.release();
- readCounter.countDown();
- }
-
- @Override
- public void readEntryFailed(ManagedLedgerException exception, Object ctx) {
- exception.printStackTrace();
- gotException.set(true);
- }
- }, null);
- } catch (Exception e) {
- e.printStackTrace();
- gotException.set(true);
- }
- }
-
- readCounter.await();
-
- assertFalse(gotException.get());
- assertEquals(successReadEntries.get(), readEntries);
- }
-}
diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorContainerTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorContainerTest.java
deleted file mode 100644
index 3e404d9be0..0000000000
--- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorContainerTest.java
+++ /dev/null
@@ -1,577 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.bookkeeper.mledger.impl;
-
-import static org.testng.Assert.*;
-
-import com.google.common.base.Predicate;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Sets;
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import org.apache.bookkeeper.mledger.AsyncCallbacks;
-import org.apache.bookkeeper.mledger.AsyncCallbacks.ClearBacklogCallback;
-import org.apache.bookkeeper.mledger.AsyncCallbacks.DeleteCallback;
-import org.apache.bookkeeper.mledger.AsyncCallbacks.MarkDeleteCallback;
-import org.apache.bookkeeper.mledger.AsyncCallbacks.ReadEntriesCallback;
-import org.apache.bookkeeper.mledger.AsyncCallbacks.ReadEntryCallback;
-import org.apache.bookkeeper.mledger.AsyncCallbacks.SkipEntriesCallback;
-import org.apache.bookkeeper.mledger.Entry;
-import org.apache.bookkeeper.mledger.ManagedCursor;
-import org.apache.bookkeeper.mledger.ManagedLedgerException;
-import org.apache.bookkeeper.mledger.Position;
-import org.testng.annotations.Test;
-
-@Test
-public class ManagedCursorContainerTest {
-
- private static class MockManagedCursor implements ManagedCursor {
-
- ManagedCursorContainer container;
- Position position;
- String name;
-
- public MockManagedCursor(ManagedCursorContainer container, String name, Position position) {
- this.container = container;
- this.name = name;
- this.position = position;
- }
-
- @Override
- public Map<String, Long> getProperties() {
- return Collections.emptyMap();
- }
-
- @Override
- public boolean isDurable() {
- return true;
- }
-
- @Override
- public List<Entry> readEntries(int numberOfEntriesToRead) throws ManagedLedgerException {
- return Lists.newArrayList();
- }
-
- @Override
- public void asyncReadEntries(int numberOfEntriesToRead, ReadEntriesCallback callback, Object ctx) {
- callback.readEntriesComplete(null, ctx);
- }
-
- @Override
- public boolean hasMoreEntries() {
- return true;
- }
-
- @Override
- public long getNumberOfEntries() {
- return 0;
- }
-
- @Override
- public long getNumberOfEntriesInBacklog() {
- return 0;
- }
-
- @Override
- public void markDelete(Position position) throws ManagedLedgerException {
- markDelete(position, Collections.emptyMap());
- }
-
- @Override
- public void markDelete(Position position, Map<String, Long> properties) throws ManagedLedgerException {
- this.position = position;
- container.cursorUpdated(this, (PositionImpl) position);
- }
-
- @Override
- public void asyncMarkDelete(Position position, MarkDeleteCallback callback, Object ctx) {
- fail();
- }
-
- @Override
- public void asyncMarkDelete(Position position, Map<String, Long> properties, MarkDeleteCallback callback,
- Object ctx) {
- fail();
- }
-
- @Override
- public Position getMarkDeletedPosition() {
- return position;
- }
-
- @Override
- public String getName() {
- return name;
- }
-
- @Override
- public long getLastActive() {
- return System.currentTimeMillis();
- }
-
- @Override
- public void updateLastActive() {
- // no-op
- }
-
- public String toString() {
- return String.format("%s=%s", name, position);
- }
-
- @Override
- public Position getReadPosition() {
- return null;
- }
-
- @Override
- public void rewind() {
- }
-
- @Override
- public void seek(Position newReadPosition) {
- }
-
- @Override
- public void close() {
- }
-
- @Override
- public void asyncClose(AsyncCallbacks.CloseCallback callback, Object ctx) {
- }
-
- @Override
- public void delete(Position position) throws InterruptedException, ManagedLedgerException {
- }
-
- @Override
- public void asyncDelete(Position position, DeleteCallback callback, Object ctx) {
- }
-
- @Override
- public void delete(Iterable<Position> positions) throws InterruptedException, ManagedLedgerException {
- }
-
- @Override
- public void asyncDelete(Iterable<Position> position, DeleteCallback callback, Object ctx) {
- }
-
- @Override
- public void clearBacklog() throws InterruptedException, ManagedLedgerException {
- }
-
- @Override
- public void asyncClearBacklog(ClearBacklogCallback callback, Object ctx) {
- }
-
- @Override
- public void skipEntries(int numEntriesToSkip, IndividualDeletedEntries deletedEntries)
- throws InterruptedException, ManagedLedgerException {
- }
-
- @Override
- public void asyncSkipEntries(int numEntriesToSkip, IndividualDeletedEntries deletedEntries,
- final SkipEntriesCallback callback, Object ctx) {
- }
-
- @Override
- public Position findNewestMatching(Predicate<Entry> condition)
- throws InterruptedException, ManagedLedgerException {
- return null;
- }
-
- @Override
- public Position findNewestMatching(FindPositionConstraint constraint, Predicate<Entry> condition) throws InterruptedException, ManagedLedgerException {
- return null;
- }
-
- @Override
- public void asyncFindNewestMatching(FindPositionConstraint constraint, Predicate<Entry> condition,
- AsyncCallbacks.FindEntryCallback callback, Object ctx) {
- }
-
- @Override
- public void asyncResetCursor(final Position position, AsyncCallbacks.ResetCursorCallback callback) {
-
- }
-
- @Override
- public void resetCursor(final Position position) throws ManagedLedgerException, InterruptedException {
-
- }
-
- @Override
- public Position getFirstPosition() {
- return null;
- }
-
- @Override
- public List<Entry> replayEntries(Set<? extends Position> positions)
- throws InterruptedException, ManagedLedgerException {
- return null;
- }
-
- @Override
- public Set<? extends Position> asyncReplayEntries(Set<? extends Position> positions, ReadEntriesCallback callback, Object ctx) {
- return Sets.newConcurrentHashSet();
- }
-
- @Override
- public List<Entry> readEntriesOrWait(int numberOfEntriesToRead)
- throws InterruptedException, ManagedLedgerException {
- return null;
- }
-
- @Override
- public void asyncReadEntriesOrWait(int numberOfEntriesToRead, ReadEntriesCallback callback, Object ctx) {
- }
-
- @Override
- public boolean cancelPendingReadRequest() {
- return true;
- }
-
- @Override
- public Entry getNthEntry(int N, IndividualDeletedEntries deletedEntries)
- throws InterruptedException, ManagedLedgerException {
- return null;
- }
-
- @Override
- public void asyncGetNthEntry(int N, IndividualDeletedEntries deletedEntries, ReadEntryCallback callback,
- Object ctx) {
- }
-
- @Override
- public void setActive() {
- }
-
- @Override
- public void setInactive() {
- }
-
- @Override
- public boolean isActive() {
- return true;
- }
-
- @Override
- public long getNumberOfEntriesSinceFirstNotAckedMessage() {
- return 0;
- }
-
- @Override
- public int getTotalNonContiguousDeletedMessagesRange() {
- return 0;
- }
-
- @Override
- public long getEstimatedSizeSinceMarkDeletePosition() {
- return 0L;
- }
-
- @Override
- public void setThrottleMarkDelete(double throttleMarkDelete) {
- }
-
- @Override
- public double getThrottleMarkDelete() {
- return -1;
- }
- }
-
- @Test
- void simple() throws Exception {
- ManagedCursorContainer container = new ManagedCursorContainer();
- assertEquals(container.getSlowestReaderPosition(), null);
-
- ManagedCursor cursor1 = new MockManagedCursor(container, "test1", new PositionImpl(5, 5));
- container.add(cursor1);
- assertEquals(container.getSlowestReaderPosition(), new PositionImpl(5, 5));
-
- ManagedCursor cursor2 = new MockManagedCursor(container, "test2", new PositionImpl(2, 2));
- container.add(cursor2);
- assertEquals(container.getSlowestReaderPosition(), new PositionImpl(2, 2));
-
- ManagedCursor cursor3 = new MockManagedCursor(container, "test3", new PositionImpl(2, 0));
- container.add(cursor3);
- assertEquals(container.getSlowestReaderPosition(), new PositionImpl(2, 0));
-
- assertEquals(container.toString(), "[test1=5:5, test2=2:2, test3=2:0]");
-
- ManagedCursor cursor4 = new MockManagedCursor(container, "test4", new PositionImpl(4, 0));
- container.add(cursor4);
- assertEquals(container.getSlowestReaderPosition(), new PositionImpl(2, 0));
-
- ManagedCursor cursor5 = new MockManagedCursor(container, "test5", new PositionImpl(3, 5));
- container.add(cursor5);
- assertEquals(container.getSlowestReaderPosition(), new PositionImpl(2, 0));
-
- cursor3.markDelete(new PositionImpl(3, 0));
- assertEquals(container.getSlowestReaderPosition(), new PositionImpl(2, 2));
-
- cursor2.markDelete(new PositionImpl(10, 5));
- assertEquals(container.getSlowestReaderPosition(), new PositionImpl(3, 0));
-
- container.removeCursor(cursor3.getName());
- assertEquals(container.getSlowestReaderPosition(), new PositionImpl(3, 5));
-
- container.removeCursor(cursor2.getName());
- container.removeCursor(cursor5.getName());
- container.removeCursor(cursor1.getName());
- assertEquals(container.getSlowestReaderPosition(), new PositionImpl(4, 0));
-
- assertFalse(container.isEmpty());
-
- container.removeCursor(cursor4.getName());
- assertEquals(container.getSlowestReaderPosition(), null);
-
- assertTrue(container.isEmpty());
-
- ManagedCursor cursor6 = new MockManagedCursor(container, "test6", new PositionImpl(6, 5));
- container.add(cursor6);
- assertEquals(container.getSlowestReaderPosition(), new PositionImpl(6, 5));
-
- assertEquals(container.toString(), "[test6=6:5]");
- }
-
- @Test
- void updatingCursorOutsideContainer() throws Exception {
- ManagedCursorContainer container = new ManagedCursorContainer();
-
- ManagedCursor cursor1 = new MockManagedCursor(container, "test1", new PositionImpl(5, 5));
- container.add(cursor1);
- assertEquals(container.getSlowestReaderPosition(), new PositionImpl(5, 5));
-
- MockManagedCursor cursor2 = new MockManagedCursor(container, "test2", new PositionImpl(2, 2));
- container.add(cursor2);
- assertEquals(container.getSlowestReaderPosition(), new PositionImpl(2, 2));
-
- cursor2.position = new PositionImpl(8, 8);
-
- // Until we don't update the container, the ordering will not change
- assertEquals(container.getSlowestReaderPosition(), new PositionImpl(2, 2));
-
- container.cursorUpdated(cursor2, cursor2.position);
-
- assertEquals(container.getSlowestReaderPosition(), new PositionImpl(5, 5));
- }
-
- @Test
- void removingCursor() throws Exception {
- ManagedCursorContainer container = new ManagedCursorContainer();
-
- ManagedCursor cursor1 = new MockManagedCursor(container, "test1", new PositionImpl(5, 5));
- container.add(cursor1);
- assertEquals(container.getSlowestReaderPosition(), new PositionImpl(5, 5));
- assertEquals(container.get("test1"), cursor1);
-
- MockManagedCursor cursor2 = new MockManagedCursor(container, "test2", new PositionImpl(2, 2));
- container.add(cursor2);
- assertEquals(container.getSlowestReaderPosition(), new PositionImpl(2, 2));
- assertEquals(container.get("test2"), cursor2);
-
- MockManagedCursor cursor3 = new MockManagedCursor(container, "test3", new PositionImpl(1, 1));
- container.add(cursor3);
- assertEquals(container.getSlowestReaderPosition(), new PositionImpl(1, 1));
- assertEquals(container.get("test3"), cursor3);
-
- assertEquals(container, Lists.newArrayList(cursor1, cursor2, cursor3));
-
- // Remove the cursor in the middle
- container.removeCursor("test2");
-
- assertEquals(container, Lists.newArrayList(cursor1, cursor3));
-
- assertEquals(container.get("test2"), null);
-
- assertEquals(container.getSlowestReaderPosition(), new PositionImpl(1, 1));
-
- container.removeCursor("test3");
-
- assertEquals(container, Lists.newArrayList(cursor1));
-
- assertEquals(container.getSlowestReaderPosition(), new PositionImpl(5, 5));
- }
-
- @Test
- void ordering() throws Exception {
- ManagedCursorContainer container = new ManagedCursorContainer();
-
- ManagedCursor cursor1 = new MockManagedCursor(container, "test1", new PositionImpl(5, 5));
- ManagedCursor cursor2 = new MockManagedCursor(container, "test2", new PositionImpl(5, 1));
- ManagedCursor cursor3 = new MockManagedCursor(container, "test3", new PositionImpl(7, 1));
- ManagedCursor cursor4 = new MockManagedCursor(container, "test4", new PositionImpl(6, 4));
- ManagedCursor cursor5 = new MockManagedCursor(container, "test5", new PositionImpl(7, 0));
-
- container.add(cursor1);
- container.add(cursor2);
- container.add(cursor3);
- container.add(cursor4);
- container.add(cursor5);
-
- assertEquals(container.getSlowestReaderPosition(), new PositionImpl(5, 1));
- container.removeCursor("test2");
-
- assertEquals(container.getSlowestReaderPosition(), new PositionImpl(5, 5));
- container.removeCursor("test1");
-
- assertEquals(container.getSlowestReaderPosition(), new PositionImpl(6, 4));
- container.removeCursor("test4");
-
- assertEquals(container.getSlowestReaderPosition(), new PositionImpl(7, 0));
- container.removeCursor("test5");
-
- assertEquals(container.getSlowestReaderPosition(), new PositionImpl(7, 1));
- container.removeCursor("test3");
-
- assertTrue(container.isEmpty());
- }
-
- @Test
- void orderingWithUpdates() throws Exception {
- ManagedCursorContainer container = new ManagedCursorContainer();
-
- MockManagedCursor c1 = new MockManagedCursor(container, "test1", new PositionImpl(5, 5));
- MockManagedCursor c2 = new MockManagedCursor(container, "test2", new PositionImpl(5, 1));
- MockManagedCursor c3 = new MockManagedCursor(container, "test3", new PositionImpl(7, 1));
- MockManagedCursor c4 = new MockManagedCursor(container, "test4", new PositionImpl(6, 4));
- MockManagedCursor c5 = new MockManagedCursor(container, "test5", new PositionImpl(7, 0));
-
- container.add(c1);
- container.add(c2);
- container.add(c3);
- container.add(c4);
- container.add(c5);
-
- assertEquals(container.getSlowestReaderPosition(), new PositionImpl(5, 1));
-
- c1.position = new PositionImpl(5, 8);
- container.cursorUpdated(c1, c1.position);
- assertEquals(container.getSlowestReaderPosition(), new PositionImpl(5, 1));
-
- c2.position = new PositionImpl(5, 6);
- container.cursorUpdated(c2, c2.position);
- assertEquals(container.getSlowestReaderPosition(), new PositionImpl(5, 6));
-
- c1.position = new PositionImpl(6, 8);
- container.cursorUpdated(c1, c1.position);
- assertEquals(container.getSlowestReaderPosition(), new PositionImpl(5, 6));
-
- c3.position = new PositionImpl(8, 5);
- container.cursorUpdated(c3, c3.position);
- assertEquals(container.getSlowestReaderPosition(), new PositionImpl(5, 6));
-
- c1.position = new PositionImpl(8, 4);
- container.cursorUpdated(c1, c1.position);
- assertEquals(container.getSlowestReaderPosition(), new PositionImpl(5, 6));
-
- c2.position = new PositionImpl(8, 4);
- container.cursorUpdated(c2, c2.position);
- assertEquals(container.getSlowestReaderPosition(), new PositionImpl(6, 4));
-
- c4.position = new PositionImpl(7, 1);
- container.cursorUpdated(c4, c4.position);
-
- // ////
-
- assertEquals(container.getSlowestReaderPosition(), new PositionImpl(7, 0));
- container.removeCursor("test5");
-
- assertEquals(container.getSlowestReaderPosition(), new PositionImpl(7, 1));
- container.removeCursor("test4");
-
- assertEquals(container.getSlowestReaderPosition(), new PositionImpl(8, 4));
- container.removeCursor("test1");
-
- assertEquals(container.getSlowestReaderPosition(), new PositionImpl(8, 4));
- container.removeCursor("test2");
-
- assertEquals(container.getSlowestReaderPosition(), new PositionImpl(8, 5));
- container.removeCursor("test3");
-
- assertTrue(container.isEmpty());
- }
-
- @Test
- void orderingWithUpdatesAndReset() throws Exception {
- ManagedCursorContainer container = new ManagedCursorContainer();
-
- MockManagedCursor c1 = new MockManagedCursor(container, "test1", new PositionImpl(5, 5));
- MockManagedCursor c2 = new MockManagedCursor(container, "test2", new PositionImpl(5, 1));
- MockManagedCursor c3 = new MockManagedCursor(container, "test3", new PositionImpl(7, 1));
- MockManagedCursor c4 = new MockManagedCursor(container, "test4", new PositionImpl(6, 4));
- MockManagedCursor c5 = new MockManagedCursor(container, "test5", new PositionImpl(7, 0));
-
- container.add(c1);
- container.add(c2);
- container.add(c3);
- container.add(c4);
- container.add(c5);
-
- assertEquals(container.getSlowestReaderPosition(), new PositionImpl(5, 1));
-
- c1.position = new PositionImpl(5, 8);
- container.cursorUpdated(c1, c1.position);
- assertEquals(container.getSlowestReaderPosition(), new PositionImpl(5, 1));
-
- c1.position = new PositionImpl(5, 6);
- container.cursorUpdated(c1, c1.position);
- assertEquals(container.getSlowestReaderPosition(), new PositionImpl(5, 1));
-
- c2.position = new PositionImpl(6, 8);
- container.cursorUpdated(c2, c2.position);
- assertEquals(container.getSlowestReaderPosition(), new PositionImpl(5, 6));
-
- c3.position = new PositionImpl(8, 5);
- container.cursorUpdated(c3, c3.position);
- assertEquals(container.getSlowestReaderPosition(), new PositionImpl(5, 6));
-
- c1.position = new PositionImpl(8, 4);
- container.cursorUpdated(c1, c1.position);
- assertEquals(container.getSlowestReaderPosition(), new PositionImpl(6, 4));
-
- c2.position = new PositionImpl(4, 4);
- container.cursorUpdated(c2, c2.position);
- assertEquals(container.getSlowestReaderPosition(), new PositionImpl(4, 4));
-
- c4.position = new PositionImpl(7, 1);
- container.cursorUpdated(c4, c4.position);
-
- // ////
-
- assertEquals(container.getSlowestReaderPosition(), new PositionImpl(4, 4));
- container.removeCursor("test2");
-
- assertEquals(container.getSlowestReaderPosition(), new PositionImpl(7, 0));
- container.removeCursor("test5");
-
- assertEquals(container.getSlowestReaderPosition(), new PositionImpl(7, 1));
- container.removeCursor("test1");
-
- assertEquals(container.getSlowestReaderPosition(), new PositionImpl(7, 1));
- container.removeCursor("test4");
-
- assertEquals(container.getSlowestReaderPosition(), new PositionImpl(8, 5));
- container.removeCursor("test3");
-
- assertTrue(container.isEmpty());
- }
-}
diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorListAckTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorListAckTest.java
deleted file mode 100644
index aaf7cd1c6f..0000000000
--- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorListAckTest.java
+++ /dev/null
@@ -1,75 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.bookkeeper.mledger.impl;
-
-import static org.testng.Assert.assertEquals;
-
-import com.google.common.base.Charsets;
-import com.google.common.collect.Lists;
-
-import java.nio.charset.Charset;
-
-import org.apache.bookkeeper.mledger.ManagedCursor;
-import org.apache.bookkeeper.mledger.ManagedLedger;
-import org.apache.bookkeeper.mledger.ManagedLedgerConfig;
-import org.apache.bookkeeper.mledger.Position;
-import org.apache.bookkeeper.test.MockedBookKeeperTestCase;
-import org.testng.annotations.Test;
-
-public class ManagedCursorListAckTest extends MockedBookKeeperTestCase {
-
- private static final Charset Encoding = Charsets.UTF_8;
-
- @Test(timeOut = 20000)
- void testMultiPositionDelete() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger", new ManagedLedgerConfig().setMaxEntriesPerLedger(2));
-
- ManagedCursor c1 = ledger.openCursor("c1");
- Position p0 = c1.getMarkDeletedPosition();
- Position p1 = ledger.addEntry("dummy-entry-1".getBytes(Encoding));
- Position p2 = ledger.addEntry("dummy-entry-2".getBytes(Encoding));
- Position p3 = ledger.addEntry("dummy-entry-3".getBytes(Encoding));
- Position p4 = ledger.addEntry("dummy-entry-4".getBytes(Encoding));
- Position p5 = ledger.addEntry("dummy-entry-5".getBytes(Encoding));
- Position p6 = ledger.addEntry("dummy-entry-6".getBytes(Encoding));
- Position p7 = ledger.addEntry("dummy-entry-7".getBytes(Encoding));
-
- assertEquals(c1.getNumberOfEntries(), 7);
- assertEquals(c1.getNumberOfEntriesInBacklog(), 7);
-
- c1.delete(Lists.newArrayList(p2, p3, p5, p7));
-
- assertEquals(c1.getNumberOfEntries(), 3);
- assertEquals(c1.getNumberOfEntriesInBacklog(), 3);
- assertEquals(c1.getMarkDeletedPosition(), p0);
-
- c1.delete(Lists.newArrayList(p1));
-
- assertEquals(c1.getNumberOfEntries(), 2);
- assertEquals(c1.getNumberOfEntriesInBacklog(), 2);
- assertEquals(c1.getMarkDeletedPosition(), p3);
-
- c1.delete(Lists.newArrayList(p4, p6, p7));
-
- assertEquals(c1.getNumberOfEntries(), 0);
- assertEquals(c1.getNumberOfEntriesInBacklog(), 0);
- assertEquals(c1.getMarkDeletedPosition(), p7);
- }
-
-}
diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorPropertiesTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorPropertiesTest.java
deleted file mode 100644
index 7febfdd0ba..0000000000
--- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorPropertiesTest.java
+++ /dev/null
@@ -1,105 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.bookkeeper.mledger.impl;
-
-import static org.testng.Assert.assertEquals;
-
-import java.util.Collections;
-import java.util.Map;
-import java.util.TreeMap;
-import org.apache.bookkeeper.mledger.ManagedCursor;
-import org.apache.bookkeeper.mledger.ManagedLedger;
-import org.apache.bookkeeper.mledger.ManagedLedgerConfig;
-import org.apache.bookkeeper.mledger.ManagedLedgerFactory;
-import org.apache.bookkeeper.mledger.ManagedLedgerFactoryConfig;
-import org.apache.bookkeeper.mledger.Position;
-import org.apache.bookkeeper.test.MockedBookKeeperTestCase;
-import org.testng.annotations.Test;
-
-public class ManagedCursorPropertiesTest extends MockedBookKeeperTestCase {
-
- @Test(timeOut = 20000)
- void testPropertiesClose() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger", new ManagedLedgerConfig());
- ManagedCursor c1 = ledger.openCursor("c1");
-
- assertEquals(c1.getProperties(), Collections.emptyMap());
-
- ledger.addEntry("entry-1".getBytes());
- ledger.addEntry("entry-2".getBytes());
- Position p3 = ledger.addEntry("entry-3".getBytes());
-
- Map<String, Long> properties = new TreeMap<>();
- properties.put("a", 1L);
- properties.put("b", 2L);
- properties.put("c", 3L);
- c1.markDelete(p3, properties);
-
- assertEquals(c1.getProperties(), properties);
-
- Map<String, Long> properties2 = new TreeMap<>();
- properties2.put("a", 4L);
- properties2.put("b", 5L);
- properties2.put("c", 6L);
- c1.markDelete(p3, properties2);
-
- assertEquals(c1.getProperties(), properties2);
-
- ledger.close();
-
- // Reopen the managed ledger
- ledger = factory.open("my_test_ledger", new ManagedLedgerConfig());
- c1 = ledger.openCursor("c1");
-
- assertEquals(c1.getMarkDeletedPosition(), p3);
- assertEquals(c1.getProperties(), properties2);
- }
-
- @Test(timeOut = 20000)
- void testPropertiesRecoveryAfterCrash() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger", new ManagedLedgerConfig());
- ManagedCursor c1 = ledger.openCursor("c1");
-
- assertEquals(c1.getProperties(), Collections.emptyMap());
-
- ledger.addEntry("entry-1".getBytes());
- ledger.addEntry("entry-2".getBytes());
- Position p3 = ledger.addEntry("entry-3".getBytes());
-
- Map<String, Long> properties = new TreeMap<>();
- properties.put("a", 1L);
- properties.put("b", 2L);
- properties.put("c", 3L);
- c1.markDelete(p3, properties);
-
- // Create a new factory to force a managed ledger close and recovery
- ManagedLedgerFactoryConfig conf = new ManagedLedgerFactoryConfig();
- ManagedLedgerFactory factory2 = new ManagedLedgerFactoryImpl(bkc, zkc, conf);
-
- // Reopen the managed ledger
- ledger = factory2.open("my_test_ledger", new ManagedLedgerConfig());
- c1 = ledger.openCursor("c1");
-
- assertEquals(c1.getMarkDeletedPosition(), p3);
- assertEquals(c1.getProperties(), properties);
-
- factory2.shutdown();
- }
-
-}
diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java
deleted file mode 100644
index d8c48ded3b..0000000000
--- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedCursorTest.java
+++ /dev/null
@@ -1,2773 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.bookkeeper.mledger.impl;
-
-import static org.mockito.Mockito.any;
-import static org.mockito.Mockito.doAnswer;
-import static org.mockito.Mockito.eq;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.when;
-import static org.testng.Assert.*;
-
-import com.google.common.base.Charsets;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Sets;
-import java.nio.charset.Charset;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.Collections;
-import java.util.List;
-import java.util.Set;
-import java.util.concurrent.Callable;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.CyclicBarrier;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.Future;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicLong;
-import java.util.concurrent.atomic.AtomicReference;
-import java.util.stream.Collectors;
-import org.apache.bookkeeper.client.BKException;
-import org.apache.bookkeeper.client.BookKeeper;
-import org.apache.bookkeeper.client.BookKeeper.DigestType;
-import org.apache.bookkeeper.client.LedgerEntry;
-import org.apache.bookkeeper.mledger.AsyncCallbacks;
-import org.apache.bookkeeper.mledger.AsyncCallbacks.AddEntryCallback;
-import org.apache.bookkeeper.mledger.AsyncCallbacks.DeleteCallback;
-import org.apache.bookkeeper.mledger.AsyncCallbacks.MarkDeleteCallback;
-import org.apache.bookkeeper.mledger.AsyncCallbacks.ReadEntriesCallback;
-import org.apache.bookkeeper.mledger.Entry;
-import org.apache.bookkeeper.mledger.ManagedCursor;
-import org.apache.bookkeeper.mledger.ManagedCursor.IndividualDeletedEntries;
-import org.apache.bookkeeper.mledger.ManagedLedger;
-import org.apache.bookkeeper.mledger.ManagedLedgerConfig;
-import org.apache.bookkeeper.mledger.ManagedLedgerException;
-import org.apache.bookkeeper.mledger.ManagedLedgerException.MetaStoreException;
-import org.apache.bookkeeper.mledger.ManagedLedgerFactory;
-import org.apache.bookkeeper.mledger.ManagedLedgerFactoryConfig;
-import org.apache.bookkeeper.mledger.Position;
-import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl.VoidCallback;
-import org.apache.bookkeeper.mledger.impl.MetaStore.MetaStoreCallback;
-import org.apache.bookkeeper.mledger.impl.MetaStore.Stat;
-import org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedCursorInfo;
-import org.apache.bookkeeper.mledger.proto.MLDataFormats.PositionInfo;
-import org.apache.bookkeeper.test.MockedBookKeeperTestCase;
-import org.apache.zookeeper.KeeperException.Code;
-import org.mockito.Mockito;
-import org.mockito.invocation.InvocationOnMock;
-import org.mockito.stubbing.Answer;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.annotations.Test;
-
-public class ManagedCursorTest extends MockedBookKeeperTestCase {
-
- private static final Charset Encoding = Charsets.UTF_8;
-
- @Test(timeOut = 20000)
- void readFromEmptyLedger() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger");
-
- ManagedCursor c1 = ledger.openCursor("c1");
- List<Entry> entries = c1.readEntries(10);
- assertEquals(entries.size(), 0);
- entries.forEach(e -> e.release());
-
- ledger.addEntry("test".getBytes(Encoding));
- entries = c1.readEntries(10);
- assertEquals(entries.size(), 1);
- entries.forEach(e -> e.release());
-
- entries = c1.readEntries(10);
- assertEquals(entries.size(), 0);
- entries.forEach(e -> e.release());
-
- // Test string representation
- assertEquals(c1.toString(), "ManagedCursorImpl{ledger=my_test_ledger, name=c1, ackPos=3:-1, readPos=3:1}");
- }
-
- @Test(timeOut = 20000)
- void readTwice() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger", new ManagedLedgerConfig().setMaxEntriesPerLedger(1));
-
- ManagedCursor c1 = ledger.openCursor("c1");
- ManagedCursor c2 = ledger.openCursor("c2");
-
- ledger.addEntry("entry-1".getBytes(Encoding));
- ledger.addEntry("entry-2".getBytes(Encoding));
-
- List<Entry> entries = c1.readEntries(2);
- assertEquals(entries.size(), 2);
- entries.forEach(e -> e.release());
-
- entries = c1.readEntries(2);
- assertEquals(entries.size(), 0);
- entries.forEach(e -> e.release());
-
- entries = c2.readEntries(2);
- assertEquals(entries.size(), 2);
- entries.forEach(e -> e.release());
-
- entries = c2.readEntries(2);
- assertEquals(entries.size(), 0);
- entries.forEach(e -> e.release());
- }
-
- @Test(timeOut = 20000)
- void readWithCacheDisabled() throws Exception {
- ManagedLedgerFactoryConfig config = new ManagedLedgerFactoryConfig();
- config.setMaxCacheSize(0);
- factory = new ManagedLedgerFactoryImpl(bkc, bkc.getZkHandle(), config);
- ManagedLedger ledger = factory.open("my_test_ledger", new ManagedLedgerConfig().setMaxEntriesPerLedger(1));
-
- ManagedCursor c1 = ledger.openCursor("c1");
- ManagedCursor c2 = ledger.openCursor("c2");
-
- ledger.addEntry("entry-1".getBytes(Encoding));
- ledger.addEntry("entry-2".getBytes(Encoding));
-
- List<Entry> entries = c1.readEntries(2);
- assertEquals(entries.size(), 2);
- assertEquals(new String(entries.get(0).getData(), Encoding), "entry-1");
- assertEquals(new String(entries.get(1).getData(), Encoding), "entry-2");
- entries.forEach(e -> e.release());
-
- entries = c1.readEntries(2);
- assertEquals(entries.size(), 0);
- entries.forEach(e -> e.release());
-
- entries = c2.readEntries(2);
- assertEquals(entries.size(), 2);
- entries.forEach(e -> e.release());
-
- entries = c2.readEntries(2);
- assertEquals(entries.size(), 0);
- entries.forEach(e -> e.release());
- }
-
- @Test(timeOut = 20000)
- void getEntryDataTwice() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger");
-
- ManagedCursor c1 = ledger.openCursor("c1");
-
- ledger.addEntry("entry-1".getBytes(Encoding));
-
- List<Entry> entries = c1.readEntries(2);
- assertEquals(entries.size(), 1);
-
- Entry entry = entries.get(0);
- assertEquals(entry.getLength(), "entry-1".length());
- byte[] data1 = entry.getData();
- byte[] data2 = entry.getData();
- assertEquals(data1, data2);
- entry.release();
- }
-
- @Test(timeOut = 20000)
- void readFromClosedLedger() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger", new ManagedLedgerConfig().setMaxEntriesPerLedger(1));
-
- ManagedCursor c1 = ledger.openCursor("c1");
-
- ledger.close();
-
- try {
- c1.readEntries(2);
- fail("ledger is closed, should fail");
- } catch (ManagedLedgerException e) {
- // ok
- }
- }
-
- @Test(timeOut = 20000)
- void testNumberOfEntries() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger", new ManagedLedgerConfig().setMaxEntriesPerLedger(2));
-
- ManagedCursor c1 = ledger.openCursor("c1");
- ledger.addEntry("dummy-entry-1".getBytes(Encoding));
- ManagedCursor c2 = ledger.openCursor("c2");
- ledger.addEntry("dummy-entry-2".getBytes(Encoding));
- ManagedCursor c3 = ledger.openCursor("c3");
- ledger.addEntry("dummy-entry-3".getBytes(Encoding));
- ManagedCursor c4 = ledger.openCursor("c4");
- ledger.addEntry("dummy-entry-4".getBytes(Encoding));
- ManagedCursor c5 = ledger.openCursor("c5");
-
- assertEquals(c1.getNumberOfEntries(), 4);
- assertEquals(c1.hasMoreEntries(), true);
-
- assertEquals(c2.getNumberOfEntries(), 3);
- assertEquals(c2.hasMoreEntries(), true);
-
- assertEquals(c3.getNumberOfEntries(), 2);
- assertEquals(c3.hasMoreEntries(), true);
-
- assertEquals(c4.getNumberOfEntries(), 1);
- assertEquals(c4.hasMoreEntries(), true);
-
- assertEquals(c5.getNumberOfEntries(), 0);
- assertEquals(c5.hasMoreEntries(), false);
-
- List<Entry> entries = c1.readEntries(2);
- assertEquals(entries.size(), 2);
- c1.markDelete(entries.get(1).getPosition());
- assertEquals(c1.getNumberOfEntries(), 2);
- entries.forEach(e -> e.release());
- }
-
- @Test(timeOut = 20000)
- void testNumberOfEntriesInBacklog() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger", new ManagedLedgerConfig().setMaxEntriesPerLedger(2));
-
- ManagedCursor c1 = ledger.openCursor("c1");
- Position p1 = ledger.addEntry("dummy-entry-1".getBytes(Encoding));
- ManagedCursor c2 = ledger.openCursor("c2");
- ledger.addEntry("dummy-entry-2".getBytes(Encoding));
- ManagedCursor c3 = ledger.openCursor("c3");
- Position p3 = ledger.addEntry("dummy-entry-3".getBytes(Encoding));
- ManagedCursor c4 = ledger.openCursor("c4");
- Position p4 = ledger.addEntry("dummy-entry-4".getBytes(Encoding));
- ManagedCursor c5 = ledger.openCursor("c5");
-
- assertEquals(c1.getNumberOfEntriesInBacklog(), 4);
- assertEquals(c2.getNumberOfEntriesInBacklog(), 3);
- assertEquals(c3.getNumberOfEntriesInBacklog(), 2);
- assertEquals(c4.getNumberOfEntriesInBacklog(), 1);
- assertEquals(c5.getNumberOfEntriesInBacklog(), 0);
-
- List<Entry> entries = c1.readEntries(2);
- assertEquals(entries.size(), 2);
- entries.forEach(e -> e.release());
-
- assertEquals(c1.getNumberOfEntries(), 2);
- assertEquals(c1.getNumberOfEntriesInBacklog(), 4);
-
- c1.markDelete(p1);
- assertEquals(c1.getNumberOfEntries(), 2);
- assertEquals(c1.getNumberOfEntriesInBacklog(), 3);
-
- c1.delete(p3);
-
- assertEquals(c1.getNumberOfEntries(), 1);
- assertEquals(c1.getNumberOfEntriesInBacklog(), 2);
-
- c1.markDelete(p4);
- assertEquals(c1.getNumberOfEntries(), 0);
- assertEquals(c1.getNumberOfEntriesInBacklog(), 0);
- }
-
- @Test(timeOut = 20000)
- void testNumberOfEntriesWithReopen() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger", new ManagedLedgerConfig().setMaxEntriesPerLedger(1));
-
- ManagedCursor c1 = ledger.openCursor("c1");
- ledger.addEntry("dummy-entry-1".getBytes(Encoding));
- ManagedCursor c2 = ledger.openCursor("c2");
- ledger.addEntry("dummy-entry-2".getBytes(Encoding));
- ManagedCursor c3 = ledger.openCursor("c3");
-
- ManagedLedgerFactory factory2 = new ManagedLedgerFactoryImpl(bkc, bkc.getZkHandle());
- ledger = factory2.open("my_test_ledger", new ManagedLedgerConfig().setMaxEntriesPerLedger(1));
-
- c1 = ledger.openCursor("c1");
- c2 = ledger.openCursor("c2");
- c3 = ledger.openCursor("c3");
-
- assertEquals(c1.getNumberOfEntries(), 2);
- assertEquals(c1.hasMoreEntries(), true);
-
- assertEquals(c2.getNumberOfEntries(), 1);
- assertEquals(c2.hasMoreEntries(), true);
-
- assertEquals(c3.getNumberOfEntries(), 0);
- assertEquals(c3.hasMoreEntries(), false);
-
- factory2.shutdown();
- }
-
- @Test(timeOut = 20000)
- void asyncReadWithoutErrors() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger");
- ManagedCursor cursor = ledger.openCursor("c1");
-
- ledger.addEntry("dummy-entry-1".getBytes(Encoding));
-
- final CountDownLatch counter = new CountDownLatch(1);
-
- cursor.asyncReadEntries(100, new ReadEntriesCallback() {
- @Override
- public void readEntriesComplete(List<Entry> entries, Object ctx) {
- assertNull(ctx);
- assertEquals(entries.size(), 1);
- entries.forEach(e -> e.release());
- counter.countDown();
- }
-
- @Override
- public void readEntriesFailed(ManagedLedgerException exception, Object ctx) {
- fail(exception.getMessage());
- }
-
- }, null);
-
- counter.await();
- }
-
- @Test(timeOut = 20000)
- void asyncReadWithErrors() throws Exception {
- ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("my_test_ledger");
- ManagedCursor cursor = ledger.openCursor("c1");
-
- ledger.addEntry("dummy-entry-1".getBytes(Encoding));
-
- final CountDownLatch counter = new CountDownLatch(1);
-
- stopBookKeeper();
-
- cursor.asyncReadEntries(100, new ReadEntriesCallback() {
- @Override
- public void readEntriesComplete(List<Entry> entries, Object ctx) {
- entries.forEach(e -> e.release());
- counter.countDown();
- }
-
- @Override
- public void readEntriesFailed(ManagedLedgerException exception, Object ctx) {
- fail("async-call should not have failed");
- }
-
- }, null);
-
- counter.await();
-
- cursor.rewind();
-
- // Clear the cache to force reading from BK
- ledger.entryCache.clear();
-
- final CountDownLatch counter2 = new CountDownLatch(1);
-
- cursor.asyncReadEntries(100, new ReadEntriesCallback() {
- @Override
- public void readEntriesComplete(List<Entry> entries, Object ctx) {
- fail("async-call should have failed");
- }
-
- @Override
- public void readEntriesFailed(ManagedLedgerException exception, Object ctx) {
- counter2.countDown();
- }
-
- }, null);
-
- counter2.await();
- }
-
- @Test(timeOut = 20000, expectedExceptions = IllegalArgumentException.class)
- void asyncReadWithInvalidParameter() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger");
- ManagedCursor cursor = ledger.openCursor("c1");
-
- ledger.addEntry("dummy-entry-1".getBytes(Encoding));
-
- final CountDownLatch counter = new CountDownLatch(1);
-
- stopBookKeeper();
-
- cursor.asyncReadEntries(0, new ReadEntriesCallback() {
- @Override
- public void readEntriesComplete(List<Entry> entries, Object ctx) {
- fail("async-call should have failed");
- }
-
- @Override
- public void readEntriesFailed(ManagedLedgerException exception, Object ctx) {
- counter.countDown();
- }
-
- }, null);
-
- counter.await();
- }
-
- @Test(timeOut = 20000)
- void markDeleteWithErrors() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger");
- ManagedCursor cursor = ledger.openCursor("c1");
- ledger.addEntry("dummy-entry-1".getBytes(Encoding));
- List<Entry> entries = cursor.readEntries(100);
-
- stopBookKeeper();
- assertEquals(entries.size(), 1);
-
- // Mark-delete should succeed if BK is down
- cursor.markDelete(entries.get(0).getPosition());
-
- entries.forEach(e -> e.release());
- }
-
- @Test(timeOut = 20000)
- void markDeleteWithZKErrors() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger");
- ManagedCursor cursor = ledger.openCursor("c1");
- ledger.addEntry("dummy-entry-1".getBytes(Encoding));
- List<Entry> entries = cursor.readEntries(100);
-
- assertEquals(entries.size(), 1);
-
- stopBookKeeper();
- stopZooKeeper();
-
- try {
- cursor.markDelete(entries.get(0).getPosition());
- fail("Should have failed");
- } catch (Exception e) {
- // Expected
- }
-
- entries.forEach(e -> e.release());
- }
-
- @Test(timeOut = 20000)
- void markDeleteAcrossLedgers() throws Exception {
- ManagedLedger ml1 = factory.open("my_test_ledger");
- ManagedCursor mc1 = ml1.openCursor("c1");
-
- // open ledger id 3 for ml1
- // markDeletePosition for mc1 is 3:-1
- // readPosition is 3:0
-
- ml1.close();
- mc1.close();
-
- // force removal of this ledger from the cache
- factory.close(ml1);
-
- ManagedLedger ml2 = factory.open("my_test_ledger");
- ManagedCursor mc2 = ml2.openCursor("c1");
-
- // open ledger id 5 for ml2
- // this entry is written at 5:0
- Position pos = ml2.addEntry("dummy-entry-1".getBytes(Encoding));
-
- List<Entry> entries = mc2.readEntries(1);
- assertEquals(entries.size(), 1);
- assertEquals(new String(entries.get(0).getData(), Encoding), "dummy-entry-1");
- entries.forEach(e -> e.release());
-
- mc2.delete(pos);
-
- // verify if the markDeletePosition moves from 3:-1 to 5:0
- assertEquals(mc2.getMarkDeletedPosition(), pos);
- assertEquals(mc2.getMarkDeletedPosition().getNext(), mc2.getReadPosition());
- }
-
- @Test(timeOut = 20000)
- void testResetCursor() throws Exception {
- ManagedLedger ledger = factory.open("my_test_move_cursor_ledger",
- new ManagedLedgerConfig().setMaxEntriesPerLedger(10));
- ManagedCursor cursor = ledger.openCursor("trc1");
- ledger.addEntry("dummy-entry-1".getBytes(Encoding));
- ledger.addEntry("dummy-entry-2".getBytes(Encoding));
- ledger.addEntry("dummy-entry-3".getBytes(Encoding));
- PositionImpl lastPosition = (PositionImpl) ledger.addEntry("dummy-entry-4".getBytes(Encoding));
- final AtomicBoolean moveStatus = new AtomicBoolean(false);
- PositionImpl resetPosition = new PositionImpl(lastPosition.getLedgerId(), lastPosition.getEntryId() - 2);
- try {
- cursor.resetCursor(resetPosition);
- moveStatus.set(true);
- } catch (Exception e) {
- log.warn("error in reset cursor", e.getCause());
- }
-
- assertTrue(moveStatus.get());
- assertTrue(cursor.getReadPosition().equals(resetPosition));
- cursor.close();
- ledger.close();
- }
-
- @Test(timeOut = 20000)
- void testasyncResetCursor() throws Exception {
- ManagedLedger ledger = factory.open("my_test_move_cursor_ledger",
- new ManagedLedgerConfig().setMaxEntriesPerLedger(10));
- ManagedCursor cursor = ledger.openCursor("tarc1");
- ledger.addEntry("dummy-entry-1".getBytes(Encoding));
- ledger.addEntry("dummy-entry-2".getBytes(Encoding));
- ledger.addEntry("dummy-entry-3".getBytes(Encoding));
- PositionImpl lastPosition = (PositionImpl) ledger.addEntry("dummy-entry-4".getBytes(Encoding));
- final AtomicBoolean moveStatus = new AtomicBoolean(false);
- CountDownLatch countDownLatch = new CountDownLatch(1);
- PositionImpl resetPosition = new PositionImpl(lastPosition.getLedgerId(), lastPosition.getEntryId() - 2);
-
- cursor.asyncResetCursor(resetPosition, new AsyncCallbacks.ResetCursorCallback() {
- @Override
- public void resetComplete(Object ctx) {
- moveStatus.set(true);
- countDownLatch.countDown();
- }
-
- @Override
- public void resetFailed(ManagedLedgerException exception, Object ctx) {
- moveStatus.set(false);
- countDownLatch.countDown();
- }
- });
- countDownLatch.await();
- assertTrue(moveStatus.get());
- assertTrue(cursor.getReadPosition().equals(resetPosition));
- cursor.close();
- ledger.close();
- }
-
- @Test(timeOut = 20000)
- void testConcurrentResetCursor() throws Exception {
- ManagedLedger ledger = factory.open("my_test_concurrent_move_ledger");
-
- final int Messages = 100;
- final int Consumers = 5;
-
- List<Future<AtomicBoolean>> futures = Lists.newArrayList();
- ExecutorService executor = Executors.newCachedThreadPool();
- final CyclicBarrier barrier = new CyclicBarrier(Consumers + 1);
-
- for (int i = 0; i < Messages; i++) {
- ledger.addEntry("test".getBytes());
- }
- final PositionImpl lastPosition = (PositionImpl) ledger.addEntry("dummy-entry-4".getBytes(Encoding));
-
- for (int i = 0; i < Consumers; i++) {
- final ManagedCursor cursor = ledger.openCursor("tcrc" + i);
- final int idx = i;
-
- futures.add(executor.submit(new Callable<AtomicBoolean>() {
- @Override
- public AtomicBoolean call() throws Exception {
- barrier.await();
-
- final AtomicBoolean moveStatus = new AtomicBoolean(false);
- CountDownLatch countDownLatch = new CountDownLatch(1);
- final PositionImpl resetPosition = new PositionImpl(lastPosition.getLedgerId(),
- lastPosition.getEntryId() - (5 * idx));
-
- cursor.asyncResetCursor(resetPosition, new AsyncCallbacks.ResetCursorCallback() {
- @Override
- public void resetComplete(Object ctx) {
- moveStatus.set(true);
- PositionImpl pos = (PositionImpl) ctx;
- log.info("move to [{}] completed for consumer [{}]", pos.toString(), idx);
- countDownLatch.countDown();
- }
-
- @Override
- public void resetFailed(ManagedLedgerException exception, Object ctx) {
- moveStatus.set(false);
- PositionImpl pos = (PositionImpl) ctx;
- log.warn("move to [{}] failed for consumer [{}]", pos.toString(), idx);
- countDownLatch.countDown();
- }
- });
- countDownLatch.await();
- assertTrue(cursor.getReadPosition().equals(resetPosition));
- cursor.close();
-
- return moveStatus;
- }
- }));
- }
-
- barrier.await();
-
- for (Future<AtomicBoolean> f : futures) {
- assertTrue(f.get().get());
- }
- ledger.close();
- }
-
- @Test(timeOut = 20000)
- void seekPosition() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger", new ManagedLedgerConfig().setMaxEntriesPerLedger(10));
- ManagedCursor cursor = ledger.openCursor("c1");
- ledger.addEntry("dummy-entry-1".getBytes(Encoding));
- ledger.addEntry("dummy-entry-2".getBytes(Encoding));
- ledger.addEntry("dummy-entry-3".getBytes(Encoding));
- PositionImpl lastPosition = (PositionImpl) ledger.addEntry("dummy-entry-4".getBytes(Encoding));
-
- cursor.seek(new PositionImpl(lastPosition.getLedgerId(), lastPosition.getEntryId() - 1));
- }
-
- @Test(timeOut = 20000)
- void seekPosition2() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger", new ManagedLedgerConfig().setMaxEntriesPerLedger(2));
- ManagedCursor cursor = ledger.openCursor("c1");
- ledger.addEntry("dummy-entry-1".getBytes(Encoding));
- ledger.addEntry("dummy-entry-2".getBytes(Encoding));
- PositionImpl seekPosition = (PositionImpl) ledger.addEntry("dummy-entry-3".getBytes(Encoding));
- ledger.addEntry("dummy-entry-4".getBytes(Encoding));
- ledger.addEntry("dummy-entry-5".getBytes(Encoding));
- ledger.addEntry("dummy-entry-6".getBytes(Encoding));
-
- cursor.seek(new PositionImpl(seekPosition.getLedgerId(), seekPosition.getEntryId()));
- }
-
- @Test(timeOut = 20000)
- void seekPosition3() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger", new ManagedLedgerConfig().setMaxEntriesPerLedger(2));
- ManagedCursor cursor = ledger.openCursor("c1");
- ledger.addEntry("dummy-entry-1".getBytes(Encoding));
- ledger.addEntry("dummy-entry-2".getBytes(Encoding));
- ledger.addEntry("dummy-entry-3".getBytes(Encoding));
- PositionImpl seekPosition = (PositionImpl) ledger.addEntry("dummy-entry-4".getBytes(Encoding));
- Position entry5 = ledger.addEntry("dummy-entry-5".getBytes(Encoding));
- Position entry6 = ledger.addEntry("dummy-entry-6".getBytes(Encoding));
-
- cursor.seek(new PositionImpl(seekPosition.getLedgerId(), seekPosition.getEntryId()));
-
- assertEquals(cursor.getReadPosition(), seekPosition);
- List<Entry> entries = cursor.readEntries(1);
- assertEquals(entries.size(), 1);
- assertEquals(new String(entries.get(0).getData(), Encoding), "dummy-entry-4");
- entries.forEach(e -> e.release());
-
- cursor.seek(entry5.getNext());
- assertEquals(cursor.getReadPosition(), entry6);
- entries = cursor.readEntries(1);
- assertEquals(entries.size(), 1);
- assertEquals(new String(entries.get(0).getData(), Encoding), "dummy-entry-6");
- entries.forEach(e -> e.release());
- }
-
- @Test(timeOut = 20000)
- void seekPosition4() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger");
- ManagedCursor cursor = ledger.openCursor("c1");
- Position p1 = ledger.addEntry("dummy-entry-1".getBytes(Encoding));
- Position p2 = ledger.addEntry("dummy-entry-2".getBytes(Encoding));
- ledger.addEntry("dummy-entry-3".getBytes(Encoding));
- ledger.addEntry("dummy-entry-4".getBytes(Encoding));
- ledger.addEntry("dummy-entry-5".getBytes(Encoding));
- ledger.addEntry("dummy-entry-6".getBytes(Encoding));
-
- cursor.markDelete(p1);
- assertEquals(cursor.getMarkDeletedPosition(), p1);
- assertEquals(cursor.getReadPosition(), p2);
-
- List<Entry> entries = cursor.readEntries(2);
- entries.forEach(e -> e.release());
-
- cursor.seek(p2);
- assertEquals(cursor.getMarkDeletedPosition(), p1);
- assertEquals(cursor.getReadPosition(), p2);
- }
-
- @Test(timeOut = 20000)
- void rewind() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger", new ManagedLedgerConfig().setMaxEntriesPerLedger(2));
- ManagedCursor c1 = ledger.openCursor("c1");
- Position p1 = ledger.addEntry("dummy-entry-1".getBytes(Encoding));
- Position p2 = ledger.addEntry("dummy-entry-2".getBytes(Encoding));
- Position p3 = ledger.addEntry("dummy-entry-3".getBytes(Encoding));
- Position p4 = ledger.addEntry("dummy-entry-4".getBytes(Encoding));
-
- log.debug("p1: {}", p1);
- log.debug("p2: {}", p2);
- log.debug("p3: {}", p3);
- log.debug("p4: {}", p4);
-
- assertEquals(c1.getNumberOfEntries(), 4);
- assertEquals(c1.getNumberOfEntriesInBacklog(), 4);
- c1.markDelete(p1);
- assertEquals(c1.getNumberOfEntries(), 3);
- assertEquals(c1.getNumberOfEntriesInBacklog(), 3);
- List<Entry> entries = c1.readEntries(10);
- assertEquals(entries.size(), 3);
- entries.forEach(e -> e.release());
-
- assertEquals(c1.getNumberOfEntries(), 0);
- assertEquals(c1.getNumberOfEntriesInBacklog(), 3);
- c1.rewind();
- assertEquals(c1.getNumberOfEntries(), 3);
- assertEquals(c1.getNumberOfEntriesInBacklog(), 3);
- c1.markDelete(p2);
- assertEquals(c1.getNumberOfEntries(), 2);
- assertEquals(c1.getNumberOfEntriesInBacklog(), 2);
-
- entries = c1.readEntries(10);
- assertEquals(entries.size(), 2);
- entries.forEach(e -> e.release());
-
- assertEquals(c1.getNumberOfEntries(), 0);
- assertEquals(c1.getNumberOfEntriesInBacklog(), 2);
- c1.rewind();
- assertEquals(c1.getNumberOfEntries(), 2);
- c1.markDelete(p4);
- assertEquals(c1.getNumberOfEntries(), 0);
- assertEquals(c1.getNumberOfEntriesInBacklog(), 0);
- c1.rewind();
- assertEquals(c1.getNumberOfEntries(), 0);
- ledger.addEntry("dummy-entry-5".getBytes(Encoding));
- assertEquals(c1.getNumberOfEntries(), 1);
- ledger.addEntry("dummy-entry-6".getBytes(Encoding));
- assertEquals(c1.getNumberOfEntries(), 2);
- }
-
- @Test(timeOut = 20000)
- void markDeleteSkippingMessage() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger", new ManagedLedgerConfig().setMaxEntriesPerLedger(10));
- ManagedCursor cursor = ledger.openCursor("c1");
- Position p1 = ledger.addEntry("dummy-entry-1".getBytes(Encoding));
- Position p2 = ledger.addEntry("dummy-entry-2".getBytes(Encoding));
- ledger.addEntry("dummy-entry-3".getBytes(Encoding));
- PositionImpl p4 = (PositionImpl) ledger.addEntry("dummy-entry-4".getBytes(Encoding));
-
- assertEquals(cursor.getNumberOfEntries(), 4);
-
- cursor.markDelete(p1);
- assertEquals(cursor.hasMoreEntries(), true);
- assertEquals(cursor.getNumberOfEntries(), 3);
-
- assertEquals(cursor.getReadPosition(), p2);
-
- List<Entry> entries = cursor.readEntries(1);
- assertEquals(entries.size(), 1);
- assertEquals(new String(entries.get(0).getData(), Encoding), "dummy-entry-2");
- entries.forEach(e -> e.release());
-
- cursor.markDelete(p4);
- assertEquals(cursor.hasMoreEntries(), false);
- assertEquals(cursor.getNumberOfEntries(), 0);
-
- assertEquals(cursor.getReadPosition(), new PositionImpl(p4.getLedgerId(), p4.getEntryId() + 1));
- }
-
- @Test(timeOut = 20000)
- void removingCursor() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger", new ManagedLedgerConfig().setMaxEntriesPerLedger(1));
- ManagedCursor cursor = ledger.openCursor("c1");
- ledger.addEntry("dummy-entry-1".getBytes(Encoding));
- ledger.addEntry("dummy-entry-2".getBytes(Encoding));
- ledger.addEntry("dummy-entry-3".getBytes(Encoding));
- ledger.addEntry("dummy-entry-4".getBytes(Encoding));
- ledger.addEntry("dummy-entry-5".getBytes(Encoding));
- ledger.addEntry("dummy-entry-6".getBytes(Encoding));
-
- assertEquals(cursor.getNumberOfEntries(), 6);
- assertEquals(ledger.getNumberOfEntries(), 6);
- ledger.deleteCursor("c1");
-
- // Verify that it's a new empty cursor
- cursor = ledger.openCursor("c1");
- assertEquals(cursor.getNumberOfEntries(), 0);
- ledger.addEntry("dummy-entry-7".getBytes(Encoding));
-
- // Verify that GC trimming kicks in
- while (ledger.getNumberOfEntries() > 2) {
- Thread.sleep(10);
- }
- }
-
- @Test(timeOut = 20000)
- void cursorPersistence() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger");
- ManagedCursor c1 = ledger.openCursor("c1");
- ManagedCursor c2 = ledger.openCursor("c2");
- ledger.addEntry("dummy-entry-1".getBytes(Encoding));
- ledger.addEntry("dummy-entry-2".getBytes(Encoding));
- ledger.addEntry("dummy-entry-3".getBytes(Encoding));
- ledger.addEntry("dummy-entry-4".getBytes(Encoding));
- ledger.addEntry("dummy-entry-5".getBytes(Encoding));
- ledger.addEntry("dummy-entry-6".getBytes(Encoding));
-
- List<Entry> entries = c1.readEntries(3);
- Position p1 = entries.get(2).getPosition();
- c1.markDelete(p1);
- entries.forEach(e -> e.release());
-
- entries = c1.readEntries(4);
- Position p2 = entries.get(2).getPosition();
- c2.markDelete(p2);
- entries.forEach(e -> e.release());
-
- // Reopen
-
- ManagedLedgerFactory factory2 = new ManagedLedgerFactoryImpl(bkc, bkc.getZkHandle());
- ledger = factory2.open("my_test_ledger");
- c1 = ledger.openCursor("c1");
- c2 = ledger.openCursor("c2");
-
- assertEquals(c1.getMarkDeletedPosition(), p1);
- assertEquals(c2.getMarkDeletedPosition(), p2);
- factory2.shutdown();
- }
-
- @Test(timeOut = 20000)
- void cursorPersistence2() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger",
- new ManagedLedgerConfig().setMetadataMaxEntriesPerLedger(1));
- ManagedCursor c1 = ledger.openCursor("c1");
- ManagedCursor c2 = ledger.openCursor("c2");
- ManagedCursor c3 = ledger.openCursor("c3");
- Position p0 = c3.getMarkDeletedPosition();
- Position p1 = ledger.addEntry("dummy-entry-1".getBytes(Encoding));
- ManagedCursor c4 = ledger.openCursor("c4");
- Position p2 = ledger.addEntry("dummy-entry-2".getBytes(Encoding));
- Position p3 = ledger.addEntry("dummy-entry-3".getBytes(Encoding));
- Position p4 = ledger.addEntry("dummy-entry-4".getBytes(Encoding));
- Position p5 = ledger.addEntry("dummy-entry-5".getBytes(Encoding));
- ledger.addEntry("dummy-entry-6".getBytes(Encoding));
-
- c1.markDelete(p1);
- c1.markDelete(p2);
- c1.markDelete(p3);
- c1.markDelete(p4);
- c1.markDelete(p5);
-
- c2.markDelete(p1);
-
- // Reopen
-
- ManagedLedgerFactory factory2 = new ManagedLedgerFactoryImpl(bkc, bkc.getZkHandle());
- ledger = factory.open("my_test_ledger");
- c1 = ledger.openCursor("c1");
- c2 = ledger.openCursor("c2");
- c3 = ledger.openCursor("c3");
- c4 = ledger.openCursor("c4");
-
- assertEquals(c1.getMarkDeletedPosition(), p5);
- assertEquals(c2.getMarkDeletedPosition(), p1);
- assertEquals(c3.getMarkDeletedPosition(), p0);
- assertEquals(c4.getMarkDeletedPosition(), p1);
- factory2.shutdown();
- }
-
- @Test(timeOut = 20000)
- public void asyncMarkDeleteBlocking() throws Exception {
- ManagedLedgerConfig config = new ManagedLedgerConfig();
- config.setMaxEntriesPerLedger(10);
- config.setMetadataMaxEntriesPerLedger(5);
- ManagedLedger ledger = factory.open("my_test_ledger", config);
- final ManagedCursor c1 = ledger.openCursor("c1");
- final AtomicReference<Position> lastPosition = new AtomicReference<Position>();
-
- final int N = 100;
- final CountDownLatch latch = new CountDownLatch(N);
- for (int i = 0; i < N; i++) {
- ledger.asyncAddEntry("entry".getBytes(Encoding), new AddEntryCallback() {
- @Override
- public void addFailed(ManagedLedgerException exception, Object ctx) {
- }
-
- @Override
- public void addComplete(Position position, Object ctx) {
- lastPosition.set(position);
- c1.asyncMarkDelete(position, new MarkDeleteCallback() {
- @Override
- public void markDeleteFailed(ManagedLedgerException exception, Object ctx) {
- }
-
- @Override
- public void markDeleteComplete(Object ctx) {
- latch.countDown();
- }
- }, null);
- }
- }, null);
- }
-
- latch.await();
-
- assertEquals(c1.getNumberOfEntries(), 0);
-
- // Reopen
- ManagedLedgerFactory factory2 = new ManagedLedgerFactoryImpl(bkc, bkc.getZkHandle());
- ledger = factory2.open("my_test_ledger");
- ManagedCursor c2 = ledger.openCursor("c1");
-
- assertEquals(c2.getMarkDeletedPosition(), lastPosition.get());
- factory2.shutdown();
- }
-
- @Test(timeOut = 20000)
- void cursorPersistenceAsyncMarkDeleteSameThread() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger",
- new ManagedLedgerConfig().setMetadataMaxEntriesPerLedger(5));
- final ManagedCursor c1 = ledger.openCursor("c1");
-
- final int N = 100;
- List<Position> positions = Lists.newArrayList();
- for (int i = 0; i < N; i++) {
- Position p = ledger.addEntry("dummy-entry".getBytes(Encoding));
- positions.add(p);
- }
-
- Position lastPosition = positions.get(N - 1);
-
- final CountDownLatch latch = new CountDownLatch(N);
- for (final Position p : positions) {
- c1.asyncMarkDelete(p, new MarkDeleteCallback() {
- @Override
- public void markDeleteComplete(Object ctx) {
- latch.countDown();
- }
-
- @Override
- public void markDeleteFailed(ManagedLedgerException exception, Object ctx) {
- log.error("Failed to markdelete", exception);
- latch.countDown();
- }
- }, null);
- }
-
- latch.await();
-
- // Reopen
- ManagedLedgerFactory factory2 = new ManagedLedgerFactoryImpl(bkc, bkc.getZkHandle());
- ledger = factory2.open("my_test_ledger");
- ManagedCursor c2 = ledger.openCursor("c1");
-
- assertEquals(c2.getMarkDeletedPosition(), lastPosition);
- factory2.shutdown();
- }
-
- @Test(timeOut = 20000)
- void unorderedMarkDelete() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger");
- final ManagedCursor c1 = ledger.openCursor("c1");
-
- Position p1 = ledger.addEntry("entry-1".getBytes(Encoding));
- Position p2 = ledger.addEntry("entry-2".getBytes(Encoding));
-
- c1.markDelete(p2);
- try {
- c1.markDelete(p1);
- fail("Should have thrown exception");
- } catch (ManagedLedgerException e) {
- // ok
- }
-
- assertEquals(c1.getMarkDeletedPosition(), p2);
- }
-
- @Test(timeOut = 20000)
- void unorderedAsyncMarkDelete() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger");
- final ManagedCursor c1 = ledger.openCursor("c1");
-
- Position p1 = ledger.addEntry("entry-1".getBytes(Encoding));
- Position p2 = ledger.addEntry("entry-2".getBytes(Encoding));
-
- final CountDownLatch latch = new CountDownLatch(2);
- c1.asyncMarkDelete(p2, new MarkDeleteCallback() {
- @Override
- public void markDeleteFailed(ManagedLedgerException exception, Object ctx) {
- fail();
- }
-
- @Override
- public void markDeleteComplete(Object ctx) {
- latch.countDown();
- }
- }, null);
-
- c1.asyncMarkDelete(p1, new MarkDeleteCallback() {
- @Override
- public void markDeleteFailed(ManagedLedgerException exception, Object ctx) {
- latch.countDown();
- }
-
- @Override
- public void markDeleteComplete(Object ctx) {
- fail();
- }
- }, null);
-
- latch.await();
-
- assertEquals(c1.getMarkDeletedPosition(), p2);
- }
-
- @Test(timeOut = 20000)
- void deleteCursor() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger");
- ManagedCursor c1 = ledger.openCursor("c1");
-
- ledger.addEntry("entry-1".getBytes(Encoding));
- Position p2 = ledger.addEntry("entry-2".getBytes(Encoding));
-
- assertEquals(c1.getNumberOfEntries(), 2);
-
- // Remove and recreate the same cursor
- ledger.deleteCursor("c1");
-
- try {
- c1.readEntries(10);
- fail("must fail, the cursor should be closed");
- } catch (ManagedLedgerException e) {
- // ok
- }
-
- try {
- c1.markDelete(p2);
- fail("must fail, the cursor should be closed");
- } catch (ManagedLedgerException e) {
- // ok
- }
-
- c1 = ledger.openCursor("c1");
- assertEquals(c1.getNumberOfEntries(), 0);
-
- c1.close();
- try {
- c1.readEntries(10);
- fail("must fail, the cursor should be closed");
- } catch (ManagedLedgerException e) {
- // ok
- }
-
- c1.close();
- }
-
- @Test(timeOut = 20000)
- void errorCreatingCursor() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger");
-
- bkc.failAfter(1, BKException.Code.NotEnoughBookiesException);
- zkc.failNow(Code.SESSIONEXPIRED);
- try {
- ledger.openCursor("c1");
- fail("should have failed");
- } catch (ManagedLedgerException e) {
- // ok
- }
- }
-
- @Test(timeOut = 20000)
- void errorRecoveringCursor() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger");
- Position p1 = ledger.addEntry("entry".getBytes());
- ledger.addEntry("entry".getBytes());
- ManagedCursor c1 = ledger.openCursor("c1");
- Position p3 = ledger.addEntry("entry".getBytes());
-
- assertEquals(c1.getReadPosition(), p3);
-
- ManagedLedgerFactory factory2 = new ManagedLedgerFactoryImpl(bkc, bkc.getZkHandle());
-
- bkc.failAfter(3, BKException.Code.LedgerRecoveryException);
-
- ledger = factory2.open("my_test_ledger");
- c1 = ledger.openCursor("c1");
-
- // Verify the ManagedCursor was rewind back to the snapshotted position
- assertEquals(c1.getReadPosition(), p3);
- factory2.shutdown();
- }
-
- @Test(timeOut = 20000)
- void errorRecoveringCursor2() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger");
- ledger.openCursor("c1");
-
- ManagedLedgerFactory factory2 = new ManagedLedgerFactoryImpl(bkc, bkc.getZkHandle());
-
- bkc.failAfter(4, BKException.Code.MetadataVersionException);
-
- try {
- ledger = factory2.open("my_test_ledger");
- fail("should have failed");
- } catch (ManagedLedgerException e) {
- // ok
- }
-
- factory2.shutdown();
- }
-
- @Test(timeOut = 20000)
- void errorRecoveringCursor3() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger");
- Position p1 = ledger.addEntry("entry".getBytes());
- ledger.addEntry("entry".getBytes());
- ManagedCursor c1 = ledger.openCursor("c1");
- Position p3 = ledger.addEntry("entry".getBytes());
-
- assertEquals(c1.getReadPosition(), p3);
-
- ManagedLedgerFactory factory2 = new ManagedLedgerFactoryImpl(bkc, bkc.getZkHandle());
-
- bkc.failAfter(4, BKException.Code.ReadException);
-
- ledger = factory2.open("my_test_ledger");
- c1 = ledger.openCursor("c1");
-
- // Verify the ManagedCursor was rewind back to the snapshotted position
- assertEquals(c1.getReadPosition(), p3);
- factory2.shutdown();
- }
-
- @Test(timeOut = 20000)
- void testSingleDelete() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger", new ManagedLedgerConfig().setMaxEntriesPerLedger(3));
- ManagedCursor cursor = ledger.openCursor("c1");
-
- Position p1 = ledger.addEntry("entry1".getBytes());
- Position p2 = ledger.addEntry("entry2".getBytes());
- Position p3 = ledger.addEntry("entry3".getBytes());
- Position p4 = ledger.addEntry("entry4".getBytes());
- Position p5 = ledger.addEntry("entry5".getBytes());
- Position p6 = ledger.addEntry("entry6".getBytes());
-
- Position p0 = cursor.getMarkDeletedPosition();
-
- cursor.delete(p4);
- assertEquals(cursor.getMarkDeletedPosition(), p0);
-
- cursor.delete(p1);
- assertEquals(cursor.getMarkDeletedPosition(), p1);
-
- cursor.delete(p3);
-
- // Delete will silently succeed
- cursor.delete(p3);
- assertEquals(cursor.getMarkDeletedPosition(), p1);
-
- cursor.delete(p2);
- assertEquals(cursor.getMarkDeletedPosition(), p4);
-
- cursor.delete(p5);
- assertEquals(cursor.getMarkDeletedPosition(), p5);
-
- cursor.close();
- try {
- cursor.delete(p6);
- } catch (ManagedLedgerException e) {
- // Ok
- }
- }
-
- @Test(timeOut = 20000)
- void testFilteringReadEntries() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger", new ManagedLedgerConfig().setMaxEntriesPerLedger(3));
- ManagedCursor cursor = ledger.openCursor("c1");
-
- /* Position p1 = */ledger.addEntry("entry1".getBytes());
- /* Position p2 = */ledger.addEntry("entry2".getBytes());
- /* Position p3 = */ledger.addEntry("entry3".getBytes());
- /* Position p4 = */ledger.addEntry("entry4".getBytes());
- Position p5 = ledger.addEntry("entry5".getBytes());
- /* Position p6 = */ledger.addEntry("entry6".getBytes());
-
- assertEquals(cursor.getNumberOfEntries(), 6);
- assertEquals(cursor.getNumberOfEntriesInBacklog(), 6);
-
- List<Entry> entries = cursor.readEntries(3);
- assertEquals(entries.size(), 3);
- entries.forEach(e -> e.release());
-
- assertEquals(cursor.getNumberOfEntries(), 3);
- assertEquals(cursor.getNumberOfEntriesInBacklog(), 6);
-
- log.info("Deleting {}", p5);
- cursor.delete(p5);
-
- assertEquals(cursor.getNumberOfEntries(), 2);
- assertEquals(cursor.getNumberOfEntriesInBacklog(), 5);
-
- entries = cursor.readEntries(3);
- assertEquals(entries.size(), 2);
- entries.forEach(e -> e.release());
- assertEquals(cursor.getNumberOfEntries(), 0);
- assertEquals(cursor.getNumberOfEntriesInBacklog(), 5);
- }
-
- @Test(timeOut = 20000)
- void testReadingAllFilteredEntries() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger", new ManagedLedgerConfig().setMaxEntriesPerLedger(3));
- ledger.openCursor("c1");
- ManagedCursor c2 = ledger.openCursor("c2");
-
- ledger.addEntry("entry1".getBytes());
- Position p2 = ledger.addEntry("entry2".getBytes());
- Position p3 = ledger.addEntry("entry3".getBytes());
- Position p4 = ledger.addEntry("entry4".getBytes());
- Position p5 = ledger.addEntry("entry5".getBytes());
-
- c2.readEntries(1).get(0).release();
- c2.delete(p2);
- c2.delete(p3);
-
- List<Entry> entries = c2.readEntries(2);
- assertEquals(entries.size(), 2);
- assertEquals(entries.get(0).getPosition(), p4);
- assertEquals(entries.get(1).getPosition(), p5);
- entries.forEach(e -> e.release());
- }
-
- @Test(timeOut = 20000)
- void testCountingWithDeletedEntries() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger", new ManagedLedgerConfig().setMaxEntriesPerLedger(2));
- ManagedCursor cursor = ledger.openCursor("c1");
-
- Position p1 = ledger.addEntry("entry1".getBytes());
- /* Position p2 = */ledger.addEntry("entry2".getBytes());
- /* Position p3 = */ledger.addEntry("entry3".getBytes());
- /* Position p4 = */ledger.addEntry("entry4".getBytes());
- Position p5 = ledger.addEntry("entry5".getBytes());
- Position p6 = ledger.addEntry("entry6".getBytes());
- Position p7 = ledger.addEntry("entry7".getBytes());
- Position p8 = ledger.addEntry("entry8".getBytes());
-
- assertEquals(cursor.getNumberOfEntries(), 8);
- assertEquals(cursor.getNumberOfEntriesInBacklog(), 8);
-
- cursor.delete(p8);
- assertEquals(cursor.getNumberOfEntries(), 7);
- assertEquals(cursor.getNumberOfEntriesInBacklog(), 7);
-
- cursor.delete(p1);
- assertEquals(cursor.getNumberOfEntries(), 6);
- assertEquals(cursor.getNumberOfEntriesInBacklog(), 6);
-
- cursor.delete(p7);
- cursor.delete(p6);
- cursor.delete(p5);
- assertEquals(cursor.getNumberOfEntries(), 3);
- assertEquals(cursor.getNumberOfEntriesInBacklog(), 3);
- }
-
- @Test(timeOut = 20000)
- void testMarkDeleteTwice() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger", new ManagedLedgerConfig().setMaxEntriesPerLedger(2));
- ManagedCursor cursor = ledger.openCursor("c1");
-
- Position p1 = ledger.addEntry("entry1".getBytes());
- cursor.markDelete(p1);
- cursor.markDelete(p1);
-
- assertEquals(cursor.getMarkDeletedPosition(), p1);
- }
-
- @Test(timeOut = 20000)
- void testSkipEntries() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger", new ManagedLedgerConfig().setMaxEntriesPerLedger(2));
- Position pos;
-
- ManagedCursor c1 = ledger.openCursor("c1");
-
- // test skip on empty ledger
- pos = c1.getReadPosition();
- c1.skipEntries(1, IndividualDeletedEntries.Exclude);
- assertEquals(c1.getReadPosition(), pos);
-
- pos = ledger.addEntry("dummy-entry-1".getBytes(Encoding));
- pos = ledger.addEntry("dummy-entry-2".getBytes(Encoding));
-
- // skip entries in same ledger
- c1.skipEntries(1, IndividualDeletedEntries.Exclude);
- assertEquals(c1.getNumberOfEntries(), 1);
-
- // skip entries until end of ledger
- c1.skipEntries(1, IndividualDeletedEntries.Exclude);
- assertEquals(c1.getNumberOfEntries(), 0);
- assertEquals(c1.getReadPosition(), pos.getNext());
- assertEquals(c1.getMarkDeletedPosition(), pos);
-
- // skip entries across ledgers
- for (int i = 0; i < 6; i++) {
- pos = ledger.addEntry("dummy-entry".getBytes(Encoding));
- }
-
- c1.skipEntries(5, IndividualDeletedEntries.Exclude);
- assertEquals(c1.getNumberOfEntries(), 1);
-
- // skip more than the current set of entries
- c1.skipEntries(10, IndividualDeletedEntries.Exclude);
- assertEquals(c1.getNumberOfEntries(), 0);
- assertEquals(c1.hasMoreEntries(), false);
- assertEquals(c1.getReadPosition(), pos.getNext());
- assertEquals(c1.getMarkDeletedPosition(), pos);
- }
-
- @Test(timeOut = 20000)
- void testSkipEntriesWithIndividualDeletedMessages() throws Exception {
- ManagedLedger ledger = factory.open("testSkipEntriesWithIndividualDeletedMessages",
- new ManagedLedgerConfig().setMaxEntriesPerLedger(5));
- ManagedCursor c1 = ledger.openCursor("c1");
-
- Position pos1 = ledger.addEntry("dummy-entry-1".getBytes(Encoding));
- Position pos2 = ledger.addEntry("dummy-entry-2".getBytes(Encoding));
- Position pos3 = ledger.addEntry("dummy-entry-3".getBytes(Encoding));
- Position pos4 = ledger.addEntry("dummy-entry-4".getBytes(Encoding));
- Position pos5 = ledger.addEntry("dummy-entry-5".getBytes(Encoding));
-
- // delete individual messages
- c1.delete(pos2);
- c1.delete(pos4);
-
- c1.skipEntries(3, IndividualDeletedEntries.Exclude);
- assertEquals(c1.getNumberOfEntries(), 0);
- assertEquals(c1.getReadPosition(), pos5.getNext());
- assertEquals(c1.getMarkDeletedPosition(), pos5);
-
- pos1 = ledger.addEntry("dummy-entry-1".getBytes(Encoding));
- pos2 = ledger.addEntry("dummy-entry-2".getBytes(Encoding));
- pos3 = ledger.addEntry("dummy-entry-3".getBytes(Encoding));
- pos4 = ledger.addEntry("dummy-entry-4".getBytes(Encoding));
- pos5 = ledger.addEntry("dummy-entry-5".getBytes(Encoding));
-
- c1.delete(pos2);
- c1.delete(pos4);
-
- c1.skipEntries(4, IndividualDeletedEntries.Include);
- assertEquals(c1.getNumberOfEntries(), 1);
- assertEquals(c1.getReadPosition(), pos5);
- assertEquals(c1.getMarkDeletedPosition(), pos4);
- }
-
- @Test(timeOut = 20000)
- void testClearBacklog() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger", new ManagedLedgerConfig().setMaxEntriesPerLedger(1));
-
- ManagedCursor c1 = ledger.openCursor("c1");
- ledger.addEntry("dummy-entry-1".getBytes(Encoding));
- ManagedCursor c2 = ledger.openCursor("c2");
- ledger.addEntry("dummy-entry-2".getBytes(Encoding));
- ManagedCursor c3 = ledger.openCursor("c3");
- ledger.addEntry("dummy-entry-3".getBytes(Encoding));
-
- assertEquals(c1.getNumberOfEntriesInBacklog(), 3);
- assertEquals(c1.getNumberOfEntries(), 3);
- assertEquals(c1.hasMoreEntries(), true);
-
- c1.clearBacklog();
- c3.clearBacklog();
-
- assertEquals(c1.getNumberOfEntriesInBacklog(), 0);
- assertEquals(c1.getNumberOfEntries(), 0);
- assertEquals(c1.hasMoreEntries(), false);
-
- assertEquals(c2.getNumberOfEntriesInBacklog(), 2);
- assertEquals(c2.getNumberOfEntries(), 2);
- assertEquals(c2.hasMoreEntries(), true);
-
- assertEquals(c3.getNumberOfEntriesInBacklog(), 0);
- assertEquals(c3.getNumberOfEntries(), 0);
- assertEquals(c3.hasMoreEntries(), false);
-
- ManagedLedgerFactory factory2 = new ManagedLedgerFactoryImpl(bkc, bkc.getZkHandle());
- ledger = factory2.open("my_test_ledger", new ManagedLedgerConfig().setMaxEntriesPerLedger(1));
-
- c1 = ledger.openCursor("c1");
- c2 = ledger.openCursor("c2");
- c3 = ledger.openCursor("c3");
-
- assertEquals(c1.getNumberOfEntriesInBacklog(), 0);
- assertEquals(c1.getNumberOfEntries(), 0);
- assertEquals(c1.hasMoreEntries(), false);
-
- assertEquals(c2.getNumberOfEntriesInBacklog(), 2);
- assertEquals(c2.getNumberOfEntries(), 2);
- assertEquals(c2.hasMoreEntries(), true);
-
- assertEquals(c3.getNumberOfEntriesInBacklog(), 0);
- assertEquals(c3.getNumberOfEntries(), 0);
- assertEquals(c3.hasMoreEntries(), false);
- factory2.shutdown();
- }
-
- @Test(timeOut = 20000)
- void testRateLimitMarkDelete() throws Exception {
- ManagedLedgerConfig config = new ManagedLedgerConfig();
- config.setThrottleMarkDelete(1); // Throttle to 1/s
- ManagedLedger ledger = factory.open("my_test_ledger", config);
-
- ManagedCursor c1 = ledger.openCursor("c1");
- Position p1 = ledger.addEntry("dummy-entry-1".getBytes(Encoding));
- Position p2 = ledger.addEntry("dummy-entry-2".getBytes(Encoding));
- Position p3 = ledger.addEntry("dummy-entry-3".getBytes(Encoding));
-
- assertEquals(c1.getNumberOfEntriesInBacklog(), 3);
- c1.markDelete(p1);
- c1.markDelete(p2);
- c1.markDelete(p3);
-
- assertEquals(c1.getNumberOfEntriesInBacklog(), 0);
-
- // Re-open to recover from storage
- ManagedLedgerFactory factory2 = new ManagedLedgerFactoryImpl(bkc, bkc.getZkHandle());
- ledger = factory2.open("my_test_ledger", new ManagedLedgerConfig());
-
- c1 = ledger.openCursor("c1");
-
- // Only the 1st mark-delete was persisted
- assertEquals(c1.getNumberOfEntriesInBacklog(), 2);
- factory2.shutdown();
- }
-
- @Test(timeOut = 20000)
- void deleteSingleMessageTwice() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger");
-
- ManagedCursor c1 = ledger.openCursor("c1");
-
- Position p1 = ledger.addEntry("entry-1".getBytes(Encoding));
- Position p2 = ledger.addEntry("entry-2".getBytes(Encoding));
- Position p3 = ledger.addEntry("entry-3".getBytes(Encoding));
- Position p4 = ledger.addEntry("entry-4".getBytes(Encoding));
-
- assertEquals(c1.getNumberOfEntriesInBacklog(), 4);
- assertEquals(c1.getNumberOfEntries(), 4);
-
- c1.delete(p1);
- assertEquals(c1.getNumberOfEntriesInBacklog(), 3);
- assertEquals(c1.getNumberOfEntries(), 3);
- assertEquals(c1.getMarkDeletedPosition(), p1);
- assertEquals(c1.getReadPosition(), p2);
-
- // Should have not effect since p1 is already deleted
- c1.delete(p1);
- assertEquals(c1.getNumberOfEntriesInBacklog(), 3);
- assertEquals(c1.getNumberOfEntries(), 3);
- assertEquals(c1.getMarkDeletedPosition(), p1);
- assertEquals(c1.getReadPosition(), p2);
-
- c1.delete(p2);
- assertEquals(c1.getNumberOfEntriesInBacklog(), 2);
- assertEquals(c1.getNumberOfEntries(), 2);
- assertEquals(c1.getMarkDeletedPosition(), p2);
- assertEquals(c1.getReadPosition(), p3);
-
- // Should have not effect since p2 is already deleted
- c1.delete(p2);
- assertEquals(c1.getNumberOfEntriesInBacklog(), 2);
- assertEquals(c1.getNumberOfEntries(), 2);
- assertEquals(c1.getMarkDeletedPosition(), p2);
- assertEquals(c1.getReadPosition(), p3);
-
- c1.delete(p3);
- assertEquals(c1.getNumberOfEntriesInBacklog(), 1);
- assertEquals(c1.getNumberOfEntries(), 1);
- assertEquals(c1.getMarkDeletedPosition(), p3);
- assertEquals(c1.getReadPosition(), p4);
-
- // Should have not effect since p3 is already deleted
- c1.delete(p3);
- assertEquals(c1.getNumberOfEntriesInBacklog(), 1);
- assertEquals(c1.getNumberOfEntries(), 1);
- assertEquals(c1.getMarkDeletedPosition(), p3);
- assertEquals(c1.getReadPosition(), p4);
-
- c1.delete(p4);
- assertEquals(c1.getNumberOfEntriesInBacklog(), 0);
- assertEquals(c1.getNumberOfEntries(), 0);
- assertEquals(c1.getMarkDeletedPosition(), p4);
- assertEquals(c1.getReadPosition(), p4.getNext());
-
- // Should have not effect since p4 is already deleted
- c1.delete(p4);
- assertEquals(c1.getNumberOfEntriesInBacklog(), 0);
- assertEquals(c1.getNumberOfEntries(), 0);
- assertEquals(c1.getMarkDeletedPosition(), p4);
- assertEquals(c1.getReadPosition(), p4.getNext());
- }
-
- @Test(timeOut = 10000)
- void testReadEntriesOrWait() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger");
-
- final int Consumers = 10;
- final CountDownLatch counter = new CountDownLatch(Consumers);
-
- for (int i = 0; i < Consumers; i++) {
- ManagedCursor c = ledger.openCursor("c" + i);
-
- c.asyncReadEntriesOrWait(1, new ReadEntriesCallback() {
- @Override
- public void readEntriesComplete(List<Entry> entries, Object ctx) {
- assertEquals(entries.size(), 1);
- entries.forEach(e -> e.release());
- counter.countDown();
- }
-
- @Override
- public void readEntriesFailed(ManagedLedgerException exception, Object ctx) {
- log.error("Error reading", exception);
- }
- }, null);
- }
-
- ledger.addEntry("test".getBytes());
- counter.await();
- }
-
- @Test(timeOut = 20000)
- void testReadEntriesOrWaitBlocking() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger");
-
- final int Messages = 100;
- final int Consumers = 10;
-
- List<Future<Void>> futures = Lists.newArrayList();
- ExecutorService executor = Executors.newCachedThreadPool();
- final CyclicBarrier barrier = new CyclicBarrier(Consumers + 1);
-
- for (int i = 0; i < Consumers; i++) {
- final ManagedCursor cursor = ledger.openCursor("c" + i);
-
- futures.add(executor.submit(new Callable<Void>() {
- @Override
- public Void call() throws Exception {
- barrier.await();
-
- int toRead = Messages;
- while (toRead > 0) {
- List<Entry> entries = cursor.readEntriesOrWait(10);
- assertTrue(entries.size() <= 10);
- toRead -= entries.size();
- entries.forEach(e -> e.release());
- }
-
- return null;
- }
- }));
- }
-
- barrier.await();
- for (int i = 0; i < Messages; i++) {
- ledger.addEntry("test".getBytes());
- }
-
- for (Future<Void> f : futures) {
- f.get();
- }
- }
-
- @Test(timeOut = 20000)
- void testFindNewestMatching() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger");
-
- ManagedCursorImpl c1 = (ManagedCursorImpl) ledger.openCursor("c1");
-
- ledger.addEntry("not-expired".getBytes(Encoding));
- ledger.addEntry("not-expired".getBytes(Encoding));
- ledger.addEntry("not-expired".getBytes(Encoding));
- ledger.addEntry("not-expired".getBytes(Encoding));
- ledger.addEntry("not-expired".getBytes(Encoding));
-
- assertNull(
- c1.findNewestMatching(entry -> Arrays.equals(entry.getDataAndRelease(), "expired".getBytes(Encoding))));
- }
-
- @Test(timeOut = 20000)
- void testFindNewestMatchingOdd1() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger");
-
- ManagedCursorImpl c1 = (ManagedCursorImpl) ledger.openCursor("c1");
-
- Position p1 = ledger.addEntry("expired".getBytes(Encoding));
- ledger.addEntry("not-expired".getBytes(Encoding));
- ledger.addEntry("not-expired".getBytes(Encoding));
- ledger.addEntry("not-expired".getBytes(Encoding));
- ledger.addEntry("not-expired".getBytes(Encoding));
-
- assertEquals(
- c1.findNewestMatching(entry -> Arrays.equals(entry.getDataAndRelease(), "expired".getBytes(Encoding))),
- p1);
- }
-
- @Test(timeOut = 20000)
- void testFindNewestMatchingOdd2() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger");
-
- ManagedCursorImpl c1 = (ManagedCursorImpl) ledger.openCursor("c1");
-
- ledger.addEntry("expired".getBytes(Encoding));
- Position p2 = ledger.addEntry("expired".getBytes(Encoding));
- ledger.addEntry("not-expired".getBytes(Encoding));
- ledger.addEntry("not-expired".getBytes(Encoding));
- ledger.addEntry("not-expired".getBytes(Encoding));
-
- assertEquals(
- c1.findNewestMatching(entry -> Arrays.equals(entry.getDataAndRelease(), "expired".getBytes(Encoding))),
- p2);
- }
-
- @Test(timeOut = 20000)
- void testFindNewestMatchingOdd3() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger");
-
- ManagedCursorImpl c1 = (ManagedCursorImpl) ledger.openCursor("c1");
-
- ledger.addEntry("expired".getBytes(Encoding));
- ledger.addEntry("expired".getBytes(Encoding));
- Position p3 = ledger.addEntry("expired".getBytes(Encoding));
- ledger.addEntry("not-expired".getBytes(Encoding));
- ledger.addEntry("not-expired".getBytes(Encoding));
-
- assertEquals(
- c1.findNewestMatching(entry -> Arrays.equals(entry.getDataAndRelease(), "expired".getBytes(Encoding))),
- p3);
- }
-
- @Test(timeOut = 20000)
- void testFindNewestMatchingOdd4() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger");
-
- ManagedCursorImpl c1 = (ManagedCursorImpl) ledger.openCursor("c1");
-
- ledger.addEntry("expired".getBytes(Encoding));
- ledger.addEntry("expired".getBytes(Encoding));
- ledger.addEntry("expired".getBytes(Encoding));
- Position p4 = ledger.addEntry("expired".getBytes(Encoding));
- ledger.addEntry("not-expired".getBytes(Encoding));
-
- assertEquals(
- c1.findNewestMatching(entry -> Arrays.equals(entry.getDataAndRelease(), "expired".getBytes(Encoding))),
- p4);
- }
-
- @Test(timeOut = 20000)
- void testFindNewestMatchingOdd5() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger");
-
- ManagedCursorImpl c1 = (ManagedCursorImpl) ledger.openCursor("c1");
-
- ledger.addEntry("expired".getBytes(Encoding));
- ledger.addEntry("expired".getBytes(Encoding));
- ledger.addEntry("expired".getBytes(Encoding));
- ledger.addEntry("expired".getBytes(Encoding));
- Position p5 = ledger.addEntry("expired".getBytes(Encoding));
-
- assertEquals(
- c1.findNewestMatching(entry -> Arrays.equals(entry.getDataAndRelease(), "expired".getBytes(Encoding))),
- p5);
- }
-
- @Test(timeOut = 20000)
- void testFindNewestMatchingEven1() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger");
-
- ManagedCursorImpl c1 = (ManagedCursorImpl) ledger.openCursor("c1");
-
- Position p1 = ledger.addEntry("expired".getBytes(Encoding));
- ledger.addEntry("not-expired".getBytes(Encoding));
- ledger.addEntry("not-expired".getBytes(Encoding));
- ledger.addEntry("not-expired".getBytes(Encoding));
-
- assertEquals(
- c1.findNewestMatching(entry -> Arrays.equals(entry.getDataAndRelease(), "expired".getBytes(Encoding))),
- p1);
- }
-
- @Test(timeOut = 20000)
- void testFindNewestMatchingEven2() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger");
-
- ManagedCursorImpl c1 = (ManagedCursorImpl) ledger.openCursor("c1");
-
- ledger.addEntry("expired".getBytes(Encoding));
- Position p2 = ledger.addEntry("expired".getBytes(Encoding));
- ledger.addEntry("not-expired".getBytes(Encoding));
- ledger.addEntry("not-expired".getBytes(Encoding));
-
- assertEquals(
- c1.findNewestMatching(entry -> Arrays.equals(entry.getDataAndRelease(), "expired".getBytes(Encoding))),
- p2);
- }
-
- @Test(timeOut = 20000)
- void testFindNewestMatchingEven3() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger");
-
- ManagedCursorImpl c1 = (ManagedCursorImpl) ledger.openCursor("c1");
-
- ledger.addEntry("expired".getBytes(Encoding));
- ledger.addEntry("expired".getBytes(Encoding));
- Position p3 = ledger.addEntry("expired".getBytes(Encoding));
- ledger.addEntry("not-expired".getBytes(Encoding));
-
- assertEquals(
- c1.findNewestMatching(entry -> Arrays.equals(entry.getDataAndRelease(), "expired".getBytes(Encoding))),
- p3);
- }
-
- @Test(timeOut = 20000)
- void testFindNewestMatchingEven4() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger");
-
- ManagedCursorImpl c1 = (ManagedCursorImpl) ledger.openCursor("c1");
-
- ledger.addEntry("expired".getBytes(Encoding));
- ledger.addEntry("expired".getBytes(Encoding));
- ledger.addEntry("expired".getBytes(Encoding));
- Position p4 = ledger.addEntry("expired".getBytes(Encoding));
-
- assertEquals(
- c1.findNewestMatching(entry -> Arrays.equals(entry.getDataAndRelease(), "expired".getBytes(Encoding))),
- p4);
- }
-
- @Test(timeOut = 20000)
- void testFindNewestMatchingEdgeCase1() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger");
-
- ManagedCursorImpl c1 = (ManagedCursorImpl) ledger.openCursor("c1");
- assertEquals(
- c1.findNewestMatching(entry -> Arrays.equals(entry.getDataAndRelease(), "expired".getBytes(Encoding))),
- null);
- }
-
- @Test(timeOut = 20000)
- void testFindNewestMatchingEdgeCase2() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger");
-
- ManagedCursorImpl c1 = (ManagedCursorImpl) ledger.openCursor("c1");
- Position p1 = ledger.addEntry("expired".getBytes(Encoding));
- assertEquals(
- c1.findNewestMatching(entry -> Arrays.equals(entry.getDataAndRelease(), "expired".getBytes(Encoding))),
- p1);
- }
-
- @Test(timeOut = 20000)
- void testFindNewestMatchingEdgeCase3() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger");
-
- ManagedCursorImpl c1 = (ManagedCursorImpl) ledger.openCursor("c1");
- Position p1 = ledger.addEntry("expired".getBytes(Encoding));
- ledger.addEntry("not-expired".getBytes(Encoding));
- assertEquals(
- c1.findNewestMatching(entry -> Arrays.equals(entry.getDataAndRelease(), "expired".getBytes(Encoding))),
- p1);
- }
-
- @Test(timeOut = 20000)
- void testFindNewestMatchingEdgeCase4() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger");
-
- ManagedCursorImpl c1 = (ManagedCursorImpl) ledger.openCursor("c1");
- Position p1 = ledger.addEntry("expired".getBytes(Encoding));
- ledger.addEntry("not-expired".getBytes(Encoding));
- ledger.addEntry("not-expired".getBytes(Encoding));
- assertEquals(
- c1.findNewestMatching(entry -> Arrays.equals(entry.getDataAndRelease(), "expired".getBytes(Encoding))),
- p1);
- }
-
- @Test(timeOut = 20000)
- void testFindNewestMatchingEdgeCase5() throws Exception {
- ManagedLedger ledger = factory.open("testFindNewestMatchingEdgeCase5");
-
- ManagedCursorImpl c1 = (ManagedCursorImpl) ledger.openCursor("c1");
- ledger.addEntry("expired".getBytes(Encoding));
- Position p2 = ledger.addEntry("expired".getBytes(Encoding));
- ledger.addEntry("not-expired".getBytes(Encoding));
- assertEquals(
- c1.findNewestMatching(entry -> Arrays.equals(entry.getDataAndRelease(), "expired".getBytes(Encoding))),
- p2);
- }
-
- @Test(timeOut = 20000)
- void testFindNewestMatchingEdgeCase6() throws Exception {
- ManagedLedger ledger = factory.open("testFindNewestMatchingEdgeCase6",
- new ManagedLedgerConfig().setMaxEntriesPerLedger(3));
-
- ManagedCursorImpl c1 = (ManagedCursorImpl) ledger.openCursor("c1");
- ledger.addEntry("expired".getBytes(Encoding));
- ledger.addEntry("expired".getBytes(Encoding));
- ledger.addEntry("expired".getBytes(Encoding));
- Position newPosition = ledger.addEntry("expired".getBytes(Encoding));
- ledger.addEntry("not-expired".getBytes(Encoding));
- List<Entry> entries = c1.readEntries(3);
- c1.markDelete(entries.get(2).getPosition());
- entries.forEach(e -> e.release());
- assertEquals(
- c1.findNewestMatching(entry -> Arrays.equals(entry.getDataAndRelease(), "expired".getBytes(Encoding))),
- newPosition);
- }
-
- @Test(timeOut = 20000)
- void testFindNewestMatchingEdgeCase7() throws Exception {
- ManagedLedger ledger = factory.open("testFindNewestMatchingEdgeCase7");
-
- ManagedCursorImpl c1 = (ManagedCursorImpl) ledger.openCursor("c1");
- ledger.addEntry("expired".getBytes(Encoding));
- ledger.addEntry("expired".getBytes(Encoding));
- ledger.addEntry("expired".getBytes(Encoding));
- ledger.addEntry("expired".getBytes(Encoding));
- ledger.addEntry("expired".getBytes(Encoding));
- Position lastPosition = ledger.addEntry("expired".getBytes(Encoding));
-
- List<Entry> entries = c1.readEntries(4);
- c1.markDelete(entries.get(0).getPosition());
- c1.delete(entries.get(2).getPosition());
- entries.forEach(e -> e.release());
-
- assertEquals(
- c1.findNewestMatching(entry -> Arrays.equals(entry.getDataAndRelease(), "expired".getBytes(Encoding))),
- lastPosition);
- }
-
- @Test(timeOut = 20000)
- void testFindNewestMatchingEdgeCase8() throws Exception {
- ManagedLedger ledger = factory.open("testFindNewestMatchingEdgeCase8");
-
- ManagedCursorImpl c1 = (ManagedCursorImpl) ledger.openCursor("c1");
- ledger.addEntry("expired".getBytes(Encoding));
- ledger.addEntry("expired".getBytes(Encoding));
- ledger.addEntry("expired".getBytes(Encoding));
- ledger.addEntry("expired".getBytes(Encoding));
- ledger.addEntry("expired".getBytes(Encoding));
- Position lastPosition = ledger.addEntry("expired".getBytes(Encoding));
- ledger.addEntry("not-expired".getBytes(Encoding));
-
- List<Entry> entries = c1.readEntries(4);
- c1.delete(entries.get(1).getPosition());
- c1.delete(entries.get(2).getPosition());
- entries.forEach(e -> e.release());
-
- assertEquals(
- c1.findNewestMatching(entry -> Arrays.equals(entry.getDataAndRelease(), "expired".getBytes(Encoding))),
- lastPosition);
- }
-
- @Test(timeOut = 20000)
- void testFindNewestMatchingEdgeCase9() throws Exception {
- ManagedLedger ledger = factory.open("testFindNewestMatchingEdgeCase9");
-
- ManagedCursorImpl c1 = (ManagedCursorImpl) ledger.openCursor("c1");
- ledger.addEntry("expired".getBytes(Encoding));
- ledger.addEntry("expired".getBytes(Encoding));
- ledger.addEntry("expired".getBytes(Encoding));
- ledger.addEntry("expired".getBytes(Encoding));
- ledger.addEntry("expired".getBytes(Encoding));
- ledger.addEntry("expired".getBytes(Encoding));
- Position lastPosition = ledger.addEntry("expired".getBytes(Encoding));
- ledger.addEntry("not-expired".getBytes(Encoding));
-
- List<Entry> entries = c1.readEntries(5);
- c1.delete(entries.get(1).getPosition());
- c1.delete(entries.get(3).getPosition());
- entries.forEach(e -> e.release());
-
- assertEquals(
- c1.findNewestMatching(entry -> Arrays.equals(entry.getDataAndRelease(), "expired".getBytes(Encoding))),
- lastPosition);
- }
-
- @Test(timeOut = 20000)
- void testFindNewestMatchingEdgeCase10() throws Exception {
- ManagedLedger ledger = factory.open("testFindNewestMatchingEdgeCase10");
-
- ManagedCursorImpl c1 = (ManagedCursorImpl) ledger.openCursor("c1");
- ledger.addEntry("expired".getBytes(Encoding));
- ledger.addEntry("expired".getBytes(Encoding));
- ledger.addEntry("expired".getBytes(Encoding));
- ledger.addEntry("expired".getBytes(Encoding));
- ledger.addEntry("expired".getBytes(Encoding));
- ledger.addEntry("expired".getBytes(Encoding));
- Position lastPosition = ledger.addEntry("expired".getBytes(Encoding));
- ledger.addEntry("not-expired".getBytes(Encoding));
-
- List<Entry> entries = c1.readEntries(7);
- c1.delete(entries.get(1).getPosition());
- c1.delete(entries.get(3).getPosition());
- c1.delete(entries.get(6).getPosition());
- entries.forEach(e -> e.release());
-
- assertEquals(
- c1.findNewestMatching(entry -> Arrays.equals(entry.getDataAndRelease(), "expired".getBytes(Encoding))),
- lastPosition);
- }
-
- @Test(timeOut = 20000)
- void testIndividuallyDeletedMessages() throws Exception {
- ManagedLedger ledger = factory.open("testIndividuallyDeletedMessages");
-
- ManagedCursorImpl c1 = (ManagedCursorImpl) ledger.openCursor("c1");
- ledger.addEntry("entry-0".getBytes(Encoding));
- ledger.addEntry("entry-1".getBytes(Encoding));
- ledger.addEntry("entry-2".getBytes(Encoding));
- ledger.addEntry("entry-3".getBytes(Encoding));
- ledger.addEntry("entry-4".getBytes(Encoding));
-
- List<Entry> entries = c1.readEntries(4);
- c1.delete(entries.get(1).getPosition());
- c1.delete(entries.get(2).getPosition());
- c1.markDelete(entries.get(3).getPosition());
- entries.forEach(e -> e.release());
-
- assertTrue(c1.isIndividuallyDeletedEntriesEmpty());
- }
-
- @Test(timeOut = 20000)
- void testIndividuallyDeletedMessages1() throws Exception {
- ManagedLedger ledger = factory.open("testIndividuallyDeletedMessages1");
-
- ManagedCursorImpl c1 = (ManagedCursorImpl) ledger.openCursor("c1");
- ledger.addEntry("entry-0".getBytes(Encoding));
- ledger.addEntry("entry-1".getBytes(Encoding));
- ledger.addEntry("entry-2".getBytes(Encoding));
- ledger.addEntry("entry-3".getBytes(Encoding));
- ledger.addEntry("entry-4".getBytes(Encoding));
-
- List<Entry> entries = c1.readEntries(4);
- c1.delete(entries.get(1).getPosition());
- c1.markDelete(entries.get(3).getPosition());
- entries.forEach(e -> e.release());
-
- assertTrue(c1.isIndividuallyDeletedEntriesEmpty());
- }
-
- @Test(timeOut = 20000)
- void testIndividuallyDeletedMessages2() throws Exception {
- ManagedLedger ledger = factory.open("testIndividuallyDeletedMessages2");
-
- ManagedCursorImpl c1 = (ManagedCursorImpl) ledger.openCursor("c1");
- ledger.addEntry("entry-0".getBytes(Encoding));
- ledger.addEntry("entry-1".getBytes(Encoding));
- ledger.addEntry("entry-2".getBytes(Encoding));
- ledger.addEntry("entry-3".getBytes(Encoding));
- ledger.addEntry("entry-4".getBytes(Encoding));
-
- List<Entry> entries = c1.readEntries(4);
- c1.delete(entries.get(1).getPosition());
- c1.delete(entries.get(2).getPosition());
- c1.delete(entries.get(0).getPosition());
- entries.forEach(e -> e.release());
-
- assertTrue(c1.isIndividuallyDeletedEntriesEmpty());
- }
-
- @Test(timeOut = 20000)
- void testIndividuallyDeletedMessages3() throws Exception {
- ManagedLedger ledger = factory.open("testIndividuallyDeletedMessages3");
-
- ManagedCursorImpl c1 = (ManagedCursorImpl) ledger.openCursor("c1");
- ledger.addEntry("entry-0".getBytes(Encoding));
- ledger.addEntry("entry-1".getBytes(Encoding));
- ledger.addEntry("entry-2".getBytes(Encoding));
- ledger.addEntry("entry-3".getBytes(Encoding));
- ledger.addEntry("entry-4".getBytes(Encoding));
-
- List<Entry> entries = c1.readEntries(4);
- c1.delete(entries.get(1).getPosition());
- c1.delete(entries.get(2).getPosition());
- c1.markDelete(entries.get(0).getPosition());
- entries.forEach(e -> e.release());
-
- assertTrue(c1.isIndividuallyDeletedEntriesEmpty());
- }
-
- public static byte[] getEntryPublishTime(String msg) throws Exception {
- return Long.toString(System.currentTimeMillis()).getBytes();
- }
-
- public Position findPositionFromAllEntries(ManagedCursor c1, final long timestamp) throws Exception {
- final CountDownLatch counter = new CountDownLatch(1);
- class Result {
- ManagedLedgerException exception = null;
- Position position = null;
- }
-
- final Result result = new Result();
- AsyncCallbacks.FindEntryCallback findEntryCallback = new AsyncCallbacks.FindEntryCallback() {
- @Override
- public void findEntryComplete(Position position, Object ctx) {
- result.position = position;
- counter.countDown();
- }
-
- @Override
- public void findEntryFailed(ManagedLedgerException exception, Object ctx) {
- result.exception = exception;
- counter.countDown();
- }
- };
-
- c1.asyncFindNewestMatching(ManagedCursor.FindPositionConstraint.SearchAllAvailableEntries, entry -> {
-
- try {
- long publishTime = Long.valueOf(new String(entry.getData()));
- return publishTime <= timestamp;
- } catch (Exception e) {
- log.error("Error de-serializing message for message position find", e);
- } finally {
- entry.release();
- }
- return false;
- }, findEntryCallback, ManagedCursorImpl.FindPositionConstraint.SearchAllAvailableEntries);
- counter.await();
- if (result.exception != null) {
- throw result.exception;
- }
- return result.position;
- }
-
- void internalTestFindNewestMatchingAllEntries(final String name, final int entriesPerLedger,
- final int expectedEntryId) throws Exception {
- final String ledgerAndCursorName = name;
- ManagedLedgerConfig config = new ManagedLedgerConfig();
- config.setRetentionSizeInMB(10);
- config.setMaxEntriesPerLedger(entriesPerLedger);
- config.setRetentionTime(1, TimeUnit.HOURS);
- ManagedLedger ledger = factory.open(ledgerAndCursorName, config);
- ManagedCursorImpl c1 = (ManagedCursorImpl) ledger.openCursor(ledgerAndCursorName);
-
- ledger.addEntry(getEntryPublishTime("retained1"));
- // space apart message publish times
- Thread.sleep(100);
- ledger.addEntry(getEntryPublishTime("retained2"));
- Thread.sleep(100);
- ledger.addEntry(getEntryPublishTime("retained3"));
- Thread.sleep(100);
- Position newPosition = ledger.addEntry(getEntryPublishTime("expectedresetposition"));
- long timestamp = System.currentTimeMillis();
- long ledgerId = ((PositionImpl) newPosition).getLedgerId();
- Thread.sleep(2);
-
- ledger.addEntry(getEntryPublishTime("not-read"));
- List<Entry> entries = c1.readEntries(3);
- c1.markDelete(entries.get(2).getPosition());
- c1.close();
- ledger.close();
- entries.forEach(e -> e.release());
- // give timed ledger trimming a chance to run
- Thread.sleep(100);
-
- ledger = factory.open(ledgerAndCursorName, config);
- c1 = (ManagedCursorImpl) ledger.openCursor(ledgerAndCursorName);
-
- PositionImpl found = (PositionImpl) findPositionFromAllEntries(c1, timestamp);
- assertEquals(found.getLedgerId(), ledgerId);
- assertEquals(found.getEntryId(), expectedEntryId);
- }
-
- @Test(timeOut = 20000)
- void testFindNewestMatchingAllEntries() throws Exception {
- final String ledgerAndCursorName = "testFindNewestMatchingAllEntries";
- // condition below assumes entries per ledger is 2
- // needs to be changed if entries per ledger is changed
- int expectedEntryId = 1;
- int entriesPerLedger = 2;
- internalTestFindNewestMatchingAllEntries(ledgerAndCursorName, entriesPerLedger, expectedEntryId);
- }
-
- @Test(timeOut = 20000)
- void testFindNewestMatchingAllEntries2() throws Exception {
- final String ledgerAndCursorName = "testFindNewestMatchingAllEntries2";
- // condition below assumes entries per ledger is 1
- // needs to be changed if entries per ledger is changed
- int expectedEntryId = 0;
- int entriesPerLedger = 1;
- internalTestFindNewestMatchingAllEntries(ledgerAndCursorName, entriesPerLedger, expectedEntryId);
- }
-
- @Test(timeOut = 20000)
- void testFindNewestMatchingAllEntriesSingleLedger() throws Exception {
- final String ledgerAndCursorName = "testFindNewestMatchingAllEntriesSingleLedger";
- ManagedLedgerConfig config = new ManagedLedgerConfig();
- // needs to be changed if entries per ledger is changed
- int expectedEntryId = 3;
- int entriesPerLedger = config.getMaxEntriesPerLedger();
- internalTestFindNewestMatchingAllEntries(ledgerAndCursorName, entriesPerLedger, expectedEntryId);
- }
-
- @Test(timeOut = 20000)
- void testReplayEntries() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger");
-
- ManagedCursorImpl c1 = (ManagedCursorImpl) ledger.openCursor("c1");
- PositionImpl p1 = (PositionImpl) ledger.addEntry("entry1".getBytes(Encoding));
- PositionImpl p2 = (PositionImpl) ledger.addEntry("entry2".getBytes(Encoding));
- PositionImpl p3 = (PositionImpl) ledger.addEntry("entry3".getBytes(Encoding));
- ledger.addEntry("entry4".getBytes(Encoding));
-
- // 1. Replay empty position set should return empty entry set
- Set<PositionImpl> positions = Sets.newHashSet();
- assertTrue(c1.replayEntries(positions).isEmpty());
-
- positions.add(p1);
- positions.add(p3);
-
- // 2. entries 1 and 3 should be returned, but they can be in any order
- List<Entry> entries = c1.replayEntries(positions);
- assertEquals(entries.size(), 2);
- assertTrue((Arrays.equals(entries.get(0).getData(), "entry1".getBytes(Encoding))
- && Arrays.equals(entries.get(1).getData(), "entry3".getBytes(Encoding)))
- || (Arrays.equals(entries.get(0).getData(), "entry3".getBytes(Encoding))
- && Arrays.equals(entries.get(1).getData(), "entry1".getBytes(Encoding))));
- entries.forEach(Entry::release);
-
- // 3. Fail on reading non-existing position
- PositionImpl invalidPosition = new PositionImpl(100, 100);
- positions.add(invalidPosition);
-
- try {
- c1.replayEntries(positions);
- fail("Should fail");
- } catch (ManagedLedgerException e) {
- // ok
- }
- positions.remove(invalidPosition);
-
- // 4. Fail to attempt to read mark-deleted position (p1)
- c1.markDelete(p2);
-
- try {
- // as mark-delete is at position: p2 it should read entry : p3
- assertEquals(1, c1.replayEntries(positions).size());
- } catch (ManagedLedgerException e) {
- fail("Should have not failed");
- }
- }
-
- @Test(timeOut = 20000)
- void outOfOrderAcks() throws Exception {
- ManagedLedger ledger = factory.open("outOfOrderAcks");
- ManagedCursor c1 = ledger.openCursor("c1");
-
- int N = 10;
-
- List<Position> positions = new ArrayList<>();
- for (int i = 0; i < N; i++) {
- positions.add(ledger.addEntry("entry".getBytes()));
- }
-
- assertEquals(c1.getNumberOfEntriesInBacklog(), N);
-
- c1.delete(positions.get(3));
- assertEquals(c1.getNumberOfEntriesInBacklog(), N - 1);
-
- c1.delete(positions.get(2));
- assertEquals(c1.getNumberOfEntriesInBacklog(), N - 2);
-
- c1.delete(positions.get(1));
- assertEquals(c1.getNumberOfEntriesInBacklog(), N - 3);
-
- c1.delete(positions.get(0));
- assertEquals(c1.getNumberOfEntriesInBacklog(), N - 4);
- }
-
- @Test(timeOut = 20000)
- void randomOrderAcks() throws Exception {
- ManagedLedger ledger = factory.open("outOfOrderAcks");
- ManagedCursor c1 = ledger.openCursor("c1");
-
- int N = 10;
-
- List<Position> positions = new ArrayList<>();
- for (int i = 0; i < N; i++) {
- positions.add(ledger.addEntry("entry".getBytes()));
- }
-
- assertEquals(c1.getNumberOfEntriesInBacklog(), N);
-
- // Randomize the ack sequence
- Collections.shuffle(positions);
-
- int toDelete = N;
- for (Position p : positions) {
- assertEquals(c1.getNumberOfEntriesInBacklog(), toDelete);
- c1.delete(p);
- --toDelete;
- assertEquals(c1.getNumberOfEntriesInBacklog(), toDelete);
- }
- }
-
- @Test(timeOut = 20000)
- void testGetEntryAfterN() throws Exception {
- ManagedLedger ledger = factory.open("testGetEntryAfterN");
-
- ManagedCursor c1 = ledger.openCursor("c1");
-
- Position pos1 = ledger.addEntry("msg1".getBytes());
- Position pos2 = ledger.addEntry("msg2".getBytes());
- Position pos3 = ledger.addEntry("msg3".getBytes());
- Position pos4 = ledger.addEntry("msg4".getBytes());
- Position pos5 = ledger.addEntry("msg5".getBytes());
-
- List<Entry> entries = c1.readEntries(4);
- entries.forEach(e -> e.release());
- long currentLedger = ((PositionImpl) c1.getMarkDeletedPosition()).getLedgerId();
-
- // check if the first message is returned for '0'
- Entry e = c1.getNthEntry(1, IndividualDeletedEntries.Exclude);
- assertEquals(e.getDataAndRelease(), "msg1".getBytes());
-
- // check that if we call get entry for the same position twice, it returns the same entry
- e = c1.getNthEntry(1, IndividualDeletedEntries.Exclude);
- assertEquals(e.getDataAndRelease(), "msg1".getBytes());
-
- // check for a position 'n' after md position
- e = c1.getNthEntry(3, IndividualDeletedEntries.Exclude);
- assertEquals(e.getDataAndRelease(), "msg3".getBytes());
-
- // check for the last position
- e = c1.getNthEntry(5, IndividualDeletedEntries.Exclude);
- assertEquals(e.getDataAndRelease(), "msg5".getBytes());
-
- // check for a position outside the limits of the number of entries that exists, it should return null
- e = c1.getNthEntry(10, IndividualDeletedEntries.Exclude);
- assertNull(e);
-
- // check that the mark delete and read positions have not been updated after all the previous operations
- assertEquals(c1.getMarkDeletedPosition(), new PositionImpl(currentLedger, -1));
- assertEquals(c1.getReadPosition(), new PositionImpl(currentLedger, 4));
-
- c1.markDelete(pos4);
- assertEquals(c1.getMarkDeletedPosition(), pos4);
- e = c1.getNthEntry(1, IndividualDeletedEntries.Exclude);
- assertEquals(e.getDataAndRelease(), "msg5".getBytes());
-
- c1.readEntries(1);
- c1.markDelete(pos5);
-
- e = c1.getNthEntry(1, IndividualDeletedEntries.Exclude);
- assertNull(e);
- }
-
- @Test(timeOut = 20000)
- void testGetEntryAfterNWithIndividualDeletedMessages() throws Exception {
- ManagedLedger ledger = factory.open("testGetEnteryAfterNWithIndividualDeletedMessages");
-
- ManagedCursor c1 = ledger.openCursor("c1");
-
- Position pos1 = ledger.addEntry("msg1".getBytes());
- Position pos2 = ledger.addEntry("msg2".getBytes());
- Position pos3 = ledger.addEntry("msg3".getBytes());
- Position pos4 = ledger.addEntry("msg4".getBytes());
- Position pos5 = ledger.addEntry("msg5".getBytes());
-
- c1.delete(pos3);
- c1.delete(pos4);
-
- Entry e = c1.getNthEntry(3, IndividualDeletedEntries.Exclude);
- assertEquals(e.getDataAndRelease(), "msg5".getBytes());
-
- e = c1.getNthEntry(3, IndividualDeletedEntries.Include);
- assertEquals(e.getDataAndRelease(), "msg3".getBytes());
- }
-
- @Test(timeOut = 20000)
- void cancelReadOperation() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger", new ManagedLedgerConfig().setMaxEntriesPerLedger(1));
-
- ManagedCursor c1 = ledger.openCursor("c1");
-
- // No read request so far
- assertEquals(c1.cancelPendingReadRequest(), false);
-
- CountDownLatch counter = new CountDownLatch(1);
-
- c1.asyncReadEntriesOrWait(1, new ReadEntriesCallback() {
- @Override
- public void readEntriesComplete(List<Entry> entries, Object ctx) {
- counter.countDown();
- }
-
- @Override
- public void readEntriesFailed(ManagedLedgerException exception, Object ctx) {
- counter.countDown();
- }
- }, null);
-
- assertEquals(c1.cancelPendingReadRequest(), true);
-
- CountDownLatch counter2 = new CountDownLatch(1);
-
- c1.asyncReadEntriesOrWait(1, new ReadEntriesCallback() {
- @Override
- public void readEntriesComplete(List<Entry> entries, Object ctx) {
- counter2.countDown();
- }
-
- @Override
- public void readEntriesFailed(ManagedLedgerException exception, Object ctx) {
- counter2.countDown();
- }
- }, null);
-
- ledger.addEntry("entry-1".getBytes(Encoding));
-
- Thread.sleep(100);
-
- // Read operation should have already been completed
- assertEquals(c1.cancelPendingReadRequest(), false);
-
- counter2.await();
- }
-
- @Test(timeOut = 20000)
- public void testReopenMultipleTimes() throws Exception {
- ManagedLedger ledger = factory.open("testReopenMultipleTimes");
- ManagedCursor c1 = ledger.openCursor("c1");
-
- Position mdPosition = c1.getMarkDeletedPosition();
-
- c1.close();
- ledger.close();
-
- ledger = factory.open("testReopenMultipleTimes");
- c1 = ledger.openCursor("c1");
-
- // since the empty data ledger will be deleted, the cursor position should also be updated
- assertNotEquals(c1.getMarkDeletedPosition(), mdPosition);
-
- c1.close();
- ledger.close();
-
- ledger = factory.open("testReopenMultipleTimes");
- c1 = ledger.openCursor("c1");
- }
-
- @Test(timeOut = 20000)
- public void testOutOfOrderDeletePersistenceWithClose() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger", new ManagedLedgerConfig());
-
- ManagedCursor c1 = ledger.openCursor("c1");
- List<Position> addedPositions = new ArrayList<>();
- for (int i = 0; i < 20; i++) {
- Position p = ledger.addEntry(("dummy-entry-" + i).getBytes(Encoding));
- addedPositions.add(p);
- }
-
- // Acknowledge few messages leaving holes
- c1.delete(addedPositions.get(2));
- c1.delete(addedPositions.get(5));
- c1.delete(addedPositions.get(7));
- c1.delete(addedPositions.get(8));
- c1.delete(addedPositions.get(9));
-
- assertEquals(c1.getNumberOfEntriesInBacklog(), 20 - 5);
-
- ledger.close();
- factory.shutdown();
-
- // Re-Open
- factory = new ManagedLedgerFactoryImpl(bkc, bkc.getZkHandle());
- ledger = factory.open("my_test_ledger", new ManagedLedgerConfig());
- c1 = ledger.openCursor("c1");
- assertEquals(c1.getNumberOfEntriesInBacklog(), 20 - 5);
-
- List<Entry> entries = c1.readEntries(20);
- assertEquals(entries.size(), 20 - 5);
-
- List<String> entriesStr = entries.stream().map(e -> new String(e.getDataAndRelease(), Encoding))
- .collect(Collectors.toList());
- assertEquals(entriesStr.get(0), "dummy-entry-0");
- assertEquals(entriesStr.get(1), "dummy-entry-1");
- // Entry-2 was deleted
- assertEquals(entriesStr.get(2), "dummy-entry-3");
- assertEquals(entriesStr.get(3), "dummy-entry-4");
- // Entry-6 was deleted
- assertEquals(entriesStr.get(4), "dummy-entry-6");
-
- assertFalse(c1.hasMoreEntries());
- }
-
- @Test(timeOut = 20000)
- public void testOutOfOrderDeletePersistenceAfterCrash() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger", new ManagedLedgerConfig());
-
- ManagedCursor c1 = ledger.openCursor("c1");
- List<Position> addedPositions = new ArrayList<>();
- for (int i = 0; i < 20; i++) {
- Position p = ledger.addEntry(("dummy-entry-" + i).getBytes(Encoding));
- addedPositions.add(p);
- }
-
- // Acknowledge few messages leaving holes
- c1.delete(addedPositions.get(2));
- c1.delete(addedPositions.get(5));
- c1.delete(addedPositions.get(7));
- c1.delete(addedPositions.get(8));
- c1.delete(addedPositions.get(9));
-
- assertEquals(c1.getNumberOfEntriesInBacklog(), 20 - 5);
-
- // Re-Open
- ManagedLedgerFactory factory2 = new ManagedLedgerFactoryImpl(bkc, bkc.getZkHandle());
- ledger = factory2.open("my_test_ledger", new ManagedLedgerConfig());
- c1 = ledger.openCursor("c1");
- assertEquals(c1.getNumberOfEntriesInBacklog(), 20 - 5);
-
- List<Entry> entries = c1.readEntries(20);
- assertEquals(entries.size(), 20 - 5);
-
- List<String> entriesStr = entries.stream().map(e -> new String(e.getDataAndRelease(), Encoding))
- .collect(Collectors.toList());
- assertEquals(entriesStr.get(0), "dummy-entry-0");
- assertEquals(entriesStr.get(1), "dummy-entry-1");
- // Entry-2 was deleted
- assertEquals(entriesStr.get(2), "dummy-entry-3");
- assertEquals(entriesStr.get(3), "dummy-entry-4");
- // Entry-6 was deleted
- assertEquals(entriesStr.get(4), "dummy-entry-6");
-
- assertFalse(c1.hasMoreEntries());
- factory2.shutdown();
- }
-
- /**
- * <pre>
- * Verifies that {@link ManagedCursorImpl#createNewMetadataLedger()} cleans up orphan ledgers if fails to switch new
- * ledger
- * </pre>
- * @throws Exception
- */
- @Test(timeOut=5000)
- public void testLeakFailedLedgerOfManageCursor() throws Exception {
-
- ManagedLedgerConfig mlConfig = new ManagedLedgerConfig();
- ManagedLedger ledger = factory.open("my_test_ledger", mlConfig);
-
- ManagedCursorImpl c1 = (ManagedCursorImpl) ledger.openCursor("c1");
- CountDownLatch latch = new CountDownLatch(1);
- c1.createNewMetadataLedger(new VoidCallback() {
- @Override
- public void operationComplete() {
- latch.countDown();
- }
-
- @Override
- public void operationFailed(ManagedLedgerException exception) {
- latch.countDown();
- }
- });
-
- // update cursor-info with data which makes bad-version for existing managed-cursor
- CountDownLatch latch1 = new CountDownLatch(1);
- String path = "/managed-ledgers/my_test_ledger/c1";
- zkc.setData(path, "".getBytes(), -1, (rc, path1, ctx, stat) -> {
- // updated path
- latch1.countDown();
- }, null);
- latch1.await();
-
- // try to create ledger again which will fail because managedCursorInfo znode is already updated with different
- // version so, this call will fail with BadVersionException
- CountDownLatch latch2 = new CountDownLatch(1);
- // create ledger will create ledgerId = 6
- long ledgerId = 6;
- c1.createNewMetadataLedger(new VoidCallback() {
- @Override
- public void operationComplete() {
- latch2.countDown();
- }
-
- @Override
- public void operationFailed(ManagedLedgerException exception) {
- latch2.countDown();
- }
- });
-
- // Wait until operation is completed and the failed ledger should have been deleted
- latch2.await();
-
- try {
- bkc.openLedgerNoRecovery(ledgerId, DigestType.fromApiDigestType(mlConfig.getDigestType()),
- mlConfig.getPassword());
- fail("ledger should have deleted due to update-cursor failure");
- } catch (BKException e) {
- // ok
- }
- }
-
- /**
- * Verifies cursor persists individually unack range into cursor-ledger if range count is higher than
- * MaxUnackedRangesToPersistInZk
- *
- * @throws Exception
- */
- @Test(timeOut = 20000)
- public void testOutOfOrderDeletePersistenceIntoLedgerWithClose() throws Exception {
-
- final int totalAddEntries = 100;
- String ledgerName = "my_test_ledger";
- String cursorName = "c1";
- ManagedLedgerConfig managedLedgerConfig = new ManagedLedgerConfig();
- // metaStore is allowed to store only up to 10 deleted entries range
- managedLedgerConfig.setMaxUnackedRangesToPersistInZk(10);
- ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open(ledgerName, managedLedgerConfig);
-
- ManagedCursorImpl c1 = (ManagedCursorImpl) ledger.openCursor(cursorName);
-
- List<Position> addedPositions = new ArrayList<>();
- for (int i = 0; i < totalAddEntries; i++) {
- Position p = ledger.addEntry(("dummy-entry-" + i).getBytes(Encoding));
- addedPositions.add(p);
- if (i % 2 == 0) {
- // Acknowledge alternative message to create totalEntries/2 holes
- c1.delete(addedPositions.get(i));
- }
- }
-
- assertEquals(c1.getNumberOfEntriesInBacklog(), totalAddEntries / 2);
-
- // Close ledger to persist individual-deleted positions into cursor-ledger
- ledger.close();
-
- // verify cursor-ledgerId is updated properly into cursor-metaStore
- CountDownLatch cursorLedgerLatch = new CountDownLatch(1);
- AtomicLong cursorLedgerId = new AtomicLong(0);
- ledger.getStore().asyncGetCursorInfo(ledger.getName(), cursorName, new MetaStoreCallback<ManagedCursorInfo>() {
- @Override
- public void operationComplete(ManagedCursorInfo result, Stat stat) {
- cursorLedgerId.set(result.getCursorsLedgerId());
- cursorLedgerLatch.countDown();
- }
-
- @Override
- public void operationFailed(MetaStoreException e) {
- cursorLedgerLatch.countDown();
- }
- });
- cursorLedgerLatch.await();
- assertEquals(cursorLedgerId.get(), c1.getCursorLedger());
-
- // verify cursor-ledger's last entry has individual-deleted positions
- final CountDownLatch latch = new CountDownLatch(1);
- final AtomicInteger individualDeletedMessagesCount = new AtomicInteger(0);
- bkc.asyncOpenLedger(c1.getCursorLedger(), DigestType.CRC32C, "".getBytes(), (rc, lh, ctx) -> {
- if (rc == BKException.Code.OK) {
- long lastEntry = lh.getLastAddConfirmed();
- lh.asyncReadEntries(lastEntry, lastEntry, (rc1, lh1, seq, ctx1) -> {
- try {
- LedgerEntry entry = seq.nextElement();
- PositionInfo positionInfo;
- positionInfo = PositionInfo.parseFrom(entry.getEntry());
- individualDeletedMessagesCount.set(positionInfo.getIndividualDeletedMessagesCount());
- } catch (Exception e) {
- }
- latch.countDown();
- }, null);
- } else {
- latch.countDown();
- }
- }, null);
-
- latch.await();
- assertEquals(individualDeletedMessagesCount.get(), totalAddEntries / 2 - 1);
-
- // Re-Open
- factory = new ManagedLedgerFactoryImpl(bkc, bkc.getZkHandle());
- ledger = (ManagedLedgerImpl) factory.open(ledgerName, managedLedgerConfig);
- c1 = (ManagedCursorImpl) ledger.openCursor("c1");
- // verify cursor has been recovered
- assertEquals(c1.getNumberOfEntriesInBacklog(), totalAddEntries / 2);
-
- // try to read entries which should only read non-deleted positions
- List<Entry> entries = c1.readEntries(totalAddEntries);
- assertEquals(entries.size(), totalAddEntries / 2);
- }
-
- /**
- * Close Cursor without MaxUnackedRangesToPersistInZK: It should store individually unack range into Zk
- *
- * @throws Exception
- */
- @Test(timeOut = 20000)
- public void testOutOfOrderDeletePersistenceIntoZkWithClose() throws Exception {
- final int totalAddEntries = 100;
- String ledgerName = "my_test_ledger_zk";
- String cursorName = "c1";
- ManagedLedgerConfig managedLedgerConfig = new ManagedLedgerConfig();
- ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open(ledgerName, managedLedgerConfig);
-
- ManagedCursorImpl c1 = (ManagedCursorImpl) ledger.openCursor(cursorName);
-
- List<Position> addedPositions = new ArrayList<>();
- for (int i = 0; i < totalAddEntries; i++) {
- Position p = ledger.addEntry(("dummy-entry-" + i).getBytes(Encoding));
- addedPositions.add(p);
- if (i % 2 == 0) {
- // Acknowledge alternative message to create totalEntries/2 holes
- c1.delete(addedPositions.get(i));
- }
- }
-
- assertEquals(c1.getNumberOfEntriesInBacklog(), totalAddEntries / 2);
-
- // Close ledger to persist individual-deleted positions into cursor-ledger
- ledger.close();
-
- // verify cursor-ledgerId is updated as -1 into cursor-metaStore
- CountDownLatch latch = new CountDownLatch(1);
- AtomicInteger individualDeletedMessagesCount = new AtomicInteger(0);
- ledger.getStore().asyncGetCursorInfo(ledger.getName(), cursorName, new MetaStoreCallback<ManagedCursorInfo>() {
- @Override
- public void operationComplete(ManagedCursorInfo result, Stat stat) {
- individualDeletedMessagesCount.set(result.getIndividualDeletedMessagesCount());
- latch.countDown();
- }
-
- @Override
- public void operationFailed(MetaStoreException e) {
- latch.countDown();
- }
- });
- latch.await();
- assertEquals(individualDeletedMessagesCount.get(), totalAddEntries / 2 - 1);
-
- // Re-Open
- factory = new ManagedLedgerFactoryImpl(bkc, bkc.getZkHandle());
- ledger = (ManagedLedgerImpl) factory.open(ledgerName, managedLedgerConfig);
- c1 = (ManagedCursorImpl) ledger.openCursor(cursorName);
- // verify cursor has been recovered
- assertEquals(c1.getNumberOfEntriesInBacklog(), totalAddEntries / 2);
-
- // try to read entries which should only read non-deleted positions
- List<Entry> entries = c1.readEntries(totalAddEntries);
- assertEquals(entries.size(), totalAddEntries / 2);
- }
-
- @Test
- public void testInvalidMarkDelete() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger", new ManagedLedgerConfig());
-
- ManagedCursor cursor = ledger.openCursor("c1");
- Position readPosition = cursor.getReadPosition();
- Position markDeletePosition = cursor.getMarkDeletedPosition();
-
- List<Position> addedPositions = new ArrayList<>();
- for (int i = 0; i < 20; i++) {
- Position p = ledger.addEntry(("dummy-entry-" + i).getBytes(Encoding));
- addedPositions.add(p);
- }
-
- // validate: cursor.asyncMarkDelete(..)
- CountDownLatch markDeleteCallbackLatch = new CountDownLatch(1);
- Position position = PositionImpl.get(100, 100);
- AtomicBoolean markDeleteCallFailed = new AtomicBoolean(false);
- cursor.asyncMarkDelete(position, new MarkDeleteCallback() {
- @Override
- public void markDeleteComplete(Object ctx) {
- markDeleteCallbackLatch.countDown();
- }
-
- @Override
- public void markDeleteFailed(ManagedLedgerException exception, Object ctx) {
- markDeleteCallFailed.set(true);
- markDeleteCallbackLatch.countDown();
- }
- }, null);
- markDeleteCallbackLatch.await();
- assertEquals(readPosition, cursor.getReadPosition());
- assertEquals(markDeletePosition, cursor.getMarkDeletedPosition());
-
- // validate : cursor.asyncDelete(..)
- CountDownLatch deleteCallbackLatch = new CountDownLatch(1);
- markDeleteCallFailed.set(false);
- cursor.asyncDelete(position, new DeleteCallback() {
- @Override
- public void deleteComplete(Object ctx) {
- deleteCallbackLatch.countDown();
- }
-
- @Override
- public void deleteFailed(ManagedLedgerException exception, Object ctx) {
- markDeleteCallFailed.set(true);
- deleteCallbackLatch.countDown();
- }
- }, null);
-
- deleteCallbackLatch.await();
- assertEquals(readPosition, cursor.getReadPosition());
- assertEquals(markDeletePosition, cursor.getMarkDeletedPosition());
- }
-
- @Test
- public void testEstimatedUnackedSize() throws Exception {
- ManagedLedgerConfig config = new ManagedLedgerConfig();
- config.setMaxEntriesPerLedger(10);
- ManagedLedger ledger = factory.open("my_test_ledger", new ManagedLedgerConfig());
-
- ManagedCursor cursor = ledger.openCursor("c1");
-
- byte[] entryData = new byte[5];
-
- // write 15 entries, saving position of 5th
- for (int i = 0; i < 4; i++) { ledger.addEntry(entryData); }
- Position deleteAt = ledger.addEntry(entryData);
- for (int i = 0; i < 10; i++) { ledger.addEntry(entryData); }
-
- assertEquals(cursor.getEstimatedSizeSinceMarkDeletePosition(), 15 * entryData.length);
-
- cursor.markDelete(deleteAt);
-
- // it's not an estimate if all entries are the same size
- assertEquals(cursor.getEstimatedSizeSinceMarkDeletePosition(), 10 * entryData.length);
- }
-
- @Test(timeOut = 20000)
- public void testRecoverCursorAheadOfLastPosition() throws Exception {
- final String mlName = "my_test_ledger";
- final PositionImpl lastPosition = new PositionImpl(1L, 10L);
- final PositionImpl nextPosition = new PositionImpl(3L, -1L);
-
- final String cursorName = "my_test_cursor";
- final long cursorsLedgerId = -1L;
- final long markDeleteLedgerId = 2L;
- final long markDeleteEntryId = -1L;
-
- MetaStoreImplZookeeper mockMetaStore = mock(MetaStoreImplZookeeper.class);
- doAnswer(new Answer<Object>() {
- public Object answer(InvocationOnMock invocation) {
- ManagedCursorInfo info = ManagedCursorInfo.newBuilder().setCursorsLedgerId(cursorsLedgerId)
- .setMarkDeleteLedgerId(markDeleteLedgerId).setMarkDeleteEntryId(markDeleteEntryId)
- .setLastActive(0L).build();
- Stat stat = mock(Stat.class);
- MetaStoreCallback<ManagedCursorInfo> callback = (MetaStoreCallback<ManagedCursorInfo>) invocation
- .getArguments()[2];
- callback.operationComplete(info, stat);
- return null;
- }
- }).when(mockMetaStore).asyncGetCursorInfo(eq(mlName), eq(cursorName), any(MetaStoreCallback.class));
-
- ManagedLedgerImpl ml = mock(ManagedLedgerImpl.class);
- when(ml.getName()).thenReturn(mlName);
- when(ml.getStore()).thenReturn(mockMetaStore);
- when(ml.getLastPosition()).thenReturn(lastPosition);
- when(ml.getNextValidLedger(markDeleteLedgerId)).thenReturn(3L);
- when(ml.getNextValidPosition(lastPosition)).thenReturn(nextPosition);
- when(ml.ledgerExists(markDeleteLedgerId)).thenReturn(false);
-
- BookKeeper mockBookKeeper = mock(BookKeeper.class);
- final ManagedCursorImpl cursor = new ManagedCursorImpl(mockBookKeeper, new ManagedLedgerConfig(), ml,
- cursorName);
-
- cursor.recover(new VoidCallback() {
- @Override
- public void operationComplete() {
- assertEquals(cursor.getMarkDeletedPosition(), lastPosition);
- assertEquals(cursor.getReadPosition(), nextPosition);
- assertEquals(cursor.getNumberOfEntries(), 0L);
- }
-
- @Override
- public void operationFailed(ManagedLedgerException exception) {
- fail("Cursor recovery should not fail");
- }
- });
- }
-
- private static final Logger log = LoggerFactory.getLogger(ManagedCursorTest.class);
-}
diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerBkTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerBkTest.java
deleted file mode 100644
index a5b58d7df9..0000000000
--- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerBkTest.java
+++ /dev/null
@@ -1,539 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.bookkeeper.mledger.impl;
-
-import static org.testng.Assert.assertEquals;
-import static org.testng.Assert.assertFalse;
-import static org.testng.Assert.assertNotNull;
-import static org.testng.Assert.assertTrue;
-import static org.testng.Assert.fail;
-
-import com.google.common.base.Charsets;
-import com.google.common.collect.Lists;
-import java.util.List;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.CyclicBarrier;
-import java.util.concurrent.Future;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicReference;
-import org.apache.bookkeeper.client.BookKeeper;
-import org.apache.bookkeeper.client.BookKeeperTestClient;
-import org.apache.bookkeeper.client.api.DigestType;
-import org.apache.bookkeeper.mledger.AsyncCallbacks.AddEntryCallback;
-import org.apache.bookkeeper.mledger.AsyncCallbacks.DeleteCallback;
-import org.apache.bookkeeper.mledger.Entry;
-import org.apache.bookkeeper.mledger.ManagedCursor;
-import org.apache.bookkeeper.mledger.ManagedLedger;
-import org.apache.bookkeeper.mledger.ManagedLedgerConfig;
-import org.apache.bookkeeper.mledger.ManagedLedgerException;
-import org.apache.bookkeeper.mledger.ManagedLedgerException.ManagedLedgerAlreadyClosedException;
-import org.apache.bookkeeper.mledger.ManagedLedgerFactory;
-import org.apache.bookkeeper.mledger.ManagedLedgerFactoryConfig;
-import org.apache.bookkeeper.mledger.Position;
-import org.apache.bookkeeper.test.BookKeeperClusterTestCase;
-import org.apache.pulsar.common.policies.data.PersistentOfflineTopicStats;
-import org.testng.annotations.Test;
-
-public class ManagedLedgerBkTest extends BookKeeperClusterTestCase {
-
- public ManagedLedgerBkTest() {
- super(2);
- }
-
- @Test
- public void testSimpleRead() throws Exception {
- ManagedLedgerFactoryConfig factoryConf = new ManagedLedgerFactoryConfig();
- factoryConf.setMaxCacheSize(0);
- ManagedLedgerFactory factory = new ManagedLedgerFactoryImpl(bkc, zkc, factoryConf);
- ManagedLedgerConfig config = new ManagedLedgerConfig();
- config.setEnsembleSize(1).setWriteQuorumSize(1).setAckQuorumSize(1).setMetadataEnsembleSize(1)
- .setMetadataAckQuorumSize(1);
- ManagedLedger ledger = factory.open("my-ledger", config);
- ManagedCursor cursor = ledger.openCursor("c1");
-
- int N = 1;
-
- for (int i = 0; i < N; i++) {
- String entry = "entry-" + i;
- ledger.addEntry(entry.getBytes());
- }
-
- List<Entry> entries = cursor.readEntries(N);
- assertEquals(N, entries.size());
- entries.forEach(e -> e.release());
- factory.shutdown();
- }
-
- @Test
- public void testBookieFailure() throws Exception {
- ManagedLedgerFactory factory = new ManagedLedgerFactoryImpl(bkc, zkc);
- ManagedLedgerConfig config = new ManagedLedgerConfig();
- config.setEnsembleSize(2).setAckQuorumSize(2).setMetadataEnsembleSize(2);
- ManagedLedger ledger = factory.open("my-ledger", config);
- ManagedCursor cursor = ledger.openCursor("my-cursor");
- ledger.addEntry("entry-0".getBytes());
-
- killBookie(1);
-
- // Now we want to simulate that:
- // 1. The write operation fails because we only have 1 bookie available
- // 2. The bk client cannot properly close the ledger (finalizing the number of entries) because ZK is also
- // not available
- // 3. When we re-establish the service one, the ledger recovery will be triggered and the half-committed entry
- // is restored
-
- // Force to close the ZK client object so that BK will fail to close the ledger
- bkc.getZkHandle().close();
-
- try {
- ledger.addEntry("entry-1".getBytes());
- fail("should fail");
- } catch (ManagedLedgerException e) {
- // ok
- }
-
- bkc = new BookKeeperTestClient(baseClientConf);
- startNewBookie();
-
- // Reconnect a new bk client
- factory = new ManagedLedgerFactoryImpl(bkc, zkc);
- ledger = factory.open("my-ledger", config);
- cursor = ledger.openCursor("my-cursor");
-
- // Next add should succeed
- ledger.addEntry("entry-2".getBytes());
-
- assertEquals(3, cursor.getNumberOfEntriesInBacklog());
-
- List<Entry> entries = cursor.readEntries(1);
- assertEquals(1, entries.size());
- assertEquals("entry-0", new String(entries.get(0).getData()));
- entries.forEach(e -> e.release());
-
- // entry-1 which was half-committed will get fully committed during the recovery phase
- entries = cursor.readEntries(1);
- assertEquals(1, entries.size());
- assertEquals("entry-1", new String(entries.get(0).getData()));
- entries.forEach(e -> e.release());
-
- entries = cursor.readEntries(1);
- assertEquals(1, entries.size());
- assertEquals("entry-2", new String(entries.get(0).getData()));
- entries.forEach(e -> e.release());
-
- factory.shutdown();
- }
-
- @Test
- public void verifyConcurrentUsage() throws Exception {
- ManagedLedgerFactoryConfig config = new ManagedLedgerFactoryConfig();
-
- config.setMaxCacheSize(100 * 1024 * 1024);
-
- ManagedLedgerFactoryImpl factory = new ManagedLedgerFactoryImpl(bkc, bkc.getZkHandle(), config);
-
- EntryCacheManager cacheManager = factory.getEntryCacheManager();
- ManagedLedgerConfig conf = new ManagedLedgerConfig();
- conf.setEnsembleSize(2).setAckQuorumSize(2).setMetadataEnsembleSize(2);
- final ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("my-ledger", conf);
-
- int NumProducers = 1;
- int NumConsumers = 1;
-
- final AtomicBoolean done = new AtomicBoolean();
- final CyclicBarrier barrier = new CyclicBarrier(NumProducers + NumConsumers + 1);
-
- List<Future<?>> futures = Lists.newArrayList();
-
- for (int i = 0; i < NumProducers; i++) {
- futures.add(executor.submit(() -> {
- try {
- barrier.await();
-
- while (!done.get()) {
- ledger.addEntry("entry".getBytes());
- Thread.sleep(1);
- }
- } catch (Exception e) {
- e.printStackTrace();
- }
- }));
- }
-
- for (int i = 0; i < NumConsumers; i++) {
- final int idx = i;
- futures.add(executor.submit(() -> {
- try {
- barrier.await();
-
- ManagedCursor cursor = ledger.openCursor("my-cursor-" + idx);
-
- while (!done.get()) {
- List<Entry> entries = cursor.readEntries(1);
- if (!entries.isEmpty()) {
- cursor.markDelete(entries.get(0).getPosition());
- }
-
- entries.forEach(e -> e.release());
- Thread.sleep(2);
- }
- } catch (Exception e) {
- e.printStackTrace();
- }
- }));
- }
-
- barrier.await();
-
- Thread.sleep(1 * 1000);
-
- done.set(true);
- for (Future<?> future : futures) {
- future.get();
- }
-
- cacheManager.mlFactoryMBean.refreshStats(1, TimeUnit.SECONDS);
-
- assertTrue(cacheManager.mlFactoryMBean.getCacheHitsRate() > 0.0);
- assertEquals(cacheManager.mlFactoryMBean.getCacheMissesRate(), 0.0);
- assertTrue(cacheManager.mlFactoryMBean.getCacheHitsThroughput() > 0.0);
- assertEquals(cacheManager.mlFactoryMBean.getNumberOfCacheEvictions(), 0);
-
- factory.shutdown();
- }
-
- @Test
- public void testSimple() throws Exception {
- ManagedLedgerFactory factory = new ManagedLedgerFactoryImpl(bkc, zkc);
- ManagedLedgerConfig mlConfig = new ManagedLedgerConfig();
- mlConfig.setEnsembleSize(1).setAckQuorumSize(1).setMetadataEnsembleSize(1);
- // set the data ledger size
- mlConfig.setMaxEntriesPerLedger(100);
- // set the metadata ledger size to 1 to kick off many ledger switching cases
- mlConfig.setMetadataMaxEntriesPerLedger(2);
- ManagedLedger ledger = factory.open("ml-simple-ledger", mlConfig);
-
- ledger.addEntry("test".getBytes());
- factory.shutdown();
- }
-
- @Test
- public void testConcurrentMarkDelete() throws Exception {
- ManagedLedgerFactory factory = new ManagedLedgerFactoryImpl(bkc, zkc);
- ManagedLedgerConfig mlConfig = new ManagedLedgerConfig();
- mlConfig.setEnsembleSize(1).setAckQuorumSize(1).setMetadataEnsembleSize(1).setMetadataAckQuorumSize(1);
- // set the data ledger size
- mlConfig.setMaxEntriesPerLedger(100);
- // set the metadata ledger size to 1 to kick off many ledger switching cases
- mlConfig.setMetadataMaxEntriesPerLedger(10);
- ManagedLedger ledger = factory.open("ml-markdelete-ledger", mlConfig);
-
- final List<Position> addedEntries = Lists.newArrayList();
-
- int numCursors = 10;
- final CyclicBarrier barrier = new CyclicBarrier(numCursors);
-
- List<ManagedCursor> cursors = Lists.newArrayList();
- for (int i = 0; i < numCursors; i++) {
- cursors.add(ledger.openCursor(String.format("c%d", i)));
- }
-
- for (int i = 0; i < 50; i++) {
- Position pos = ledger.addEntry("entry".getBytes());
- addedEntries.add(pos);
- }
-
- List<Future<?>> futures = Lists.newArrayList();
-
- for (ManagedCursor cursor : cursors) {
- futures.add(executor.submit(() -> {
- barrier.await();
-
- for (Position position : addedEntries) {
- cursor.markDelete(position);
- }
-
- return null;
- }));
- }
-
- for (Future<?> future : futures) {
- future.get();
- }
-
- // Since in this test we roll-over the cursor ledger every 10 entries acknowledged, the background roll back
- // might still be happening when the futures are completed.
- Thread.sleep(1000);
- factory.shutdown();
- }
-
- @Test
- public void asyncMarkDeleteAndClose() throws Exception {
- ManagedLedgerFactory factory = new ManagedLedgerFactoryImpl(bkc, zkc);
-
- ManagedLedgerConfig config = new ManagedLedgerConfig().setEnsembleSize(1).setWriteQuorumSize(1)
- .setAckQuorumSize(1).setMetadataEnsembleSize(1).setMetadataWriteQuorumSize(1)
- .setMetadataAckQuorumSize(1);
- ManagedLedger ledger = factory.open("my_test_ledger", config);
- ManagedCursor cursor = ledger.openCursor("c1");
-
- List<Position> positions = Lists.newArrayList();
-
- for (int i = 0; i < 10; i++) {
- Position p = ledger.addEntry("entry".getBytes());
- positions.add(p);
- }
-
- final CountDownLatch counter = new CountDownLatch(positions.size());
- final AtomicBoolean gotException = new AtomicBoolean(false);
-
- for (Position p : positions) {
- cursor.asyncDelete(p, new DeleteCallback() {
- @Override
- public void deleteComplete(Object ctx) {
- // Ok
- counter.countDown();
- }
-
- @Override
- public void deleteFailed(ManagedLedgerException exception, Object ctx) {
- exception.printStackTrace();
- gotException.set(true);
- counter.countDown();
- }
- }, null);
- }
-
- cursor.close();
- ledger.close();
-
- counter.await();
-
- assertFalse(gotException.get());
- factory.shutdown();
- }
-
- /**
- * When auto-replication is triggered, if there were no writes on the ML during the grace period, auto-replication
- * will close the ledger an re-replicate it. After that, the next write will get a FencedException. We should
- * recover from this condition by creating a new ledger and retrying the write.
- */
- @Test
- public void ledgerFencedByAutoReplication() throws Exception {
- ManagedLedgerFactoryImpl factory = new ManagedLedgerFactoryImpl(bkc, bkc.getZkHandle());
- ManagedLedgerConfig config = new ManagedLedgerConfig();
- config.setEnsembleSize(2).setAckQuorumSize(2).setMetadataEnsembleSize(2);
- ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("my_test_ledger", config);
- ManagedCursor c1 = ledger.openCursor("c1");
-
- PositionImpl p1 = (PositionImpl) ledger.addEntry("entry-1".getBytes());
-
- // Trigger the closure of the data ledger
- bkc.openLedger(p1.getLedgerId(), BookKeeper.DigestType.CRC32C, new byte[] {});
-
- ledger.addEntry("entry-2".getBytes());
-
- assertEquals(2, c1.getNumberOfEntries());
- assertEquals(2, c1.getNumberOfEntriesInBacklog());
-
- PositionImpl p3 = (PositionImpl) ledger.addEntry("entry-3".getBytes());
-
- // Now entry-2 should have been written before entry-3
- assertEquals(3, c1.getNumberOfEntries());
- assertEquals(3, c1.getNumberOfEntriesInBacklog());
- assertTrue(p1.getLedgerId() != p3.getLedgerId());
- factory.shutdown();
- }
-
- /**
- * When another process steals the ML, the old instance should not succeed in any operation
- */
- @Test
- public void ledgerFencedByFailover() throws Exception {
- ManagedLedgerFactoryImpl factory1 = new ManagedLedgerFactoryImpl(bkc, bkc.getZkHandle());
- ManagedLedgerConfig config = new ManagedLedgerConfig();
- config.setEnsembleSize(2).setAckQuorumSize(2).setMetadataEnsembleSize(2);
- ManagedLedgerImpl ledger1 = (ManagedLedgerImpl) factory1.open("my_test_ledger", config);
- ledger1.openCursor("c");
-
- ledger1.addEntry("entry-1".getBytes());
-
- // Open the ML from another factory
- ManagedLedgerFactoryImpl factory2 = new ManagedLedgerFactoryImpl(bkc, bkc.getZkHandle());
- ManagedLedgerImpl ledger2 = (ManagedLedgerImpl) factory2.open("my_test_ledger", config);
- ManagedCursor c2 = ledger2.openCursor("c");
-
- try {
- ledger1.addEntry("entry-2".getBytes());
- fail("Should have failed");
- } catch (ManagedLedgerException e) {
- // Ok
- }
-
- ledger2.addEntry("entry-2".getBytes());
-
- try {
- ledger1.addEntry("entry-2".getBytes());
- fail("Should have failed");
- } catch (ManagedLedgerException bve) {
- // Ok
- }
-
- assertEquals(2, c2.getNumberOfEntriesInBacklog());
- factory1.shutdown();
- factory2.shutdown();
- }
-
- @Test
- public void testOfflineTopicBacklog() throws Exception {
- ManagedLedgerFactoryConfig factoryConf = new ManagedLedgerFactoryConfig();
- factoryConf.setMaxCacheSize(0);
- ManagedLedgerFactory factory = new ManagedLedgerFactoryImpl(bkc, zkc, factoryConf);
- ManagedLedgerConfig config = new ManagedLedgerConfig();
- config.setEnsembleSize(1).setWriteQuorumSize(1).setAckQuorumSize(1).setMetadataEnsembleSize(1)
- .setMetadataAckQuorumSize(1);
- ManagedLedger ledger = factory.open("property/cluster/namespace/my-ledger", config);
- ManagedCursor cursor = ledger.openCursor("c1");
-
- int N = 1;
-
- for (int i = 0; i < N; i++) {
- String entry = "entry-" + i;
- ledger.addEntry(entry.getBytes());
- }
-
- List<Entry> entries = cursor.readEntries(N);
- assertEquals(N, entries.size());
- entries.forEach(e -> e.release());
- ledger.close();
-
- ManagedLedgerOfflineBacklog offlineTopicBacklog = new ManagedLedgerOfflineBacklog(
- DigestType.CRC32, "".getBytes(Charsets.UTF_8), "", false);
- PersistentOfflineTopicStats offlineTopicStats = offlineTopicBacklog.getEstimatedUnloadedTopicBacklog(
- (ManagedLedgerFactoryImpl) factory, "property/cluster/namespace/my-ledger");
- factory.shutdown();
- assertNotNull(offlineTopicStats);
- }
-
- @Test(timeOut = 20000)
- void testResetCursorAfterRecovery() throws Exception {
- ManagedLedgerFactory factory = new ManagedLedgerFactoryImpl(bkc, zkc);
- ManagedLedgerConfig conf = new ManagedLedgerConfig().setMaxEntriesPerLedger(10).setEnsembleSize(1)
- .setWriteQuorumSize(1).setAckQuorumSize(1).setMetadataEnsembleSize(1).setMetadataWriteQuorumSize(1)
- .setMetadataAckQuorumSize(1);
- ManagedLedger ledger = factory.open("my_test_move_cursor_ledger", conf);
- ManagedCursor cursor = ledger.openCursor("trc1");
- Position p1 = ledger.addEntry("dummy-entry-1".getBytes());
- Position p2 = ledger.addEntry("dummy-entry-2".getBytes());
- Position p3 = ledger.addEntry("dummy-entry-3".getBytes());
- Position p4 = ledger.addEntry("dummy-entry-4".getBytes());
-
- cursor.markDelete(p3);
-
- ManagedLedgerFactory factory2 = new ManagedLedgerFactoryImpl(bkc, zkc);
- ledger = factory2.open("my_test_move_cursor_ledger", conf);
- cursor = ledger.openCursor("trc1");
-
- assertEquals(cursor.getMarkDeletedPosition(), p3);
- assertEquals(cursor.getReadPosition(), p4);
- assertEquals(cursor.getNumberOfEntriesInBacklog(), 1);
-
- cursor.resetCursor(p2);
- assertEquals(cursor.getMarkDeletedPosition(), p1);
- assertEquals(cursor.getReadPosition(), p2);
- assertEquals(cursor.getNumberOfEntriesInBacklog(), 3);
-
- factory2.shutdown();
- factory.shutdown();
- }
-
- @Test(timeOut = 30000)
- public void managedLedgerClosed() throws Exception {
- ManagedLedgerFactoryImpl factory = new ManagedLedgerFactoryImpl(bkc, bkc.getZkHandle());
- ManagedLedgerConfig config = new ManagedLedgerConfig();
- config.setEnsembleSize(2).setAckQuorumSize(2).setMetadataEnsembleSize(2);
- ManagedLedgerImpl ledger1 = (ManagedLedgerImpl) factory.open("my_test_ledger", config);
-
- int N = 100;
-
- AtomicReference<ManagedLedgerException> res = new AtomicReference<>();
- CountDownLatch latch = new CountDownLatch(N);
-
- for (int i = 0; i < N; i++) {
- ledger1.asyncAddEntry(("entry-" + i).getBytes(), new AddEntryCallback() {
-
- @Override
- public void addComplete(Position position, Object ctx) {
- latch.countDown();
- }
-
- @Override
- public void addFailed(ManagedLedgerException exception, Object ctx) {
- res.compareAndSet(null, exception);
- latch.countDown();
- }
- }, null);
-
- if (i == 1) {
- ledger1.close();
- }
- }
-
- // Ensures all the callback must have been invoked
- latch.await();
- assertNotNull(res.get());
- assertEquals(res.get().getClass(), ManagedLedgerAlreadyClosedException.class);
- factory.shutdown();
- }
-
- @Test
- public void testChangeCrcType() throws Exception {
- ManagedLedgerFactoryImpl factory = new ManagedLedgerFactoryImpl(bkc, bkc.getZkHandle());
- ManagedLedgerConfig config = new ManagedLedgerConfig();
- config.setEnsembleSize(2).setAckQuorumSize(2).setMetadataEnsembleSize(2);
- config.setDigestType(DigestType.CRC32);
- ManagedLedger ledger = factory.open("my_test_ledger", config);
- ManagedCursor c1 = ledger.openCursor("c1");
-
- ledger.addEntry("entry-0".getBytes());
- ledger.addEntry("entry-1".getBytes());
- ledger.addEntry("entry-2".getBytes());
-
- ledger.close();
-
- config.setDigestType(DigestType.CRC32C);
- ledger = factory.open("my_test_ledger", config);
- c1 = ledger.openCursor("c1");
-
- ledger.addEntry("entry-3".getBytes());
-
- assertEquals(c1.getNumberOfEntries(), 4);
- assertEquals(c1.getNumberOfEntriesInBacklog(), 4);
-
- List<Entry> entries = c1.readEntries(4);
- assertEquals(entries.size(), 4);
- for (int i = 0; i < 4; i++) {
- assertEquals(new String(entries.get(i).getData()), "entry-" + i);
- }
-
- factory.shutdown();
- }
-
-}
diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerErrorsTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerErrorsTest.java
deleted file mode 100644
index 0f0d78917a..0000000000
--- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerErrorsTest.java
+++ /dev/null
@@ -1,528 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.bookkeeper.mledger.impl;
-
-import static org.testng.Assert.*;
-
-import java.util.List;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.atomic.AtomicReference;
-import org.apache.bookkeeper.client.BKException;
-import org.apache.bookkeeper.client.api.DigestType;
-import org.apache.bookkeeper.mledger.AsyncCallbacks.AddEntryCallback;
-import org.apache.bookkeeper.mledger.AsyncCallbacks.CloseCallback;
-import org.apache.bookkeeper.mledger.Entry;
-import org.apache.bookkeeper.mledger.ManagedCursor;
-import org.apache.bookkeeper.mledger.ManagedLedger;
-import org.apache.bookkeeper.mledger.ManagedLedgerConfig;
-import org.apache.bookkeeper.mledger.ManagedLedgerException;
-import org.apache.bookkeeper.mledger.ManagedLedgerException.BadVersionException;
-import org.apache.bookkeeper.mledger.ManagedLedgerException.ManagedLedgerFencedException;
-import org.apache.bookkeeper.mledger.ManagedLedgerFactory;
-import org.apache.bookkeeper.mledger.Position;
-import org.apache.bookkeeper.test.MockedBookKeeperTestCase;
-import org.apache.zookeeper.KeeperException.Code;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.annotations.Test;
-
-public class ManagedLedgerErrorsTest extends MockedBookKeeperTestCase {
-
- @Test
- public void removingCursor() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger");
- ManagedCursor c1 = ledger.openCursor("c1");
-
- assertEquals(zkc.exists("/managed-ledgers/my_test_ledger/c1", false) != null, true);
-
- zkc.failNow(Code.BADVERSION);
-
- try {
- c1.close();
- fail("should fail");
- } catch (ManagedLedgerException e) {
- // ok
- }
-
- bkc.failNow(BKException.Code.NoSuchLedgerExistsException);
-
- // Cursor ledger deletion will fail, but that should not prevent the deleteCursor to fail
- ledger.deleteCursor("c1");
-
- assertEquals(zkc.exists("/managed-ledgers/my_test_ledger/c1", false) != null, false);
- assertEquals(bkc.getLedgers().size(), 2);
- }
-
- @Test
- public void removingCursor2() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger");
- ledger.openCursor("c1");
-
- zkc.failNow(Code.CONNECTIONLOSS);
-
- try {
- ledger.deleteCursor("c1");
- fail("should fail");
- } catch (ManagedLedgerException e) {
- // ok
- }
- }
-
- @Test
- public void closingManagedLedger() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger");
- ledger.openCursor("c1");
- ledger.addEntry("entry".getBytes());
-
- bkc.failNow(BKException.Code.NoSuchLedgerExistsException);
-
- try {
- ledger.close();
- fail("should fail");
- } catch (ManagedLedgerException e) {
- // ok
- }
-
- // ML should be closed even if it failed before
- try {
- ledger.addEntry("entry".getBytes());
- fail("managed ledger was closed");
- } catch (ManagedLedgerException e) {
- // ok
- }
- }
-
- @Test
- public void asyncClosingManagedLedger() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger");
- ledger.openCursor("c1");
-
- bkc.failNow(BKException.Code.NoSuchLedgerExistsException);
-
- final CountDownLatch latch = new CountDownLatch(1);
- ledger.asyncClose(new CloseCallback() {
- public void closeFailed(ManagedLedgerException exception, Object ctx) {
- latch.countDown();
- }
-
- public void closeComplete(Object ctx) {
- fail("should have failed");
- }
- }, null);
-
- latch.await();
- }
-
- @Test
- public void errorInRecovering() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger");
- ledger.addEntry("entry".getBytes());
-
- ledger.close();
-
- factory = new ManagedLedgerFactoryImpl(bkc, zkc);
-
- bkc.failNow(BKException.Code.LedgerFencedException);
-
- try {
- ledger = factory.open("my_test_ledger");
- fail("should fail");
- } catch (ManagedLedgerException e) {
- // ok
- }
-
- // It should be fine now
- ledger = factory.open("my_test_ledger");
- }
-
- @Test
- public void errorInRecovering2() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger");
- ledger.addEntry("entry".getBytes());
-
- ledger.close();
-
- factory = new ManagedLedgerFactoryImpl(bkc, zkc);
-
- bkc.failAfter(1, BKException.Code.LedgerFencedException);
-
- try {
- ledger = factory.open("my_test_ledger");
- fail("should fail");
- } catch (ManagedLedgerException e) {
- // ok
- }
-
- // It should be fine now
- ledger = factory.open("my_test_ledger");
- }
-
- @Test
- public void errorInRecovering3() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger");
- ledger.addEntry("entry".getBytes());
-
- ledger.close();
-
- factory = new ManagedLedgerFactoryImpl(bkc, zkc);
-
- bkc.failAfter(1, BKException.Code.LedgerFencedException);
-
- try {
- ledger = factory.open("my_test_ledger");
- fail("should fail");
- } catch (ManagedLedgerException e) {
- // ok
- }
-
- // It should be fine now
- ledger = factory.open("my_test_ledger");
- }
-
- @Test
- public void errorInRecovering4() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger");
- ledger.addEntry("entry".getBytes());
-
- ledger.close();
-
- factory = new ManagedLedgerFactoryImpl(bkc, zkc);
-
- zkc.failAfter(1, Code.CONNECTIONLOSS);
-
- try {
- ledger = factory.open("my_test_ledger");
- fail("should fail");
- } catch (ManagedLedgerException e) {
- // ok
- }
-
- // It should be fine now
- ledger = factory.open("my_test_ledger");
- }
-
- @Test
- public void errorInRecovering5() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger");
- ledger.addEntry("entry".getBytes());
-
- ledger.close();
-
- factory = new ManagedLedgerFactoryImpl(bkc, zkc);
-
- zkc.failAfter(2, Code.CONNECTIONLOSS);
-
- try {
- ledger = factory.open("my_test_ledger");
- fail("should fail");
- } catch (ManagedLedgerException e) {
- // ok
- }
-
- // It should be fine now
- ledger = factory.open("my_test_ledger");
- }
-
- @Test
- public void errorInRecovering6() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger");
- ledger.openCursor("c1");
- ledger.addEntry("entry".getBytes());
-
- ledger.close();
-
- factory = new ManagedLedgerFactoryImpl(bkc, zkc);
-
- zkc.failAfter(3, Code.CONNECTIONLOSS);
-
- try {
- ledger = factory.open("my_test_ledger");
- fail("should fail");
- } catch (ManagedLedgerException e) {
- // ok
- }
-
- // It should be fine now
- ledger = factory.open("my_test_ledger");
- }
-
- @Test
- public void passwordError() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger", new ManagedLedgerConfig().setPassword("password"));
- ledger.openCursor("c1");
- ledger.addEntry("entry".getBytes());
-
- ledger.close();
-
- try {
- ledger = factory.open("my_test_ledger", new ManagedLedgerConfig().setPassword("wrong-password"));
- fail("should fail for password error");
- } catch (ManagedLedgerException e) {
- // ok
- }
- }
-
- @Test
- public void digestError() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger",
- new ManagedLedgerConfig().setDigestType(DigestType.CRC32));
- ledger.openCursor("c1");
- ledger.addEntry("entry".getBytes());
-
- ledger.close();
-
- try {
- ledger = factory.open("my_test_ledger", new ManagedLedgerConfig().setDigestType(DigestType.MAC));
- fail("should fail for digest error");
- } catch (ManagedLedgerException e) {
- // ok
- }
- }
-
- @Test(timeOut = 20000, invocationCount = 1, skipFailedInvocations = true, enabled = false)
- public void errorInUpdatingLedgersList() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger", new ManagedLedgerConfig().setMaxEntriesPerLedger(1));
-
- final CountDownLatch latch = new CountDownLatch(1);
-
- zkc.failAfter(0, Code.CONNECTIONLOSS);
-
- ledger.asyncAddEntry("entry".getBytes(), new AddEntryCallback() {
- public void addFailed(ManagedLedgerException exception, Object ctx) {
- // not-ok
- }
-
- public void addComplete(Position position, Object ctx) {
- // ok
- }
- }, null);
-
- ledger.asyncAddEntry("entry".getBytes(), new AddEntryCallback() {
- public void addFailed(ManagedLedgerException exception, Object ctx) {
- latch.countDown();
- }
-
- public void addComplete(Position position, Object ctx) {
- fail("should have failed");
- }
- }, null);
-
- latch.await();
- }
-
- @Test
- public void recoverAfterWriteError() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger");
- ManagedCursor cursor = ledger.openCursor("c1");
-
- bkc.failNow(BKException.Code.BookieHandleNotAvailableException);
-
- // With one single error, the write should succeed
- ledger.addEntry("entry".getBytes());
-
- assertEquals(cursor.getNumberOfEntriesInBacklog(), 1);
-
- bkc.failNow(BKException.Code.BookieHandleNotAvailableException);
- zkc.failNow(Code.CONNECTIONLOSS);
- try {
- ledger.addEntry("entry".getBytes());
- fail("should fail");
- } catch (ManagedLedgerException e) {
- // ok
- }
-
- assertEquals(cursor.getNumberOfEntriesInBacklog(), 1);
-
- // Next add will fail as well
- try {
- ledger.addEntry("entry".getBytes());
- fail("should fail");
- } catch (ManagedLedgerException e) {
- // ok
- }
-
- assertEquals(cursor.getNumberOfEntriesInBacklog(), 1);
- }
-
- @Test
- public void recoverAfterZnodeVersionError() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger", new ManagedLedgerConfig().setMaxEntriesPerLedger(1));
-
- zkc.failNow(Code.BADVERSION);
-
- // First write will succeed
- ledger.addEntry("test".getBytes());
-
- try {
- // This write will try to create a ledger and it will fail at it
- ledger.addEntry("entry".getBytes());
- fail("should fail");
- } catch (BadVersionException e) {
- // ok
- }
-
- try {
- // At this point the ledger should be fenced for good
- ledger.addEntry("entry".getBytes());
- fail("should fail");
- } catch (ManagedLedgerFencedException e) {
- // ok
- }
- }
-
- @Test
- public void recoverLongTimeAfterWriteError() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger");
- ManagedCursor cursor = ledger.openCursor("c1");
-
- bkc.failNow(BKException.Code.BookieHandleNotAvailableException);
-
- // With one single error, the write should succeed
- ledger.addEntry("entry-1".getBytes());
-
- assertEquals(cursor.getNumberOfEntriesInBacklog(), 1);
-
- bkc.failNow(BKException.Code.BookieHandleNotAvailableException);
- zkc.failNow(Code.CONNECTIONLOSS);
- try {
- ledger.addEntry("entry-2".getBytes());
- fail("should fail");
- } catch (ManagedLedgerException e) {
- // ok
- }
-
- Thread.sleep(ManagedLedgerImpl.WaitTimeAfterLedgerCreationFailureMs / 2);
- try {
- ledger.addEntry("entry-3".getBytes());
- fail("should fail");
- } catch (ManagedLedgerException e) {
- // ok
- }
-
- // After some time, the managed ledger will be available for writes again
- Thread.sleep(ManagedLedgerImpl.WaitTimeAfterLedgerCreationFailureMs / 2 + 10);
-
- // Next add should succeed, and the previous write should not appear
- ledger.addEntry("entry-4".getBytes());
-
- assertEquals(cursor.getNumberOfEntriesInBacklog(), 2);
-
- List<Entry> entries = cursor.readEntries(10);
- assertEquals(entries.size(), 2);
- assertEquals(new String(entries.get(0).getData()), "entry-1");
- assertEquals(new String(entries.get(1).getData()), "entry-4");
- entries.forEach(e -> e.release());
- }
-
- @Test
- public void recoverLongTimeAfterMultipleWriteErrors() throws Exception {
- ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("recoverLongTimeAfterMultipleWriteErrors");
- ManagedCursor cursor = ledger.openCursor("c1");
-
- bkc.failAfter(0, BKException.Code.BookieHandleNotAvailableException);
- bkc.failAfter(1, BKException.Code.BookieHandleNotAvailableException);
-
- CountDownLatch counter = new CountDownLatch(2);
- AtomicReference<ManagedLedgerException> ex = new AtomicReference<>();
-
- // Write 2 entries, both should fail the first time and get re-tried internally in a new ledger
- AddEntryCallback cb = new AddEntryCallback() {
-
- @Override
- public void addComplete(Position position, Object ctx) {
- counter.countDown();
- }
-
- @Override
- public void addFailed(ManagedLedgerException exception, Object ctx) {
- log.warn("Error in write", exception);
- ex.set(exception);
- counter.countDown();
- }
-
- };
- ledger.asyncAddEntry("entry-1".getBytes(), cb, null);
- ledger.asyncAddEntry("entry-2".getBytes(), cb, null);
-
- counter.await();
- assertNull(ex.get());
-
- assertEquals(cursor.getNumberOfEntriesInBacklog(), 2);
-
- // Ensure that we are only creating one new ledger
- // even when there are multiple (here, 2) add entry failed ops
- assertEquals(ledger.getLedgersInfoAsList().size(), 1);
-
- ledger.addEntry("entry-3".getBytes());
-
- List<Entry> entries = cursor.readEntries(10);
- assertEquals(entries.size(), 3);
- assertEquals(new String(entries.get(0).getData()), "entry-1");
- assertEquals(new String(entries.get(1).getData()), "entry-2");
- assertEquals(new String(entries.get(2).getData()), "entry-3");
- entries.forEach(e -> e.release());
- }
-
- @Test
- public void recoverAfterMarkDeleteError() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger");
- ManagedCursor cursor = ledger.openCursor("my-cursor");
- Position position = ledger.addEntry("entry".getBytes());
-
- bkc.failNow(BKException.Code.BookieHandleNotAvailableException);
- zkc.failNow(Code.CONNECTIONLOSS);
-
- try {
- cursor.markDelete(position);
- fail("should fail");
- } catch (ManagedLedgerException e) {
- // ok
- }
-
- // The metadata ledger is reopened in background, until it's not reopened the mark-delete will fail
- Thread.sleep(100);
-
- // Next markDelete should succeed
- cursor.markDelete(position);
- }
-
- @Test
- public void handleCursorRecoveryFailure() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger");
- ManagedCursor cursor = ledger.openCursor("my-cursor");
-
- Position p0 = cursor.getMarkDeletedPosition();
-
- Position p1 = ledger.addEntry("entry-1".getBytes());
- cursor.markDelete(p1);
-
- // Re-open from a different factory
- ManagedLedgerFactory factory2 = new ManagedLedgerFactoryImpl(bkc, zkc);
-
- bkc.failAfter(3, BKException.Code.LedgerRecoveryException);
- ledger = factory2.open("my_test_ledger");
- cursor = ledger.openCursor("my-cursor");
-
- // Since the cursor was rewind, it will be back to p0
- assertEquals(cursor.getMarkDeletedPosition(), p0);
- factory2.shutdown();
- }
-
- private static final Logger log = LoggerFactory.getLogger(ManagedLedgerErrorsTest.class);
-}
diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryTest.java
deleted file mode 100644
index b704db7168..0000000000
--- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerFactoryTest.java
+++ /dev/null
@@ -1,74 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.bookkeeper.mledger.impl;
-
-import static org.testng.Assert.assertEquals;
-
-import org.apache.bookkeeper.mledger.ManagedCursor;
-import org.apache.bookkeeper.mledger.ManagedLedgerConfig;
-import org.apache.bookkeeper.mledger.ManagedLedgerInfo;
-import org.apache.bookkeeper.mledger.ManagedLedgerInfo.CursorInfo;
-import org.apache.bookkeeper.mledger.ManagedLedgerInfo.MessageRangeInfo;
-import org.apache.bookkeeper.test.MockedBookKeeperTestCase;
-import org.testng.annotations.Test;
-
-public class ManagedLedgerFactoryTest extends MockedBookKeeperTestCase {
-
- @Test(timeOut = 20000)
- public void testGetManagedLedgerInfoWithClose() throws Exception {
- ManagedLedgerConfig conf = new ManagedLedgerConfig();
- conf.setMaxEntriesPerLedger(1);
- ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("testGetManagedLedgerInfo", conf);
- ManagedCursor c1 = ledger.openCursor("c1");
-
- PositionImpl p1 = (PositionImpl) ledger.addEntry("entry1".getBytes());
- PositionImpl p2 = (PositionImpl) ledger.addEntry("entry2".getBytes());
- PositionImpl p3 = (PositionImpl) ledger.addEntry("entry3".getBytes());
- ledger.addEntry("entry4".getBytes());
-
- c1.delete(p2);
- c1.delete(p3);
-
- ledger.close();
-
- ManagedLedgerInfo info = factory.getManagedLedgerInfo("testGetManagedLedgerInfo");
-
- assertEquals(info.ledgers.size(), 4);
-
- assertEquals(info.ledgers.get(0).ledgerId, 3);
- assertEquals(info.ledgers.get(1).ledgerId, 5);
- assertEquals(info.ledgers.get(2).ledgerId, 6);
- assertEquals(info.ledgers.get(3).ledgerId, 7);
-
- assertEquals(info.cursors.size(), 1);
-
- CursorInfo cursorInfo = info.cursors.get("c1");
- assertEquals(cursorInfo.markDelete.ledgerId, 3);
- assertEquals(cursorInfo.markDelete.entryId, -1);
-
- assertEquals(cursorInfo.individualDeletedMessages.size(), 1);
-
- MessageRangeInfo mri = cursorInfo.individualDeletedMessages.get(0);
- assertEquals(mri.from.ledgerId, p1.getLedgerId());
- assertEquals(mri.from.entryId, p1.getEntryId());
- assertEquals(mri.to.ledgerId, p3.getLedgerId());
- assertEquals(mri.to.entryId, p3.getEntryId());
- }
-
-}
diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerMBeanTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerMBeanTest.java
deleted file mode 100644
index ebaa1fdab2..0000000000
--- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerMBeanTest.java
+++ /dev/null
@@ -1,136 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.bookkeeper.mledger.impl;
-
-import static org.testng.Assert.assertEquals;
-import static org.testng.Assert.assertTrue;
-
-import java.util.List;
-import java.util.concurrent.TimeUnit;
-import org.apache.bookkeeper.mledger.Entry;
-import org.apache.bookkeeper.mledger.ManagedCursor;
-import org.apache.bookkeeper.mledger.ManagedLedgerFactoryConfig;
-import org.apache.bookkeeper.mledger.Position;
-import org.apache.bookkeeper.test.MockedBookKeeperTestCase;
-import org.testng.annotations.Test;
-
-public class ManagedLedgerMBeanTest extends MockedBookKeeperTestCase {
-
- private void waitForRefresh(ManagedLedgerMBeanImpl mbean) {
- try {
- Thread.sleep(100);
- } catch (Exception e) {
- // do nothing
- }
- }
-
- @Test
- public void simple() throws Exception {
- ManagedLedgerFactoryConfig config = new ManagedLedgerFactoryConfig();
- config.setMaxCacheSize(0);
- ManagedLedgerFactoryImpl factory = new ManagedLedgerFactoryImpl(bkc, zkc, config);
- ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("my_test_ledger");
- ManagedCursor cursor = ledger.openCursor("c1");
- ManagedLedgerMBeanImpl mbean = ledger.mbean;
-
- assertEquals(mbean.getName(), "my_test_ledger");
-
- assertEquals(mbean.getStoredMessagesSize(), 0);
- assertEquals(mbean.getNumberOfMessagesInBacklog(), 0);
-
- waitForRefresh(mbean);
-
- mbean.addAddEntryLatencySample(1, TimeUnit.MILLISECONDS);
- mbean.addAddEntryLatencySample(10, TimeUnit.MILLISECONDS);
- mbean.addAddEntryLatencySample(1, TimeUnit.SECONDS);
-
- mbean.addLedgerSwitchLatencySample(1, TimeUnit.MILLISECONDS);
- mbean.addLedgerSwitchLatencySample(10, TimeUnit.MILLISECONDS);
- mbean.addLedgerSwitchLatencySample(1, TimeUnit.SECONDS);
-
- // Simulate stats getting update from different thread
- factory.scheduledExecutor.submit(() -> {
- mbean.refreshStats(1, TimeUnit.SECONDS);
- }).get();
-
- assertEquals(mbean.getAddEntryBytesRate(), 0.0);
- assertEquals(mbean.getAddEntryMessagesRate(), 0.0);
- assertEquals(mbean.getAddEntrySucceed(), 0);
- assertEquals(mbean.getAddEntryErrors(), 0);
- assertEquals(mbean.getReadEntriesBytesRate(), 0.0);
- assertEquals(mbean.getReadEntriesRate(), 0.0);
- assertEquals(mbean.getReadEntriesSucceeded(), 0);
- assertEquals(mbean.getReadEntriesErrors(), 0);
- assertEquals(mbean.getMarkDeleteRate(), 0.0);
-
- assertEquals(mbean.getAddEntryLatencyBuckets(), new long[] { 0, 1, 0, 1, 0, 0, 0, 0, 1, 0 });
- assertEquals(mbean.getAddEntryLatencyAverageUsec(), 337_000.0);
- assertEquals(mbean.getEntrySizeBuckets(), new long[] { 0, 0, 0, 0, 0, 0, 0, 0, 0 });
-
- assertEquals(mbean.getLedgerSwitchLatencyBuckets(), new long[] { 0, 1, 0, 1, 0, 0, 0, 0, 1, 0 });
- assertEquals(mbean.getLedgerSwitchLatencyAverageUsec(), 337_000.0);
-
- Position p1 = ledger.addEntry(new byte[200]);
- ledger.addEntry(new byte[600]);
- cursor.markDelete(p1);
-
- factory.scheduledExecutor.submit(() -> {
- mbean.refreshStats(1, TimeUnit.SECONDS);
- }).get();
-
- assertEquals(mbean.getAddEntryBytesRate(), 800.0);
- assertEquals(mbean.getAddEntryMessagesRate(), 2.0);
- assertEquals(mbean.getAddEntrySucceed(), 2);
- assertEquals(mbean.getAddEntryErrors(), 0);
- assertEquals(mbean.getReadEntriesBytesRate(), 0.0);
- assertEquals(mbean.getReadEntriesRate(), 0.0);
- assertEquals(mbean.getReadEntriesSucceeded(), 0);
- assertEquals(mbean.getReadEntriesErrors(), 0);
- assertTrue(mbean.getMarkDeleteRate() > 0.0);
-
- assertEquals(mbean.getEntrySizeBuckets(), new long[] { 0, 1, 1, 0, 0, 0, 0, 0, 0 });
-
- mbean.recordAddEntryError();
- mbean.recordReadEntriesError();
-
- factory.scheduledExecutor.submit(() -> {
- mbean.refreshStats(1, TimeUnit.SECONDS);
- }).get();
-
- assertEquals(mbean.getAddEntryErrors(), 1);
- assertEquals(mbean.getReadEntriesErrors(), 1);
-
- List<Entry> entries = cursor.readEntries(100);
- assertEquals(entries.size(), 1);
-
- factory.scheduledExecutor.submit(() -> {
- mbean.refreshStats(1, TimeUnit.SECONDS);
- }).get();
-
- assertEquals(mbean.getReadEntriesBytesRate(), 600.0);
- assertEquals(mbean.getReadEntriesRate(), 1.0);
- assertEquals(mbean.getReadEntriesSucceeded(), 1);
- assertEquals(mbean.getReadEntriesErrors(), 0);
- assertEquals(mbean.getNumberOfMessagesInBacklog(), 1);
- assertEquals(mbean.getMarkDeleteRate(), 0.0);
-
- factory.shutdown();
- }
-
-}
diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerSingleBookieTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerSingleBookieTest.java
deleted file mode 100644
index f663de5221..0000000000
--- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerSingleBookieTest.java
+++ /dev/null
@@ -1,77 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.bookkeeper.mledger.impl;
-
-import static org.testng.Assert.assertEquals;
-
-import com.google.common.base.Charsets;
-import java.nio.charset.Charset;
-import java.util.ArrayList;
-import java.util.List;
-import org.apache.bookkeeper.mledger.Entry;
-import org.apache.bookkeeper.mledger.ManagedCursor;
-import org.apache.bookkeeper.mledger.ManagedLedger;
-import org.apache.bookkeeper.mledger.ManagedLedgerConfig;
-import org.apache.bookkeeper.test.MockedBookKeeperTestCase;
-import org.testng.annotations.Test;
-
-public class ManagedLedgerSingleBookieTest extends MockedBookKeeperTestCase {
-
- private static final Charset Encoding = Charsets.UTF_8;
-
- public ManagedLedgerSingleBookieTest() {
- // Just one bookie
- super(1);
- }
-
- @Test // (timeOut = 20000)
- public void simple() throws Exception {
- ManagedLedgerConfig config = new ManagedLedgerConfig().setEnsembleSize(1).setWriteQuorumSize(1)
- .setAckQuorumSize(1).setMetadataEnsembleSize(1).setMetadataWriteQuorumSize(1)
- .setMetadataAckQuorumSize(1);
- ManagedLedger ledger = factory.open("my_test_ledger", config);
-
- assertEquals(ledger.getNumberOfEntries(), 0);
- assertEquals(ledger.getTotalSize(), 0);
-
- ledger.addEntry("dummy-entry-1".getBytes(Encoding));
-
- assertEquals(ledger.getNumberOfEntries(), 1);
- assertEquals(ledger.getTotalSize(), "dummy-entry-1".getBytes(Encoding).length);
-
- ManagedCursor cursor = ledger.openCursor("c1");
-
- assertEquals(cursor.hasMoreEntries(), false);
- assertEquals(cursor.readEntries(100), new ArrayList<Entry>());
-
- ledger.addEntry("dummy-entry-2".getBytes(Encoding));
-
- assertEquals(cursor.hasMoreEntries(), true);
-
- List<Entry> entries = cursor.readEntries(100);
- assertEquals(entries.size(), 1);
- entries.forEach(e -> e.release());
-
- entries = cursor.readEntries(100);
- assertEquals(entries.size(), 0);
- entries.forEach(e -> e.release());
-
- ledger.close();
- }
-}
diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTerminationTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTerminationTest.java
deleted file mode 100644
index 052c9fb5b6..0000000000
--- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTerminationTest.java
+++ /dev/null
@@ -1,162 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.bookkeeper.mledger.impl;
-
-import static org.testng.Assert.assertEquals;
-import static org.testng.Assert.fail;
-
-import java.util.Collections;
-import java.util.List;
-import org.apache.bookkeeper.mledger.Entry;
-import org.apache.bookkeeper.mledger.ManagedCursor;
-import org.apache.bookkeeper.mledger.ManagedLedger;
-import org.apache.bookkeeper.mledger.ManagedLedgerException.ManagedLedgerTerminatedException;
-import org.apache.bookkeeper.mledger.ManagedLedgerException.NoMoreEntriesToReadException;
-import org.apache.bookkeeper.mledger.Position;
-import org.apache.bookkeeper.test.MockedBookKeeperTestCase;
-import org.testng.annotations.Test;
-
-public class ManagedLedgerTerminationTest extends MockedBookKeeperTestCase {
-
- @Test(timeOut = 20000)
- public void terminateSimple() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger");
-
- Position p0 = ledger.addEntry("entry-0".getBytes());
-
- Position lastPosition = ledger.terminate();
-
- assertEquals(lastPosition, p0);
-
- try {
- ledger.addEntry("entry-1".getBytes());
- } catch (ManagedLedgerTerminatedException e) {
- // Expected
- }
- }
-
- @Test(timeOut = 20000)
- public void terminateReopen() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger");
-
- Position p0 = ledger.addEntry("entry-0".getBytes());
-
- Position lastPosition = ledger.terminate();
-
- assertEquals(lastPosition, p0);
-
- ledger.close();
-
- ledger = factory.open("my_test_ledger");
-
- try {
- ledger.addEntry("entry-1".getBytes());
- fail("Should have thrown exception");
- } catch (ManagedLedgerTerminatedException e) {
- // Expected
- }
- }
-
- @Test(timeOut = 20000)
- public void terminateWithCursor() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger");
- ManagedCursor c1 = ledger.openCursor("c1");
-
- Position p0 = ledger.addEntry("entry-0".getBytes());
- Position p1 = ledger.addEntry("entry-1".getBytes());
-
- List<Entry> entries = c1.readEntries(1);
- assertEquals(entries.size(), 1);
- assertEquals(entries.get(0).getPosition(), p0);
- entries.forEach(Entry::release);
-
- Position lastPosition = ledger.terminate();
- assertEquals(lastPosition, p1);
-
- // Cursor can keep reading
- entries = c1.readEntries(1);
- assertEquals(entries.size(), 1);
- assertEquals(entries.get(0).getPosition(), p1);
- entries.forEach(Entry::release);
- }
-
- @Test(timeOut = 20000)
- public void terminateWithCursorReadOrWait() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger");
- ManagedCursor c1 = ledger.openCursor("c1");
-
- Position p0 = ledger.addEntry("entry-0".getBytes());
- Position p1 = ledger.addEntry("entry-1".getBytes());
- assertEquals(ledger.isTerminated(), false);
-
- Position lastPosition = ledger.terminate();
- assertEquals(ledger.isTerminated(), true);
- assertEquals(lastPosition, p1);
-
- List<Entry> entries = c1.readEntries(10);
- assertEquals(entries.size(), 2);
- assertEquals(entries.get(0).getPosition(), p0);
- assertEquals(entries.get(1).getPosition(), p1);
- entries.forEach(Entry::release);
-
- // Normal read will just return no entries
- assertEquals(c1.readEntries(10), Collections.emptyList());
-
- // Read or wait will fail
- try {
- c1.readEntriesOrWait(10);
- fail("Should have thrown exception");
- } catch (NoMoreEntriesToReadException e) {
- // Expected
- }
- }
-
- @Test(timeOut = 20000)
- public void terminateWithNonDurableCursor() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger");
-
- Position p0 = ledger.addEntry("entry-0".getBytes());
- Position p1 = ledger.addEntry("entry-1".getBytes());
- assertEquals(ledger.isTerminated(), false);
-
- Position lastPosition = ledger.terminate();
- assertEquals(ledger.isTerminated(), true);
- assertEquals(lastPosition, p1);
-
- ManagedCursor c1 = ledger.newNonDurableCursor(PositionImpl.earliest);
-
- List<Entry> entries = c1.readEntries(10);
- assertEquals(entries.size(), 2);
- assertEquals(entries.get(0).getPosition(), p0);
- assertEquals(entries.get(1).getPosition(), p1);
- entries.forEach(Entry::release);
-
- // Normal read will just return no entries
- assertEquals(c1.readEntries(10), Collections.emptyList());
-
- // Read or wait will fail
- try {
- c1.readEntriesOrWait(10);
- fail("Should have thrown exception");
- } catch (NoMoreEntriesToReadException e) {
- // Expected
- }
- }
-
-}
diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java
deleted file mode 100644
index aa85d8027b..0000000000
--- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ManagedLedgerTest.java
+++ /dev/null
@@ -1,2251 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.bookkeeper.mledger.impl;
-
-import static org.mockito.Matchers.any;
-import static org.mockito.Matchers.anyInt;
-import static org.mockito.Mockito.doNothing;
-import static org.mockito.Mockito.mock;
-import static org.testng.Assert.assertEquals;
-import static org.testng.Assert.assertFalse;
-import static org.testng.Assert.assertNotNull;
-import static org.testng.Assert.assertNull;
-import static org.testng.Assert.assertTrue;
-import static org.testng.Assert.fail;
-
-import java.io.IOException;
-import java.lang.reflect.Field;
-import java.lang.reflect.Modifier;
-import java.nio.charset.Charset;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Set;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.CyclicBarrier;
-import java.util.concurrent.Executor;
-import java.util.concurrent.Executors;
-import java.util.concurrent.RejectedExecutionException;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicReference;
-import java.util.function.Predicate;
-
-import org.apache.bookkeeper.client.AsyncCallback.CreateCallback;
-import org.apache.bookkeeper.client.BKException;
-import org.apache.bookkeeper.client.BookKeeper;
-import org.apache.bookkeeper.client.BookKeeper.DigestType;
-import org.apache.bookkeeper.client.LedgerHandle;
-import org.apache.bookkeeper.mledger.AsyncCallbacks.AddEntryCallback;
-import org.apache.bookkeeper.mledger.AsyncCallbacks.CloseCallback;
-import org.apache.bookkeeper.mledger.AsyncCallbacks.DeleteLedgerCallback;
-import org.apache.bookkeeper.mledger.AsyncCallbacks.MarkDeleteCallback;
-import org.apache.bookkeeper.mledger.AsyncCallbacks.OpenCursorCallback;
-import org.apache.bookkeeper.mledger.AsyncCallbacks.OpenLedgerCallback;
-import org.apache.bookkeeper.mledger.AsyncCallbacks.ReadEntriesCallback;
-import org.apache.bookkeeper.mledger.Entry;
-import org.apache.bookkeeper.mledger.ManagedCursor;
-import org.apache.bookkeeper.mledger.ManagedCursor.IndividualDeletedEntries;
-import org.apache.bookkeeper.mledger.ManagedLedger;
-import org.apache.bookkeeper.mledger.ManagedLedgerConfig;
-import org.apache.bookkeeper.mledger.ManagedLedgerException;
-import org.apache.bookkeeper.mledger.ManagedLedgerException.ManagedLedgerFencedException;
-import org.apache.bookkeeper.mledger.ManagedLedgerException.ManagedLedgerNotFoundException;
-import org.apache.bookkeeper.mledger.ManagedLedgerException.MetaStoreException;
-import org.apache.bookkeeper.mledger.ManagedLedgerFactory;
-import org.apache.bookkeeper.mledger.Position;
-import org.apache.bookkeeper.mledger.impl.MetaStore.MetaStoreCallback;
-import org.apache.bookkeeper.mledger.impl.MetaStore.Stat;
-import org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedLedgerInfo;
-import org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedLedgerInfo.LedgerInfo;
-import org.apache.bookkeeper.test.MockedBookKeeperTestCase;
-import org.apache.commons.lang3.tuple.Pair;
-import org.apache.pulsar.common.api.ByteBufPair;
-import org.apache.pulsar.common.api.proto.PulsarApi.CommandSubscribe.InitialPosition;
-import org.apache.pulsar.common.api.proto.PulsarApi.MessageMetadata;
-import org.apache.pulsar.common.util.protobuf.ByteBufCodedOutputStream;
-import org.apache.zookeeper.CreateMode;
-import org.apache.zookeeper.KeeperException.Code;
-import org.apache.zookeeper.ZooDefs;
-import org.apache.zookeeper.ZooKeeper;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.annotations.Test;
-
-import com.google.common.base.Charsets;
-import com.google.common.collect.Sets;
-
-import io.netty.buffer.ByteBuf;
-import io.netty.buffer.PooledByteBufAllocator;
-import io.netty.buffer.Unpooled;
-
-public class ManagedLedgerTest extends MockedBookKeeperTestCase {
-
- private static final Logger log = LoggerFactory.getLogger(ManagedLedgerTest.class);
-
- private static final Charset Encoding = Charsets.UTF_8;
-
- @Test
- public void managedLedgerApi() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger");
-
- ManagedCursor cursor = ledger.openCursor("c1");
-
- for (int i = 0; i < 100; i++) {
- String content = "entry-" + i;
- ledger.addEntry(content.getBytes());
- }
-
- // Reads all the entries in batches of 20
- while (cursor.hasMoreEntries()) {
-
- List<Entry> entries = cursor.readEntries(20);
- log.debug("Read {} entries", entries.size());
-
- // Acknowledge only on last entry
- Entry lastEntry = entries.get(entries.size() - 1);
- cursor.markDelete(lastEntry.getPosition());
-
- for (Entry entry : entries) {
- log.info("Read entry. Position={} Content='{}'", entry.getPosition(), new String(entry.getData()));
- entry.release();
- }
-
- log.info("-----------------------");
- }
-
- log.info("Finished reading entries");
-
- ledger.close();
- }
-
- @Test(timeOut = 20000)
- public void simple() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger");
-
- assertEquals(ledger.getNumberOfEntries(), 0);
- assertEquals(ledger.getNumberOfActiveEntries(), 0);
- assertEquals(ledger.getTotalSize(), 0);
-
- ledger.addEntry("dummy-entry-1".getBytes(Encoding));
-
- assertEquals(ledger.getNumberOfEntries(), 1);
- assertEquals(ledger.getNumberOfActiveEntries(), 0);
- assertEquals(ledger.getTotalSize(), "dummy-entry-1".getBytes(Encoding).length);
-
- ManagedCursor cursor = ledger.openCursor("c1");
-
- assertEquals(cursor.hasMoreEntries(), false);
- assertEquals(cursor.getNumberOfEntries(), 0);
- assertEquals(cursor.getNumberOfEntriesInBacklog(), 0);
- assertEquals(cursor.readEntries(100), new ArrayList<Entry>());
-
- ledger.addEntry("dummy-entry-2".getBytes(Encoding));
-
- assertEquals(cursor.hasMoreEntries(), true);
- assertEquals(cursor.getNumberOfEntries(), 1);
- assertEquals(cursor.getNumberOfEntriesInBacklog(), 1);
- assertEquals(ledger.getNumberOfActiveEntries(), 1);
-
- List<Entry> entries = cursor.readEntries(100);
- assertEquals(entries.size(), 1);
- entries.forEach(e -> e.release());
-
- entries = cursor.readEntries(100);
- assertEquals(entries.size(), 0);
-
- ledger.close();
- factory.shutdown();
- }
-
- @Test(timeOut = 20000)
- public void closeAndReopen() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger");
-
- ledger.addEntry("dummy-entry-1".getBytes(Encoding));
-
- ManagedCursor cursor = ledger.openCursor("c1");
-
- ledger.addEntry("dummy-entry-2".getBytes(Encoding));
-
- ledger.close();
-
- log.info("Closing ledger and reopening");
-
- // / Reopen the same managed-ledger
- ManagedLedgerFactoryImpl factory2 = new ManagedLedgerFactoryImpl(bkc, bkc.getZkHandle());
- ledger = factory2.open("my_test_ledger");
-
- cursor = ledger.openCursor("c1");
-
- assertEquals(ledger.getNumberOfEntries(), 2);
- assertEquals(ledger.getTotalSize(), "dummy-entry-1".getBytes(Encoding).length * 2);
-
- List<Entry> entries = cursor.readEntries(100);
- assertEquals(entries.size(), 1);
- entries.forEach(e -> e.release());
-
- ledger.close();
- factory2.shutdown();
- }
-
- @Test(timeOut = 20000)
- public void acknowledge1() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger");
-
- ManagedCursor cursor = ledger.openCursor("c1");
-
- ledger.addEntry("dummy-entry-1".getBytes(Encoding));
- ledger.addEntry("dummy-entry-2".getBytes(Encoding));
-
- assertEquals(cursor.hasMoreEntries(), true);
-
- List<Entry> entries = cursor.readEntries(2);
- assertEquals(entries.size(), 2);
-
- assertEquals(cursor.getNumberOfEntries(), 0);
- assertEquals(cursor.getNumberOfEntriesInBacklog(), 2);
- assertEquals(cursor.hasMoreEntries(), false);
-
- assertEquals(ledger.getNumberOfEntries(), 2);
- assertEquals(ledger.getNumberOfActiveEntries(), 2);
- cursor.markDelete(entries.get(0).getPosition());
- entries.forEach(e -> e.release());
-
- assertEquals(cursor.getNumberOfEntries(), 0);
- assertEquals(cursor.getNumberOfEntriesInBacklog(), 1);
- assertEquals(cursor.hasMoreEntries(), false);
- assertEquals(ledger.getNumberOfActiveEntries(), 1);
-
- ledger.close();
-
- // / Reopen the same managed-ledger
-
- ledger = factory.open("my_test_ledger");
- cursor = ledger.openCursor("c1");
-
- assertEquals(ledger.getNumberOfEntries(), 2);
- assertEquals(ledger.getTotalSize(), "dummy-entry-1".getBytes(Encoding).length * 2);
-
- assertEquals(cursor.getNumberOfEntries(), 1);
- assertEquals(cursor.getNumberOfEntriesInBacklog(), 1);
- assertEquals(cursor.hasMoreEntries(), true);
-
- entries = cursor.readEntries(100);
- assertEquals(entries.size(), 1);
- entries.forEach(e -> e.release());
-
- ledger.close();
- }
-
- @Test(timeOut = 20000)
- public void asyncAPI() throws Throwable {
- final CountDownLatch counter = new CountDownLatch(1);
-
- factory.asyncOpen("my_test_ledger", new ManagedLedgerConfig(), new OpenLedgerCallback() {
- @Override
- public void openLedgerComplete(ManagedLedger ledger, Object ctx) {
- ledger.asyncOpenCursor("test-cursor", new OpenCursorCallback() {
- @Override
- public void openCursorComplete(ManagedCursor cursor, Object ctx) {
- ManagedLedger ledger = (ManagedLedger) ctx;
-
- ledger.asyncAddEntry("test".getBytes(Encoding), new AddEntryCallback() {
- @Override
- public void addComplete(Position position, Object ctx) {
- @SuppressWarnings("unchecked")
- Pair<ManagedLedger, ManagedCursor> pair = (Pair<ManagedLedger, ManagedCursor>) ctx;
- ManagedLedger ledger = pair.getLeft();
- ManagedCursor cursor = pair.getRight();
-
- assertEquals(ledger.getNumberOfEntries(), 1);
- assertEquals(ledger.getTotalSize(), "test".getBytes(Encoding).length);
-
- cursor.asyncReadEntries(2, new ReadEntriesCallback() {
- @Override
- public void readEntriesComplete(List<Entry> entries, Object ctx) {
- ManagedCursor cursor = (ManagedCursor) ctx;
-
- assertEquals(entries.size(), 1);
- Entry entry = entries.get(0);
- final Position position = entry.getPosition();
- assertEquals(new String(entry.getDataAndRelease(), Encoding), "test");
-
- log.debug("Mark-Deleting to position {}", position);
- cursor.asyncMarkDelete(position, new MarkDeleteCallback() {
- @Override
- public void markDeleteComplete(Object ctx) {
- log.debug("Mark delete complete");
- ManagedCursor cursor = (ManagedCursor) ctx;
- assertEquals(cursor.hasMoreEntries(), false);
-
- counter.countDown();
- }
-
- @Override
- public void markDeleteFailed(ManagedLedgerException exception, Object ctx) {
- fail(exception.getMessage());
- }
-
- }, cursor);
- }
-
- @Override
- public void readEntriesFailed(ManagedLedgerException exception, Object ctx) {
- fail(exception.getMessage());
- }
- }, cursor);
- }
-
- @Override
- public void addFailed(ManagedLedgerException exception, Object ctx) {
- fail(exception.getMessage());
- }
- }, Pair.of(ledger, cursor));
- }
-
- @Override
- public void openCursorFailed(ManagedLedgerException exception, Object ctx) {
- fail(exception.getMessage());
- }
-
- }, ledger);
- }
-
- @Override
- public void openLedgerFailed(ManagedLedgerException exception, Object ctx) {
- fail(exception.getMessage());
- }
- }, null);
-
- counter.await();
-
- log.info("Test completed");
- }
-
- @Test(timeOut = 20000)
- public void spanningMultipleLedgers() throws Exception {
- ManagedLedgerConfig config = new ManagedLedgerConfig().setMaxEntriesPerLedger(10);
- ManagedLedger ledger = factory.open("my_test_ledger", config);
-
- assertEquals(ledger.getNumberOfEntries(), 0);
- assertEquals(ledger.getTotalSize(), 0);
-
- ManagedCursor cursor = ledger.openCursor("c1");
-
- for (int i = 0; i < 11; i++)
- ledger.addEntry(("dummy-entry-" + i).getBytes(Encoding));
-
- List<Entry> entries = cursor.readEntries(100);
- assertEquals(entries.size(), 11);
- assertEquals(cursor.hasMoreEntries(), false);
-
- PositionImpl first = (PositionImpl) entries.get(0).getPosition();
- PositionImpl last = (PositionImpl) entries.get(entries.size() - 1).getPosition();
- entries.forEach(e -> e.release());
-
- log.info("First={} Last={}", first, last);
- assertTrue(first.getLedgerId() < last.getLedgerId());
- assertEquals(first.getEntryId(), 0);
- assertEquals(last.getEntryId(), 0);
-
- // Read again, from next ledger id
- entries = cursor.readEntries(100);
- assertEquals(entries.size(), 0);
- assertEquals(cursor.hasMoreEntries(), false);
-
- ledger.close();
- }
-
- @Test(timeOut = 20000)
- public void spanningMultipleLedgersWithSize() throws Exception {
- ManagedLedgerConfig config = new ManagedLedgerConfig().setMaxEntriesPerLedger(1000000);
- config.setMaxSizePerLedgerMb(1);
- config.setEnsembleSize(1);
- config.setWriteQuorumSize(1).setAckQuorumSize(1);
- config.setMetadataWriteQuorumSize(1).setMetadataAckQuorumSize(1);
- ManagedLedger ledger = factory.open("my_test_ledger", config);
-
- assertEquals(ledger.getNumberOfEntries(), 0);
- assertEquals(ledger.getTotalSize(), 0);
-
- ManagedCursor cursor = ledger.openCursor("c1");
-
- byte[] content = new byte[1023 * 1024];
-
- for (int i = 0; i < 3; i++)
- ledger.addEntry(content);
-
- List<Entry> entries = cursor.readEntries(100);
- assertEquals(entries.size(), 3);
- assertEquals(cursor.hasMoreEntries(), false);
-
- PositionImpl first = (PositionImpl) entries.get(0).getPosition();
- PositionImpl last = (PositionImpl) entries.get(entries.size() - 1).getPosition();
- entries.forEach(e -> e.release());
-
- // Read again, from next ledger id
- entries = cursor.readEntries(100);
- assertEquals(entries.size(), 0);
- assertEquals(cursor.hasMoreEntries(), false);
- entries.forEach(e -> e.release());
-
- log.info("First={} Last={}", first, last);
- assertTrue(first.getLedgerId() < last.getLedgerId());
- assertEquals(first.getEntryId(), 0);
- assertEquals(last.getEntryId(), 0);
- ledger.close();
- }
-
- @Test(expectedExceptions = IllegalArgumentException.class)
- public void invalidReadEntriesArg1() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger");
- ManagedCursor cursor = ledger.openCursor("c1");
-
- ledger.addEntry("entry".getBytes());
- cursor.readEntries(-1);
-
- fail("Should have thrown an exception in the above line");
- }
-
- @Test(expectedExceptions = IllegalArgumentException.class)
- public void invalidReadEntriesArg2() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger");
- ManagedCursor cursor = ledger.openCursor("c1");
-
- ledger.addEntry("entry".getBytes());
- cursor.readEntries(0);
-
- fail("Should have thrown an exception in the above line");
- }
-
- @Test(timeOut = 20000)
- public void deleteAndReopen() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger");
-
- ledger.addEntry("dummy-entry-1".getBytes(Encoding));
- assertEquals(ledger.getNumberOfEntries(), 1);
- ledger.close();
-
- // Reopen
- ledger = factory.open("my_test_ledger");
- assertEquals(ledger.getNumberOfEntries(), 1);
-
- // Delete and reopen
- ledger.delete();
- ledger = factory.open("my_test_ledger");
- assertEquals(ledger.getNumberOfEntries(), 0);
- ledger.close();
- }
-
- @Test(timeOut = 20000)
- public void deleteAndReopenWithCursors() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger");
- ledger.openCursor("test-cursor");
-
- ledger.addEntry("dummy-entry-1".getBytes(Encoding));
- assertEquals(ledger.getNumberOfEntries(), 1);
- ledger.close();
-
- // Reopen
- ledger = factory.open("my_test_ledger");
- assertEquals(ledger.getNumberOfEntries(), 1);
-
- // Delete and reopen
- ledger.delete();
- ledger = factory.open("my_test_ledger");
- assertEquals(ledger.getNumberOfEntries(), 0);
- ManagedCursor cursor = ledger.openCursor("test-cursor");
- assertEquals(cursor.hasMoreEntries(), false);
- ledger.close();
- }
-
- @Test(timeOut = 20000)
- public void asyncDeleteWithError() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger");
- ledger.openCursor("test-cursor");
-
- ledger.addEntry("dummy-entry-1".getBytes(Encoding));
- assertEquals(ledger.getNumberOfEntries(), 1);
- ledger.close();
-
- // Reopen
- ledger = factory.open("my_test_ledger");
- assertEquals(ledger.getNumberOfEntries(), 1);
-
- final CountDownLatch counter = new CountDownLatch(1);
- stopBookKeeper();
- stopZooKeeper();
-
- // Delete and reopen
- factory.open("my_test_ledger", new ManagedLedgerConfig()).asyncDelete(new DeleteLedgerCallback() {
-
- @Override
- public void deleteLedgerComplete(Object ctx) {
- assertNull(ctx);
- fail("The async-call should have failed");
- }
-
- @Override
- public void deleteLedgerFailed(ManagedLedgerException exception, Object ctx) {
- counter.countDown();
- }
-
- }, null);
-
- counter.await();
- }
-
- @Test(timeOut = 20000)
- public void asyncAddEntryWithoutError() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger");
- ledger.openCursor("test-cursor");
-
- final CountDownLatch counter = new CountDownLatch(1);
-
- ledger.asyncAddEntry("dummy-entry-1".getBytes(Encoding), new AddEntryCallback() {
- @Override
- public void addComplete(Position position, Object ctx) {
- assertNull(ctx);
-
- counter.countDown();
- }
-
- @Override
- public void addFailed(ManagedLedgerException exception, Object ctx) {
- fail(exception.getMessage());
- }
-
- }, null);
-
- counter.await();
- assertEquals(ledger.getNumberOfEntries(), 1);
- assertEquals(ledger.getTotalSize(), "dummy-entry-1".getBytes(Encoding).length);
- }
-
- @Test(timeOut = 20000)
- public void doubleAsyncAddEntryWithoutError() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger");
- ledger.openCursor("test-cursor");
-
- final CountDownLatch done = new CountDownLatch(10);
-
- for (int i = 0; i < 10; i++) {
- final String content = "dummy-entry-" + i;
- ledger.asyncAddEntry(content.getBytes(Encoding), new AddEntryCallback() {
- @Override
- public void addComplete(Position position, Object ctx) {
- assertNotNull(ctx);
-
- log.info("Successfully added {}", content);
- done.countDown();
- }
-
- @Override
- public void addFailed(ManagedLedgerException exception, Object ctx) {
- fail(exception.getMessage());
- }
-
- }, this);
- }
-
- done.await();
- assertEquals(ledger.getNumberOfEntries(), 10);
- }
-
- @Test(timeOut = 20000)
- public void asyncAddEntryWithError() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger");
- ledger.openCursor("test-cursor");
-
- final CountDownLatch counter = new CountDownLatch(1);
- stopBookKeeper();
- stopZooKeeper();
-
- ledger.asyncAddEntry("dummy-entry-1".getBytes(Encoding), new AddEntryCallback() {
- @Override
- public void addComplete(Position position, Object ctx) {
- fail("Should have failed");
- }
-
- @Override
- public void addFailed(ManagedLedgerException exception, Object ctx) {
- counter.countDown();
- }
-
- }, null);
-
- counter.await();
- }
-
- @Test(timeOut = 20000)
- public void asyncCloseWithoutError() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger");
- ledger.openCursor("test-cursor");
- ledger.addEntry("dummy-entry-1".getBytes(Encoding));
-
- final CountDownLatch counter = new CountDownLatch(1);
-
- ledger.asyncClose(new CloseCallback() {
- @Override
- public void closeComplete(Object ctx) {
- assertNull(ctx);
- counter.countDown();
- }
-
- @Override
- public void closeFailed(ManagedLedgerException exception, Object ctx) {
- fail(exception.getMessage());
- }
-
- }, null);
-
- counter.await();
- }
-
- @Test(timeOut = 20000)
- public void asyncOpenCursorWithoutError() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger");
-
- final CountDownLatch counter = new CountDownLatch(1);
-
- ledger.asyncOpenCursor("test-cursor", new OpenCursorCallback() {
- @Override
- public void openCursorComplete(ManagedCursor cursor, Object ctx) {
- assertNull(ctx);
- assertNotNull(cursor);
-
- counter.countDown();
- }
-
- @Override
- public void openCursorFailed(ManagedLedgerException exception, Object ctx) {
- fail(exception.getMessage());
- }
-
- }, null);
-
- counter.await();
- }
-
- @Test(timeOut = 20000)
- public void asyncOpenCursorWithError() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger");
-
- final CountDownLatch counter = new CountDownLatch(1);
-
- stopBookKeeper();
- stopZooKeeper();
-
- ledger.asyncOpenCursor("test-cursor", new OpenCursorCallback() {
- @Override
- public void openCursorComplete(ManagedCursor cursor, Object ctx) {
- fail("The async-call should have failed");
- }
-
- @Override
- public void openCursorFailed(ManagedLedgerException exception, Object ctx) {
- counter.countDown();
- }
- }, null);
-
- counter.await();
- }
-
- @Test(timeOut = 20000)
- public void readFromOlderLedger() throws Exception {
- ManagedLedgerConfig config = new ManagedLedgerConfig().setMaxEntriesPerLedger(1);
- ManagedLedger ledger = factory.open("my_test_ledger", config);
- ManagedCursor cursor = ledger.openCursor("test");
-
- ledger.addEntry("entry-1".getBytes(Encoding));
- ledger.addEntry("entry-2".getBytes(Encoding));
-
- assertEquals(cursor.hasMoreEntries(), true);
- }
-
- @Test(timeOut = 20000)
- public void readFromOlderLedgers() throws Exception {
- ManagedLedgerConfig config = new ManagedLedgerConfig().setMaxEntriesPerLedger(1);
- ManagedLedger ledger = factory.open("my_test_ledger", config);
- ManagedCursor cursor = ledger.openCursor("test");
-
- ledger.addEntry("entry-1".getBytes(Encoding));
- ledger.addEntry("entry-2".getBytes(Encoding));
- ledger.addEntry("entry-3".getBytes(Encoding));
-
- assertEquals(cursor.hasMoreEntries(), true);
- cursor.readEntries(1).forEach(e -> e.release());
-
- assertEquals(cursor.hasMoreEntries(), true);
- cursor.readEntries(1).forEach(e -> e.release());
- assertEquals(cursor.hasMoreEntries(), true);
- cursor.readEntries(1).forEach(e -> e.release());
- assertEquals(cursor.hasMoreEntries(), false);
- }
-
- @Test(timeOut = 20000)
- public void triggerLedgerDeletion() throws Exception {
- ManagedLedgerConfig config = new ManagedLedgerConfig().setMaxEntriesPerLedger(1);
- ManagedLedger ledger = factory.open("my_test_ledger", config);
- ManagedCursor cursor = ledger.openCursor("test");
-
- ledger.addEntry("entry-1".getBytes(Encoding));
- ledger.addEntry("entry-2".getBytes(Encoding));
- ledger.addEntry("entry-3".getBytes(Encoding));
-
- assertEquals(cursor.hasMoreEntries(), true);
- List<Entry> entries = cursor.readEntries(1);
- assertEquals(entries.size(), 1);
- assertEquals(ledger.getNumberOfEntries(), 3);
- entries.forEach(e -> e.release());
-
- assertEquals(cursor.hasMoreEntries(), true);
- entries = cursor.readEntries(1);
- assertEquals(cursor.hasMoreEntries(), true);
-
- cursor.markDelete(entries.get(0).getPosition());
- entries.forEach(e -> e.release());
- }
-
- @Test(timeOut = 20000)
- public void testEmptyManagedLedgerContent() throws Exception {
- ZooKeeper zk = bkc.getZkHandle();
- zk.create("/managed-ledger", new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
- zk.create("/managed-ledger/my_test_ledger", " ".getBytes(), ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
-
- ManagedLedger ledger = factory.open("my_test_ledger");
- ledger.openCursor("test");
-
- ledger.addEntry("entry-1".getBytes(Encoding));
- assertEquals(ledger.getNumberOfEntries(), 1);
- }
-
- @Test(timeOut = 20000)
- public void testProducerAndNoConsumer() throws Exception {
- ManagedLedgerConfig config = new ManagedLedgerConfig().setMaxEntriesPerLedger(1);
- ManagedLedger ledger = factory.open("my_test_ledger", config);
-
- assertEquals(ledger.getNumberOfEntries(), 0);
-
- ledger.addEntry("entry-1".getBytes(Encoding));
- assertEquals(ledger.getNumberOfEntries(), 1);
-
- // Since there are no consumers, older ledger will be deleted
- // in a short time (in a background thread)
- ledger.addEntry("entry-2".getBytes(Encoding));
- while (ledger.getNumberOfEntries() > 1) {
- log.debug("entries={}", ledger.getNumberOfEntries());
- Thread.sleep(100);
- }
-
- ledger.addEntry("entry-3".getBytes(Encoding));
- while (ledger.getNumberOfEntries() > 1) {
- log.debug("entries={}", ledger.getNumberOfEntries());
- Thread.sleep(100);
- }
- }
-
- @Test(timeOut = 20000)
- public void testTrimmer() throws Exception {
- ManagedLedgerConfig config = new ManagedLedgerConfig().setMaxEntriesPerLedger(1);
- ManagedLedger ledger = factory.open("my_test_ledger", config);
- ManagedCursor cursor = ledger.openCursor("c1");
-
- assertEquals(ledger.getNumberOfEntries(), 0);
-
- ledger.addEntry("entry-1".getBytes(Encoding));
- ledger.addEntry("entry-2".getBytes(Encoding));
- ledger.addEntry("entry-3".getBytes(Encoding));
- ledger.addEntry("entry-4".getBytes(Encoding));
- assertEquals(ledger.getNumberOfEntries(), 4);
-
- cursor.readEntries(1).forEach(e -> e.release());
- cursor.readEntries(1).forEach(e -> e.release());
- List<Entry> entries = cursor.readEntries(1);
- Position lastPosition = entries.get(0).getPosition();
- entries.forEach(e -> e.release());
-
- assertEquals(ledger.getNumberOfEntries(), 4);
-
- cursor.markDelete(lastPosition);
-
- while (ledger.getNumberOfEntries() != 2) {
- Thread.sleep(10);
- }
- }
-
- @Test(timeOut = 20000)
- public void testAsyncAddEntryAndSyncClose() throws Exception {
- ManagedLedgerConfig config = new ManagedLedgerConfig().setMaxEntriesPerLedger(10);
- ManagedLedger ledger = factory.open("my_test_ledger", config);
- ledger.openCursor("c1");
-
- assertEquals(ledger.getNumberOfEntries(), 0);
-
- final CountDownLatch counter = new CountDownLatch(100);
-
- for (int i = 0; i < 100; i++) {
- String content = "entry-" + i;
- ledger.asyncAddEntry(content.getBytes(Encoding), new AddEntryCallback() {
- @Override
- public void addComplete(Position position, Object ctx) {
- counter.countDown();
- }
-
- @Override
- public void addFailed(ManagedLedgerException exception, Object ctx) {
- fail(exception.getMessage());
- }
-
- }, null);
- }
-
- counter.await();
-
- assertEquals(ledger.getNumberOfEntries(), 100);
- }
-
- @Test(timeOut = 20000)
- public void moveCursorToNextLedger() throws Exception {
- ManagedLedgerConfig config = new ManagedLedgerConfig().setMaxEntriesPerLedger(1);
- ManagedLedger ledger = factory.open("my_test_ledger", config);
- ManagedCursor cursor = ledger.openCursor("test");
-
- ledger.addEntry("entry-1".getBytes(Encoding));
- log.debug("Added 1st message");
- List<Entry> entries = cursor.readEntries(1);
- log.debug("read message ok");
- assertEquals(entries.size(), 1);
- entries.forEach(e -> e.release());
-
- ledger.addEntry("entry-2".getBytes(Encoding));
- log.debug("Added 2nd message");
- ledger.addEntry("entry-3".getBytes(Encoding));
- log.debug("Added 3nd message");
-
- assertEquals(cursor.hasMoreEntries(), true);
- assertEquals(cursor.getNumberOfEntries(), 2);
-
- entries = cursor.readEntries(2);
- assertEquals(entries.size(), 2);
- entries.forEach(e -> e.release());
-
- entries = cursor.readEntries(2);
- assertEquals(entries.size(), 0);
-
- entries = cursor.readEntries(2);
- assertEquals(entries.size(), 0);
- }
-
- @Test(timeOut = 20000)
- public void differentSessions() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger");
-
- assertEquals(ledger.getNumberOfEntries(), 0);
- assertEquals(ledger.getTotalSize(), 0);
-
- ManagedCursor cursor = ledger.openCursor("c1");
-
- ledger.addEntry("dummy-entry-1".getBytes(Encoding));
-
- assertEquals(ledger.getNumberOfEntries(), 1);
- assertEquals(ledger.getTotalSize(), "dummy-entry-1".getBytes(Encoding).length);
-
- assertEquals(cursor.hasMoreEntries(), true);
- assertEquals(cursor.getNumberOfEntries(), 1);
-
- ledger.close();
-
- // Create a new factory and re-open the same managed ledger
- factory = new ManagedLedgerFactoryImpl(bkc, zkc);
-
- ledger = factory.open("my_test_ledger");
-
- assertEquals(ledger.getNumberOfEntries(), 1);
- assertEquals(ledger.getTotalSize(), "dummy-entry-1".getBytes(Encoding).length);
-
- cursor = ledger.openCursor("c1");
-
- assertEquals(cursor.hasMoreEntries(), true);
- assertEquals(cursor.getNumberOfEntries(), 1);
-
- ledger.addEntry("dummy-entry-2".getBytes(Encoding));
-
- assertEquals(ledger.getNumberOfEntries(), 2);
- assertEquals(ledger.getTotalSize(), "dummy-entry-1".getBytes(Encoding).length * 2);
-
- assertEquals(cursor.hasMoreEntries(), true);
- assertEquals(cursor.getNumberOfEntries(), 2);
-
- ledger.close();
- }
-
- @Test(enabled = false)
- public void fenceManagedLedger() throws Exception {
- ManagedLedgerFactory factory1 = new ManagedLedgerFactoryImpl(bkc, bkc.getZkHandle());
- ManagedLedger ledger1 = factory1.open("my_test_ledger");
- ManagedCursor cursor1 = ledger1.openCursor("c1");
- ledger1.addEntry("entry-1".getBytes(Encoding));
-
- ManagedLedgerFactory factory2 = new ManagedLedgerFactoryImpl(bkc, bkc.getZkHandle());
- ManagedLedger ledger2 = factory2.open("my_test_ledger");
- ManagedCursor cursor2 = ledger2.openCursor("c1");
-
- // At this point ledger1 must have been fenced
- try {
- ledger1.addEntry("entry-1".getBytes(Encoding));
- fail("Expecting exception");
- } catch (ManagedLedgerFencedException e) {
- }
-
- try {
- ledger1.addEntry("entry-2".getBytes(Encoding));
- fail("Expecting exception");
- } catch (ManagedLedgerFencedException e) {
- }
-
- try {
- cursor1.readEntries(10);
- fail("Expecting exception");
- } catch (ManagedLedgerFencedException e) {
- }
-
- try {
- ledger1.openCursor("new cursor");
- fail("Expecting exception");
- } catch (ManagedLedgerFencedException e) {
- }
-
- ledger2.addEntry("entry-2".getBytes(Encoding));
-
- assertEquals(cursor2.getNumberOfEntries(), 2);
- factory1.shutdown();
- factory2.shutdown();
- }
-
- @Test
- public void forceCloseLedgers() throws Exception {
- ManagedLedger ledger1 = factory.open("my_test_ledger", new ManagedLedgerConfig().setMaxEntriesPerLedger(1));
- ledger1.openCursor("c1");
- ManagedCursor c2 = ledger1.openCursor("c2");
- ledger1.addEntry("entry-1".getBytes(Encoding));
- ledger1.addEntry("entry-2".getBytes(Encoding));
- ledger1.addEntry("entry-3".getBytes(Encoding));
-
- c2.readEntries(1).forEach(e -> e.release());
- c2.readEntries(1).forEach(e -> e.release());
- c2.readEntries(1).forEach(e -> e.release());
-
- ledger1.close();
-
- try {
- ledger1.addEntry("entry-3".getBytes(Encoding));
- fail("should not have reached this point");
- } catch (ManagedLedgerException e) {
- // ok
- }
-
- try {
- ledger1.openCursor("new-cursor");
- fail("should not have reached this point");
- } catch (ManagedLedgerException e) {
- // ok
- }
- }
-
- @Test
- public void closeLedgerWithError() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger");
- ledger.addEntry("entry-1".getBytes(Encoding));
-
- stopZooKeeper();
- stopBookKeeper();
-
- try {
- ledger.close();
- // fail("should have thrown exception");
- } catch (ManagedLedgerException e) {
- // Ok
- }
- }
-
- @Test(timeOut = 20000)
- public void deleteWithErrors1() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger");
-
- PositionImpl position = (PositionImpl) ledger.addEntry("dummy-entry-1".getBytes(Encoding));
- assertEquals(ledger.getNumberOfEntries(), 1);
-
- // Force delete a ledger and test that deleting the ML still happens
- // without errors
- bkc.deleteLedger(position.getLedgerId());
- ledger.delete();
- }
-
- @Test(timeOut = 20000)
- public void deleteWithErrors2() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger");
- ledger.addEntry("dummy-entry-1".getBytes(Encoding));
-
- stopZooKeeper();
-
- try {
- ledger.delete();
- fail("should have failed");
- } catch (ManagedLedgerException e) {
- // ok
- } catch (RejectedExecutionException e) {
- // ok
- }
- }
-
- @Test(timeOut = 20000)
- public void readWithErrors1() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger", new ManagedLedgerConfig().setMaxEntriesPerLedger(1));
- ManagedCursor cursor = ledger.openCursor("c1");
- ledger.addEntry("dummy-entry-1".getBytes(Encoding));
- ledger.addEntry("dummy-entry-2".getBytes(Encoding));
-
- stopZooKeeper();
- stopBookKeeper();
-
- try {
- cursor.readEntries(10);
- fail("should have failed");
- } catch (ManagedLedgerException e) {
- // ok
- }
-
- try {
- ledger.addEntry("dummy-entry-3".getBytes(Encoding));
- fail("should have failed");
- } catch (ManagedLedgerException e) {
- // ok
- }
- }
-
- @Test(timeOut = 20000, enabled = false)
- void concurrentAsyncOpen() throws Exception {
- final CountDownLatch counter = new CountDownLatch(2);
-
- class Result {
- ManagedLedger instance1 = null;
- ManagedLedger instance2 = null;
- }
-
- final Result result = new Result();
- factory.asyncOpen("my-test-ledger", new OpenLedgerCallback() {
-
- @Override
- public void openLedgerComplete(ManagedLedger ledger, Object ctx) {
- result.instance1 = ledger;
- counter.countDown();
- }
-
- @Override
- public void openLedgerFailed(ManagedLedgerException exception, Object ctx) {
- }
- }, null);
-
- factory.asyncOpen("my-test-ledger", new OpenLedgerCallback() {
-
- @Override
- public void openLedgerComplete(ManagedLedger ledger, Object ctx) {
- result.instance2 = ledger;
- counter.countDown();
- }
-
- @Override
- public void openLedgerFailed(ManagedLedgerException exception, Object ctx) {
- }
- }, null);
-
- counter.await();
- assertEquals(result.instance1, result.instance2);
- assertNotNull(result.instance1);
- }
-
- @Test // (timeOut = 20000)
- public void asyncOpenClosedLedger() throws Exception {
- ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("my-closed-ledger");
-
- ManagedCursor c1 = ledger.openCursor("c1");
- ledger.addEntry("dummy-entry-1".getBytes(Encoding));
- c1.close();
-
- assertEquals(ledger.getNumberOfEntries(), 1);
-
- ledger.setFenced();
-
- final CountDownLatch counter = new CountDownLatch(1);
- class Result {
- ManagedLedger instance1 = null;
- }
-
- final Result result = new Result();
- factory.asyncOpen("my-closed-ledger", new OpenLedgerCallback() {
-
- @Override
- public void openLedgerComplete(ManagedLedger ledger, Object ctx) {
- result.instance1 = ledger;
- counter.countDown();
- }
-
- @Override
- public void openLedgerFailed(ManagedLedgerException exception, Object ctx) {
- }
- }, null);
- counter.await();
- assertNotNull(result.instance1);
-
- ManagedCursor c2 = result.instance1.openCursor("c1");
- List<Entry> entries = c2.readEntries(1);
- assertEquals(entries.size(), 1);
- entries.forEach(e -> e.release());
-
- }
-
- @Test
- public void getCursors() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger");
- ManagedCursor c1 = ledger.openCursor("c1");
- ManagedCursor c2 = ledger.openCursor("c2");
-
- assertEquals(Sets.newHashSet(ledger.getCursors()), Sets.newHashSet(c1, c2));
-
- c1.close();
- ledger.deleteCursor("c1");
- assertEquals(Sets.newHashSet(ledger.getCursors()), Sets.newHashSet(c2));
-
- c2.close();
- ledger.deleteCursor("c2");
- assertEquals(Sets.newHashSet(ledger.getCursors()), Sets.newHashSet());
- }
-
- @Test
- public void ledgersList() throws Exception {
- MetaStore store = factory.getMetaStore();
-
- assertEquals(Sets.newHashSet(store.getManagedLedgers()), Sets.newHashSet());
- ManagedLedger ledger1 = factory.open("ledger1");
- assertEquals(Sets.newHashSet(store.getManagedLedgers()), Sets.newHashSet("ledger1"));
- ManagedLedger ledger2 = factory.open("ledger2");
- assertEquals(Sets.newHashSet(store.getManagedLedgers()), Sets.newHashSet("ledger1", "ledger2"));
- ledger1.delete();
- assertEquals(Sets.newHashSet(store.getManagedLedgers()), Sets.newHashSet("ledger2"));
- ledger2.delete();
- assertEquals(Sets.newHashSet(store.getManagedLedgers()), Sets.newHashSet());
- }
-
- @Test
- public void testCleanup() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger");
- ledger.openCursor("c1");
-
- ledger.addEntry("data".getBytes(Encoding));
- assertEquals(bkc.getLedgers().size(), 2);
-
- ledger.delete();
- assertEquals(bkc.getLedgers().size(), 0);
- }
-
- @Test(timeOut = 20000)
- public void testAsyncCleanup() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger");
- ledger.openCursor("c1");
-
- ledger.addEntry("data".getBytes(Encoding));
- assertEquals(bkc.getLedgers().size(), 2);
-
- final CountDownLatch latch = new CountDownLatch(1);
-
- ledger.asyncDelete(new DeleteLedgerCallback() {
- @Override
- public void deleteLedgerFailed(ManagedLedgerException exception, Object ctx) {
- fail("should have succeeded");
- }
-
- @Override
- public void deleteLedgerComplete(Object ctx) {
- latch.countDown();
- }
- }, null);
-
- latch.await();
- assertEquals(bkc.getLedgers().size(), 0);
- }
-
- @Test(timeOut = 20000)
- public void testReopenAndCleanup() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger");
- ledger.openCursor("c1");
-
- ledger.addEntry("data".getBytes(Encoding));
- ledger.close();
- Thread.sleep(100);
- assertEquals(bkc.getLedgers().size(), 1);
-
- factory.shutdown();
-
- factory = new ManagedLedgerFactoryImpl(bkc, bkc.getZkHandle());
- ledger = factory.open("my_test_ledger");
- ledger.openCursor("c1");
- Thread.sleep(100);
- assertEquals(bkc.getLedgers().size(), 2);
-
- ledger.close();
- factory.open("my_test_ledger", new ManagedLedgerConfig()).delete();
- Thread.sleep(100);
- assertEquals(bkc.getLedgers().size(), 0);
-
- factory.shutdown();
- }
-
- @Test(timeOut = 20000)
- public void doubleOpen() throws Exception {
- ManagedLedger ledger1 = factory.open("my_test_ledger");
- ManagedLedger ledger2 = factory.open("my_test_ledger");
-
- assertTrue(ledger1 == ledger2);
- }
-
- @Test
- public void compositeNames() throws Exception {
- // Should not throw exception
- factory.open("my/test/ledger");
- }
-
- @Test
- public void previousPosition() throws Exception {
- ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("my_test_ledger",
- new ManagedLedgerConfig().setMaxEntriesPerLedger(2));
- ManagedCursor cursor = ledger.openCursor("my_cursor");
-
- Position p0 = cursor.getMarkDeletedPosition();
- // This is expected because p0 is already an "invalid" position (since no entry has been mark-deleted yet)
- assertEquals(ledger.getPreviousPosition((PositionImpl) p0), p0);
-
- // Force to close an empty ledger
- ledger.close();
-
- ledger = (ManagedLedgerImpl) factory.open("my_test_ledger",
- new ManagedLedgerConfig().setMaxEntriesPerLedger(2));
- // again
- ledger.close();
-
- ledger = (ManagedLedgerImpl) factory.open("my_test_ledger",
- new ManagedLedgerConfig().setMaxEntriesPerLedger(2));
- PositionImpl pBeforeWriting = ledger.getLastPosition();
- PositionImpl p1 = (PositionImpl) ledger.addEntry("entry".getBytes());
- ledger.close();
-
- ledger = (ManagedLedgerImpl) factory.open("my_test_ledger",
- new ManagedLedgerConfig().setMaxEntriesPerLedger(2));
- Position p2 = ledger.addEntry("entry".getBytes());
- Position p3 = ledger.addEntry("entry".getBytes());
- Position p4 = ledger.addEntry("entry".getBytes());
-
- assertEquals(ledger.getPreviousPosition(p1), pBeforeWriting);
- assertEquals(ledger.getPreviousPosition((PositionImpl) p2), p1);
- assertEquals(ledger.getPreviousPosition((PositionImpl) p3), p2);
- assertEquals(ledger.getPreviousPosition((PositionImpl) p4), p3);
- }
-
- /**
- * Reproduce a race condition between opening cursors and concurrent mark delete operations
- */
- @Test(timeOut = 20000)
- public void testOpenRaceCondition() throws Exception {
- ManagedLedgerConfig config = new ManagedLedgerConfig();
- config.setEnsembleSize(2).setAckQuorumSize(2).setMetadataEnsembleSize(2);
- final ManagedLedger ledger = factory.open("my-ledger", config);
- final ManagedCursor c1 = ledger.openCursor("c1");
-
- final int N = 1000;
- final Position position = ledger.addEntry("entry-0".getBytes());
- Executor executor = Executors.newCachedThreadPool();
- final CountDownLatch counter = new CountDownLatch(2);
- executor.execute(new Runnable() {
- @Override
- public void run() {
- try {
- for (int i = 0; i < N; i++) {
- c1.markDelete(position);
- }
- counter.countDown();
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
- });
-
- executor.execute(new Runnable() {
- @Override
- public void run() {
- try {
- for (int i = 0; i < N; i++) {
- ledger.openCursor("cursor-" + i);
- }
- counter.countDown();
- } catch (Exception e) {
- e.printStackTrace();
- }
- }
- });
-
- // If there is the race condition, this method will not complete triggering the test timeout
- counter.await();
- }
-
- @Test
- public void invalidateConsumedEntriesFromCache() throws Exception {
- ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("my_test_ledger");
-
- EntryCacheManager cacheManager = factory.getEntryCacheManager();
- EntryCache entryCache = ledger.entryCache;
-
- ManagedCursorImpl c1 = (ManagedCursorImpl) ledger.openCursor("c1");
- ManagedCursorImpl c2 = (ManagedCursorImpl) ledger.openCursor("c2");
-
- PositionImpl p1 = (PositionImpl) ledger.addEntry("entry-1".getBytes());
- PositionImpl p2 = (PositionImpl) ledger.addEntry("entry-2".getBytes());
- PositionImpl p3 = (PositionImpl) ledger.addEntry("entry-3".getBytes());
- PositionImpl p4 = (PositionImpl) ledger.addEntry("entry-4".getBytes());
-
- assertEquals(entryCache.getSize(), 7 * 4);
- assertEquals(cacheManager.getSize(), entryCache.getSize());
-
- c2.setReadPosition(p3);
- ledger.discardEntriesFromCache(c2, p2);
-
- assertEquals(entryCache.getSize(), 7 * 4);
- assertEquals(cacheManager.getSize(), entryCache.getSize());
-
- c1.setReadPosition(p2);
- ledger.discardEntriesFromCache(c1, p1);
- assertEquals(entryCache.getSize(), 7 * 3);
- assertEquals(cacheManager.getSize(), entryCache.getSize());
-
- c1.setReadPosition(p3);
- ledger.discardEntriesFromCache(c1, p2);
- assertEquals(entryCache.getSize(), 7 * 2);
- assertEquals(cacheManager.getSize(), entryCache.getSize());
-
- ledger.deactivateCursor(c1);
- assertEquals(entryCache.getSize(), 7 * 2); // as c2.readPosition=p3 => Cache contains p3,p4
- assertEquals(cacheManager.getSize(), entryCache.getSize());
-
- c2.setReadPosition(p4);
- ledger.discardEntriesFromCache(c2, p3);
- assertEquals(entryCache.getSize(), 7);
- assertEquals(cacheManager.getSize(), entryCache.getSize());
-
- ledger.deactivateCursor(c2);
- assertEquals(entryCache.getSize(), 0);
- assertEquals(cacheManager.getSize(), entryCache.getSize());
- }
-
- @Test
- public void discardEmptyLedgersOnClose() throws Exception {
- ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("my_test_ledger");
- ManagedCursor c1 = ledger.openCursor("c1");
-
- ledger.addEntry("entry".getBytes());
-
- assertEquals(ledger.getLedgersInfoAsList().size(), 1);
-
- c1.close();
- ledger.close();
-
- // re-open
- ledger = (ManagedLedgerImpl) factory.open("my_test_ledger");
- assertEquals(ledger.getLedgersInfoAsList().size(), 2); // 1 ledger with 1 entry and the current writing ledger
-
- c1.close();
- ledger.close();
-
- // re-open, now the previous empty ledger should have been discarded
- ledger = (ManagedLedgerImpl) factory.open("my_test_ledger");
- assertEquals(ledger.getLedgersInfoAsList().size(), 2); // 1 ledger with 1 entry, and the current
- // writing ledger
- }
-
- @Test
- public void discardEmptyLedgersOnError() throws Exception {
- ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("my_test_ledger");
-
- assertEquals(ledger.getLedgersInfoAsList().size(), 1);
-
- bkc.failNow(BKException.Code.NoBookieAvailableException);
- zkc.failNow(Code.CONNECTIONLOSS);
- try {
- ledger.addEntry("entry".getBytes());
- fail("Should have received exception");
- } catch (ManagedLedgerException e) {
- // Ok
- }
-
- assertEquals(ledger.getLedgersInfoAsList().size(), 0);
-
- // Next write should fail as well
- try {
- ledger.addEntry("entry".getBytes());
- fail("Should have received exception");
- } catch (ManagedLedgerException e) {
- // Ok
- }
-
- assertEquals(ledger.getLedgersInfoAsList().size(), 0);
- assertEquals(ledger.getNumberOfEntries(), 0);
- }
-
- @Test
- public void cursorReadsWithDiscardedEmptyLedgers() throws Exception {
- ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("my_test_ledger");
- ManagedCursor c1 = ledger.openCursor("c1");
-
- Position p1 = c1.getReadPosition();
-
- c1.close();
- ledger.close();
-
- // re-open
- ledger = (ManagedLedgerImpl) factory.open("my_test_ledger");
- c1 = ledger.openCursor("c1");
-
- assertEquals(c1.getNumberOfEntries(), 0);
- assertEquals(c1.hasMoreEntries(), false);
-
- ledger.addEntry("entry".getBytes());
-
- assertEquals(c1.getNumberOfEntries(), 1);
- assertEquals(c1.hasMoreEntries(), true);
-
- assertEquals(ledger.getLedgersInfoAsList().size(), 1);
-
- List<Entry> entries = c1.readEntries(1);
- assertEquals(entries.size(), 1);
- entries.forEach(e -> e.release());
-
- assertEquals(c1.hasMoreEntries(), false);
- assertEquals(c1.readEntries(1).size(), 0);
-
- c1.seek(p1);
- assertEquals(c1.hasMoreEntries(), true);
- assertEquals(c1.getNumberOfEntries(), 1);
-
- entries = c1.readEntries(1);
- assertEquals(entries.size(), 1);
- entries.forEach(e -> e.release());
- assertEquals(c1.readEntries(1).size(), 0);
- }
-
- @Test
- public void cursorReadsWithDiscardedEmptyLedgersStillListed() throws Exception {
- ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("my_test_ledger");
- ManagedCursor c1 = ledger.openCursor("c1");
-
- ledger.addEntry("entry-1".getBytes());
- ledger.close();
-
- ledger = (ManagedLedgerImpl) factory.open("my_test_ledger");
- c1 = ledger.openCursor("c1");
- ledger.addEntry("entry-2".getBytes());
-
- final LedgerInfo l1info = ledger.getLedgersInfoAsList().get(0);
- final LedgerInfo l2info = ledger.getLedgersInfoAsList().get(1);
-
- ledger.close();
-
- // Add the deleted ledger back in the meta-data to simulate an empty ledger that was deleted but not removed
- // from the list of ledgers
- final CountDownLatch counter = new CountDownLatch(1);
- final MetaStore store = factory.getMetaStore();
- store.getManagedLedgerInfo("my_test_ledger", false, new MetaStoreCallback<ManagedLedgerInfo>() {
- @Override
- public void operationComplete(ManagedLedgerInfo result, Stat version) {
- // Update the list
- ManagedLedgerInfo.Builder info = ManagedLedgerInfo.newBuilder(result);
- info.clearLedgerInfo();
- info.addLedgerInfo(LedgerInfo.newBuilder().setLedgerId(l1info.getLedgerId()).build());
- info.addLedgerInfo(l2info);
-
- store.asyncUpdateLedgerIds("my_test_ledger", info.build(), version, new MetaStoreCallback<Void>() {
- @Override
- public void operationComplete(Void result, Stat version) {
- counter.countDown();
- }
-
- @Override
- public void operationFailed(MetaStoreException e) {
- counter.countDown();
- }
- });
- }
-
- @Override
- public void operationFailed(MetaStoreException e) {
- counter.countDown();
- }
- });
-
- // Wait for the change to be effective
- counter.await();
-
- // Delete the ledger and mantain it in the ledgers list
- bkc.deleteLedger(l1info.getLedgerId());
-
- // re-open
- ledger = (ManagedLedgerImpl) factory.open("my_test_ledger");
- c1 = ledger.openCursor("c1");
-
- assertEquals(c1.getNumberOfEntries(), 1);
- assertEquals(c1.hasMoreEntries(), true);
- assertEquals(ledger.getLedgersInfoAsList().size(), 2);
-
- List<Entry> entries = c1.readEntries(10);
- assertEquals(entries.size(), 1);
- entries.forEach(e -> e.release());
-
- assertEquals(c1.hasMoreEntries(), false);
- entries = c1.readEntries(1);
- assertEquals(entries.size(), 0);
- entries.forEach(e -> e.release());
- }
-
- @Test
- public void addEntryWithOffset() throws Exception {
- ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("my_test_ledger");
- ManagedCursor c1 = ledger.openCursor("c1");
-
- ledger.addEntry("012345678".getBytes(), 2, 3);
-
- List<Entry> entries = c1.readEntries(1);
- assertEquals(entries.get(0).getLength(), 3);
- Entry entry = entries.get(0);
- assertEquals(new String(entry.getData()), "234");
- entry.release();
- }
-
- @Test
- public void totalSizeTest() throws Exception {
- ManagedLedgerConfig conf = new ManagedLedgerConfig();
- conf.setMaxEntriesPerLedger(1);
- ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("my_test_ledger", conf);
- ManagedCursor c1 = ledger.openCursor("c1");
-
- ledger.addEntry(new byte[10], 1, 8);
-
- assertEquals(ledger.getTotalSize(), 8);
-
- PositionImpl p2 = (PositionImpl) ledger.addEntry(new byte[12], 2, 5);
-
- assertEquals(ledger.getTotalSize(), 13);
- c1.markDelete(new PositionImpl(p2.getLedgerId(), -1));
-
- // Wait for background trimming
- Thread.sleep(400);
- assertEquals(ledger.getTotalSize(), 5);
- }
-
- @Test
- public void testMinimumRolloverTime() throws Exception {
- ManagedLedgerConfig conf = new ManagedLedgerConfig();
- conf.setMaxEntriesPerLedger(1);
- conf.setMinimumRolloverTime(1, TimeUnit.SECONDS);
- ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("my_test_ledger", conf);
- ledger.openCursor("c1");
-
- ledger.addEntry("data".getBytes());
- ledger.addEntry("data".getBytes());
-
- assertEquals(ledger.getLedgersInfoAsList().size(), 1);
-
- Thread.sleep(1000);
-
- ledger.addEntry("data".getBytes());
- ledger.addEntry("data".getBytes());
-
- assertEquals(ledger.getLedgersInfoAsList().size(), 2);
- }
-
- @Test
- public void testMaximumRolloverTime() throws Exception {
- ManagedLedgerConfig conf = new ManagedLedgerConfig();
- conf.setMaxEntriesPerLedger(5);
- conf.setMinimumRolloverTime(1, TimeUnit.SECONDS);
- conf.setMaximumRolloverTime(1, TimeUnit.SECONDS);
- ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("my_test_maxtime_ledger", conf);
- ledger.openCursor("c1");
-
- ledger.addEntry("data".getBytes());
- ledger.addEntry("data".getBytes());
-
- assertEquals(ledger.getLedgersInfoAsList().size(), 1);
-
- Thread.sleep(2000);
-
- ledger.addEntry("data".getBytes());
- ledger.addEntry("data".getBytes());
- assertEquals(ledger.getLedgersInfoAsList().size(), 2);
- }
-
- @Test
- public void testRetention() throws Exception {
- ManagedLedgerFactory factory = new ManagedLedgerFactoryImpl(bkc, bkc.getZkHandle());
- ManagedLedgerConfig config = new ManagedLedgerConfig();
- config.setRetentionSizeInMB(10);
- config.setMaxEntriesPerLedger(1);
- config.setRetentionTime(1, TimeUnit.HOURS);
-
- ManagedLedgerImpl ml = (ManagedLedgerImpl) factory.open("retention_test_ledger", config);
- ManagedCursor c1 = ml.openCursor("c1");
- ml.addEntry("iamaverylongmessagethatshouldberetained".getBytes());
- c1.skipEntries(1, IndividualDeletedEntries.Exclude);
- ml.close();
-
- // reopen ml
- ml = (ManagedLedgerImpl) factory.open("retention_test_ledger", config);
- c1 = ml.openCursor("c1");
- ml.addEntry("shortmessage".getBytes());
- c1.skipEntries(1, IndividualDeletedEntries.Exclude);
- ml.close();
- assertTrue(ml.getLedgersInfoAsList().size() > 1);
- assertTrue(ml.getTotalSize() > "shortmessage".getBytes().length);
- }
-
- @Test(enabled = true)
- public void testNoRetention() throws Exception {
- ManagedLedgerFactory factory = new ManagedLedgerFactoryImpl(bkc, bkc.getZkHandle());
- ManagedLedgerConfig config = new ManagedLedgerConfig();
- config.setRetentionSizeInMB(0);
- config.setMaxEntriesPerLedger(1);
- // Default is no-retention
-
- ManagedLedgerImpl ml = (ManagedLedgerImpl) factory.open("noretention_test_ledger", config);
- ManagedCursor c1 = ml.openCursor("c1noretention");
- ml.addEntry("iamaverylongmessagethatshouldnotberetained".getBytes());
- c1.skipEntries(1, IndividualDeletedEntries.Exclude);
- ml.close();
-
- // reopen ml
- ml = (ManagedLedgerImpl) factory.open("noretention_test_ledger", config);
- c1 = ml.openCursor("c1noretention");
- ml.addEntry("shortmessage".getBytes());
- c1.skipEntries(1, IndividualDeletedEntries.Exclude);
- // sleep for trim
- Thread.sleep(1000);
- ml.close();
-
- assertTrue(ml.getLedgersInfoAsList().size() <= 1);
- assertTrue(ml.getTotalSize() <= "shortmessage".getBytes().length);
- }
-
- @Test
- public void testDeletionAfterRetention() throws Exception {
- ManagedLedgerFactory factory = new ManagedLedgerFactoryImpl(bkc, bkc.getZkHandle());
- ManagedLedgerConfig config = new ManagedLedgerConfig();
- config.setRetentionSizeInMB(0);
- config.setMaxEntriesPerLedger(1);
- config.setRetentionTime(1, TimeUnit.SECONDS);
-
- ManagedLedgerImpl ml = (ManagedLedgerImpl) factory.open("deletion_after_retention_test_ledger", config);
- ManagedCursor c1 = ml.openCursor("c1noretention");
- ml.addEntry("iamaverylongmessagethatshouldnotberetained".getBytes());
- c1.skipEntries(1, IndividualDeletedEntries.Exclude);
- ml.close();
-
- // reopen ml
- ml = (ManagedLedgerImpl) factory.open("deletion_after_retention_test_ledger", config);
- c1 = ml.openCursor("c1noretention");
- ml.addEntry("shortmessage".getBytes());
- c1.skipEntries(1, IndividualDeletedEntries.Exclude);
- // let retention expire
- Thread.sleep(1000);
- ml.internalTrimConsumedLedgers(CompletableFuture.completedFuture(null));
-
- assertTrue(ml.getLedgersInfoAsList().size() <= 1);
- assertTrue(ml.getTotalSize() <= "shortmessage".getBytes().length);
- ml.close();
- }
-
- @Test
- public void testInfiniteRetention() throws Exception {
- ManagedLedgerFactory factory = new ManagedLedgerFactoryImpl(bkc, bkc.getZkHandle());
- ManagedLedgerConfig config = new ManagedLedgerConfig();
- config.setRetentionSizeInMB(-1);
- config.setRetentionTime(-1, TimeUnit.HOURS);
- config.setMaxEntriesPerLedger(1);
-
- ManagedLedgerImpl ml = (ManagedLedgerImpl) factory.open("retention_test_ledger", config);
- ManagedCursor c1 = ml.openCursor("c1");
- ml.addEntry("iamaverylongmessagethatshouldberetained".getBytes());
- c1.skipEntries(1, IndividualDeletedEntries.Exclude);
- ml.close();
-
- // reopen ml
- ml = (ManagedLedgerImpl) factory.open("retention_test_ledger", config);
- c1 = ml.openCursor("c1");
- ml.addEntry("shortmessage".getBytes());
- c1.skipEntries(1, IndividualDeletedEntries.Exclude);
- ml.close();
- assertTrue(ml.getLedgersInfoAsList().size() > 1);
- assertTrue(ml.getTotalSize() > "shortmessage".getBytes().length);
- }
-
- @Test
- public void testTimestampOnWorkingLedger() throws Exception {
- ManagedLedgerFactory factory = new ManagedLedgerFactoryImpl(bkc, bkc.getZkHandle());
- ManagedLedgerConfig conf = new ManagedLedgerConfig();
- conf.setMaxEntriesPerLedger(1);
- conf.setRetentionSizeInMB(10);
- conf.setRetentionTime(1, TimeUnit.HOURS);
-
- ManagedLedgerImpl ml = (ManagedLedgerImpl) factory.open("my_test_ledger", conf);
- ml.openCursor("c1");
- ml.addEntry("msg1".getBytes());
- Iterator<LedgerInfo> iter = ml.getLedgersInfoAsList().iterator();
- long ts = -1;
- while (iter.hasNext()) {
- LedgerInfo i = iter.next();
- if (iter.hasNext()) {
- assertTrue(ts <= i.getTimestamp(), i.toString());
- ts = i.getTimestamp();
- } else {
- // the last timestamp can be
- // 0 if it is still opened
- // >0 if it is closed after the addEntry see OpAddEntry#addComplete()
- assertTrue(i.getTimestamp() == 0 || ts <= i.getTimestamp(), i.toString());
- }
- }
-
- ml.addEntry("msg02".getBytes());
-
- ml.close();
- // Thread.sleep(1000);
- iter = ml.getLedgersInfoAsList().iterator();
- ts = -1;
- while (iter.hasNext()) {
- LedgerInfo i = iter.next();
- if (iter.hasNext()) {
- assertTrue(ts <= i.getTimestamp(), i.toString());
- ts = i.getTimestamp();
- } else {
- assertTrue(i.getTimestamp() > 0, "well closed LedgerInfo should set a timestamp > 0");
- }
- }
- }
-
- @Test
- public void testBackwardCompatiblityForMeta() throws Exception {
- final ManagedLedgerInfo[] storedMLInfo = new ManagedLedgerInfo[3];
- final Stat[] versions = new Stat[1];
-
- ManagedLedgerFactory factory = new ManagedLedgerFactoryImpl(bkc, bkc.getZkHandle());
- ManagedLedgerConfig conf = new ManagedLedgerConfig();
- conf.setMaxEntriesPerLedger(1);
- conf.setRetentionSizeInMB(10);
- conf.setRetentionTime(1, TimeUnit.HOURS);
-
- ManagedLedger ml = factory.open("backward_test_ledger", conf);
- ml.openCursor("c1");
- ml.addEntry("msg1".getBytes());
- ml.addEntry("msg2".getBytes());
- ml.close();
-
- MetaStore store = new MetaStoreImplZookeeper(zkc, executor);
- CountDownLatch l1 = new CountDownLatch(1);
-
- // obtain the ledger info
- store.getManagedLedgerInfo("backward_test_ledger", false, new MetaStoreCallback<ManagedLedgerInfo>() {
- @Override
- public void operationComplete(ManagedLedgerInfo result, Stat version) {
- storedMLInfo[0] = result;
- versions[0] = version;
- l1.countDown();
- }
-
- @Override
- public void operationFailed(MetaStoreException e) {
- fail("on get ManagedLedgerInfo backward_test_ledger");
- }
- });
-
- l1.await();
- ManagedLedgerInfo.Builder builder1 = ManagedLedgerInfo.newBuilder();
-
- // simulate test for old ledger with no timestampl
- for (LedgerInfo info : storedMLInfo[0].getLedgerInfoList()) {
- LedgerInfo noTimestamp = ManagedLedgerInfo.LedgerInfo.newBuilder().mergeFrom(info).clearTimestamp().build();
- assertFalse(noTimestamp.hasTimestamp(), "expected old version info with no timestamp");
- builder1.addLedgerInfo(noTimestamp);
-
- }
- storedMLInfo[1] = builder1.build();
-
- // test timestamp on new ledger
-
- CountDownLatch l2 = new CountDownLatch(1);
- store.asyncUpdateLedgerIds("backward_test_ledger", storedMLInfo[1], versions[0], new MetaStoreCallback<Void>() {
- @Override
- public void operationComplete(Void result, Stat version) {
- l2.countDown();
- }
-
- @Override
- public void operationFailed(MetaStoreException e) {
- fail("on asyncUpdateLedgerIds");
- }
- });
-
- // verify that after update ledgers have timestamp
-
- ManagedLedgerImpl newVersionLedger = (ManagedLedgerImpl) factory.open("backward_test_ledger", conf);
- List<LedgerInfo> mlInfo = newVersionLedger.getLedgersInfoAsList();
-
- assertTrue(mlInfo.stream().allMatch(new Predicate<LedgerInfo>() {
- @Override
- public boolean test(LedgerInfo ledgerInfo) {
- return ledgerInfo.hasTimestamp();
- }
- }));
- }
-
- @Test
- public void testEstimatedBacklogSize() throws Exception {
- ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("testEstimatedBacklogSize");
- ManagedCursor c1 = ledger.openCursor("c1");
-
- ledger.addEntry(new byte[1024]);
- Position position2 = ledger.addEntry(new byte[1024]);
- ledger.addEntry(new byte[1024]);
- ledger.addEntry(new byte[1024]);
- Position lastPosition = ledger.addEntry(new byte[1024]);
-
- long backlog = ledger.getEstimatedBacklogSize();
- assertEquals(backlog, 1024 * 5);
-
- List<Entry> entries = c1.readEntries(2);
- entries.forEach(Entry::release);
- c1.markDelete(position2);
-
- backlog = ledger.getEstimatedBacklogSize();
- assertEquals(backlog, 1024 * 3);
-
- entries = c1.readEntries(3);
- entries.forEach(Entry::release);
- c1.markDelete(lastPosition);
-
- backlog = ledger.getEstimatedBacklogSize();
- assertEquals(backlog, 0);
- }
-
- @Test
- public void testGetNextValidPosition() throws Exception {
- ManagedLedgerConfig conf = new ManagedLedgerConfig();
- conf.setMaxEntriesPerLedger(1);
- ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("testGetNextValidPosition", conf);
- ManagedCursor c1 = ledger.openCursor("c1");
-
- PositionImpl p1 = (PositionImpl) ledger.addEntry("entry1".getBytes());
- PositionImpl p2 = (PositionImpl) ledger.addEntry("entry2".getBytes());
- PositionImpl p3 = (PositionImpl) ledger.addEntry("entry3".getBytes());
-
- assertEquals(ledger.getNextValidPosition((PositionImpl) c1.getMarkDeletedPosition()), p1);
- assertEquals(ledger.getNextValidPosition(p1), p2);
- assertEquals(ledger.getNextValidPosition(p3), PositionImpl.get(p3.getLedgerId(), p3.getEntryId() + 1));
- }
-
- /**
- * Validations:
- *
- * 1. openCursor : activates cursor 2. EntryCache keeps entries: till entry will be read by all active cursors a.
- * active cursor1 reads entry b. EntryCache keeps entry till cursor2 reads c. active cursor2 reads entry d.
- * EntryCache deletes all read entries by cursor1 and cursor2 3. EntryCache discard entries: deactivate slower
- * cursor a. active cursor1 read all entries b. EntryCache keeps entry till cursor2 reads c. deactivate cursor2 d.
- * EntryCache deletes all read entries by cursor1
- *
- * @throws Exception
- */
- @Test
- public void testActiveDeactiveCursorWithDiscardEntriesFromCache() throws Exception {
- ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("cache_eviction_ledger");
-
- // Open Cursor also adds cursor into activeCursor-container
- ManagedCursor cursor1 = ledger.openCursor("c1");
- ManagedCursor cursor2 = ledger.openCursor("c2");
- Set<ManagedCursor> activeCursors = Sets.newHashSet();
- activeCursors.add(cursor1);
- activeCursors.add(cursor2);
- Field cacheField = ManagedLedgerImpl.class.getDeclaredField("entryCache");
- cacheField.setAccessible(true);
- EntryCacheImpl entryCache = (EntryCacheImpl) cacheField.get(ledger);
-
- Iterator<ManagedCursor> activeCursor = ledger.getActiveCursors().iterator();
-
- // (1) validate cursors are part of activeCursorContainer
- activeCursors.remove(activeCursor.next());
- activeCursors.remove(activeCursor.next());
- assertTrue(activeCursors.isEmpty());
- assertFalse(activeCursor.hasNext());
-
- final int totalInsertedEntries = 50;
- for (int i = 0; i < totalInsertedEntries; i++) {
- String content = "entry"; // 5 bytes
- ledger.addEntry(content.getBytes());
- }
-
- // (2) Validate: as ledger has active cursors: all entries have been cached
- assertEquals((5 * totalInsertedEntries), entryCache.getSize());
-
- // read 20 entries
- final int readEntries = 20;
- List<Entry> entries1 = cursor1.readEntries(readEntries);
- // Acknowledge only on last entry
- cursor1.markDelete(entries1.get(entries1.size() - 1).getPosition());
- for (Entry entry : entries1) {
- log.info("Read entry. Position={} Content='{}'", entry.getPosition(), new String(entry.getData()));
- entry.release();
- }
-
- // read after a second: as RateLimiter limits triggering of removing cache
- Thread.sleep(1000);
-
- List<Entry> entries2 = cursor2.readEntries(readEntries);
- // Acknowledge only on last entry
- cursor2.markDelete((entries2.get(entries2.size() - 1)).getPosition());
- for (Entry entry : entries2) {
- log.info("Read entry. Position={} Content='{}'", entry.getPosition(), new String(entry.getData()));
- entry.release();
- }
-
- // (3) Validate: cache should remove all entries read by both active cursors
- log.info("expected, found : {}, {}", (5 * (totalInsertedEntries - readEntries)), entryCache.getSize());
- assertEquals((5 * (totalInsertedEntries - readEntries)), entryCache.getSize());
-
- final int remainingEntries = totalInsertedEntries - readEntries;
- entries1 = cursor1.readEntries(remainingEntries);
- // Acknowledge only on last entry
- cursor1.markDelete(entries1.get(entries1.size() - 1).getPosition());
- for (Entry entry : entries1) {
- log.info("Read entry. Position={} Content='{}'", entry.getPosition(), new String(entry.getData()));
- entry.release();
- }
-
- // (4) Validate: cursor2 is active cursor and has not read these entries yet: so, cache should not remove these
- // entries
- assertEquals((5 * (remainingEntries)), entryCache.getSize());
-
- ledger.deactivateCursor(cursor2);
-
- // (5) Validate: cursor2 is not active cursor now: cache should have removed all entries read by active cursor1
- assertEquals(0, entryCache.getSize());
-
- log.info("Finished reading entries");
-
- ledger.close();
- }
-
- @Test
- public void testActiveDeactiveCursor() throws Exception {
- ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("cache_eviction_ledger");
-
- Field cacheField = ManagedLedgerImpl.class.getDeclaredField("entryCache");
- cacheField.setAccessible(true);
- EntryCacheImpl entryCache = (EntryCacheImpl) cacheField.get(ledger);
-
- final int totalInsertedEntries = 20;
- for (int i = 0; i < totalInsertedEntries; i++) {
- String content = "entry"; // 5 bytes
- ledger.addEntry(content.getBytes());
- }
-
- // (1) Validate: cache not stores entries as no active cursor
- assertEquals(0, entryCache.getSize());
-
- // Open Cursor also adds cursor into activeCursor-container
- ManagedCursor cursor1 = ledger.openCursor("c1");
- ManagedCursor cursor2 = ledger.openCursor("c2");
- ledger.deactivateCursor(cursor2);
-
- for (int i = 0; i < totalInsertedEntries; i++) {
- String content = "entry"; // 5 bytes
- ledger.addEntry(content.getBytes());
- }
-
- // (2) Validate: cache stores entries as active cursor has not read message
- assertEquals((5 * totalInsertedEntries), entryCache.getSize());
-
- // read 20 entries
- List<Entry> entries1 = cursor1.readEntries(totalInsertedEntries);
- for (Entry entry : entries1) {
- log.info("Read entry. Position={} Content='{}'", entry.getPosition(), new String(entry.getData()));
- entry.release();
- }
-
- // (3) Validate: cache discards all entries as read by active cursor
- assertEquals(0, entryCache.getSize());
-
- ledger.close();
- }
-
- @Test
- public void testCursorRecoveryForEmptyLedgers() throws Exception {
- ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("testCursorRecoveryForEmptyLedgers");
- ManagedCursor c1 = ledger.openCursor("c1");
-
- assertEquals(ledger.getLedgersInfoAsList().size(), 1);
- assertEquals(c1.getMarkDeletedPosition(), ledger.lastConfirmedEntry);
-
- c1.close();
- ledger.close();
-
- ledger = (ManagedLedgerImpl) factory.open("testCursorRecoveryForEmptyLedgers");
- c1 = ledger.openCursor("c1");
-
- assertEquals(ledger.getLedgersInfoAsList().size(), 1);
- assertEquals(c1.getMarkDeletedPosition(), ledger.lastConfirmedEntry);
- }
-
- @Test
- public void testBacklogCursor() throws Exception {
- ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("cache_backlog_ledger");
-
- final long maxMessageCacheRetentionTimeMillis = 100;
- Field field = ManagedLedgerImpl.class.getDeclaredField("maxMessageCacheRetentionTimeMillis");
- field.setAccessible(true);
- Field modifiersField = Field.class.getDeclaredField("modifiers");
- modifiersField.setAccessible(true);
- modifiersField.setInt(field, field.getModifiers() & ~Modifier.FINAL);
- field.set(ledger, maxMessageCacheRetentionTimeMillis);
- Field backlogThresholdField = ManagedLedgerImpl.class.getDeclaredField("maxActiveCursorBacklogEntries");
- backlogThresholdField.setAccessible(true);
- final long maxActiveCursorBacklogEntries = (long) backlogThresholdField.get(ledger);
-
- // Open Cursor also adds cursor into activeCursor-container
- ManagedCursor cursor1 = ledger.openCursor("c1");
- ManagedCursor cursor2 = ledger.openCursor("c2");
-
- final int totalBacklogSizeEntries = (int) maxActiveCursorBacklogEntries;
- CountDownLatch latch = new CountDownLatch(totalBacklogSizeEntries);
- for (int i = 0; i < totalBacklogSizeEntries + 1; i++) {
- String content = "entry"; // 5 bytes
- ByteBuf entry = getMessageWithMetadata(content.getBytes());
- ledger.asyncAddEntry(entry, new AddEntryCallback() {
- @Override
- public void addComplete(Position position, Object ctx) {
- latch.countDown();
- entry.release();
- }
-
- @Override
- public void addFailed(ManagedLedgerException exception, Object ctx) {
- latch.countDown();
- entry.release();
- }
-
- }, null);
- }
- latch.await();
-
- // Verify: cursors are active as :haven't started deactivateBacklogCursor scan
- assertTrue(cursor1.isActive());
- assertTrue(cursor2.isActive());
-
- // it allows message to be older enough to be considered in backlog
- Thread.sleep(maxMessageCacheRetentionTimeMillis * 2);
-
- // deactivate backlog cursors
- ledger.checkBackloggedCursors();
- Thread.sleep(100);
-
- // both cursors have to be inactive
- assertFalse(cursor1.isActive());
- assertFalse(cursor2.isActive());
-
- // read entries so, cursor1 reaches maxBacklog threshold again to be active again
- List<Entry> entries1 = cursor1.readEntries(50);
- for (Entry entry : entries1) {
- log.info("Read entry. Position={} Content='{}'", entry.getPosition(), new String(entry.getData()));
- entry.release();
- }
-
- // activate cursors which caught up maxbacklog threshold
- ledger.checkBackloggedCursors();
-
- // verify: cursor1 has consumed messages so, under maxBacklog threshold => active
- assertTrue(cursor1.isActive());
-
- // verify: cursor2 has not consumed messages so, above maxBacklog threshold => inactive
- assertFalse(cursor2.isActive());
-
- ledger.close();
- }
-
- @Test
- public void testConcurrentOpenCursor() throws Exception {
- ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("testConcurrentOpenCursor");
-
- final AtomicReference<ManagedCursor> cursor1 = new AtomicReference<>(null);
- final AtomicReference<ManagedCursor> cursor2 = new AtomicReference<>(null);
- final CyclicBarrier barrier = new CyclicBarrier(2);
- final CountDownLatch latch = new CountDownLatch(2);
-
- cachedExecutor.execute(() -> {
- try {
- barrier.await();
- } catch (Exception e) {
- }
- ledger.asyncOpenCursor("c1", new OpenCursorCallback() {
-
- @Override
- public void openCursorFailed(ManagedLedgerException exception, Object ctx) {
- latch.countDown();
- }
-
- @Override
- public void openCursorComplete(ManagedCursor cursor, Object ctx) {
- cursor1.set(cursor);
- latch.countDown();
- }
- }, null);
- });
-
- cachedExecutor.execute(() -> {
- try {
- barrier.await();
- } catch (Exception e) {
- }
- ledger.asyncOpenCursor("c1", new OpenCursorCallback() {
-
- @Override
- public void openCursorFailed(ManagedLedgerException exception, Object ctx) {
- latch.countDown();
- }
-
- @Override
- public void openCursorComplete(ManagedCursor cursor, Object ctx) {
- cursor2.set(cursor);
- latch.countDown();
- }
- }, null);
- });
-
- latch.await();
- assertNotNull(cursor1.get());
- assertNotNull(cursor2.get());
- assertEquals(cursor1.get(), cursor2.get());
-
- ledger.close();
- }
-
- public ByteBuf getMessageWithMetadata(byte[] data) throws IOException {
- MessageMetadata messageData = MessageMetadata.newBuilder().setPublishTime(System.currentTimeMillis())
- .setProducerName("prod-name").setSequenceId(0).build();
- ByteBuf payload = Unpooled.wrappedBuffer(data, 0, data.length);
-
- int msgMetadataSize = messageData.getSerializedSize();
- int headersSize = 4 + msgMetadataSize;
- ByteBuf headers = PooledByteBufAllocator.DEFAULT.buffer(headersSize, headersSize);
- ByteBufCodedOutputStream outStream = ByteBufCodedOutputStream.get(headers);
- headers.writeInt(msgMetadataSize);
- messageData.writeTo(outStream);
- outStream.recycle();
- return ByteBufPair.coalesce(ByteBufPair.get(headers, payload));
- }
-
- @Test
- public void testConsumerSubscriptionInitializePosition() throws Exception{
- final int MAX_ENTRY_PER_LEDGER = 2;
- ManagedLedgerConfig config = new ManagedLedgerConfig().setMaxEntriesPerLedger(MAX_ENTRY_PER_LEDGER);
- ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("lastest_earliest_ledger", config);
-
- final int totalInsertedEntries = 20;
- for (int i = 0; i < totalInsertedEntries; i++) {
- String content = "entry" + i; // 5 bytes
- ledger.addEntry(content.getBytes());
- }
- // Open Cursor also adds cursor into activeCursor-container
- ManagedCursor latestCursor = ledger.openCursor("c1", InitialPosition.Latest);
- ManagedCursor earliestCursor = ledger.openCursor("c2", InitialPosition.Earliest);
-
- // Since getReadPosition returns the next position, we decrease the entryId by 1
- PositionImpl p1 = (PositionImpl) latestCursor.getReadPosition();
- PositionImpl p2 = (PositionImpl) earliestCursor.getReadPosition();
-
- Pair<PositionImpl, Long> latestPositionAndCounter = ledger.getLastPositionAndCounter();
- Pair<PositionImpl, Long> earliestPositionAndCounter = ledger.getFirstPositionAndCounter();
-
- assertEquals(latestPositionAndCounter.getLeft().getNext(), p1);
- assertEquals(earliestPositionAndCounter.getLeft().getNext(), p2);
-
- assertEquals(latestPositionAndCounter.getRight().longValue(), totalInsertedEntries);
- assertEquals(earliestPositionAndCounter.getRight().longValue(), totalInsertedEntries - earliestCursor.getNumberOfEntriesInBacklog());
-
- ledger.close();
-
- }
-
- @Test
- public void testManagedLedgerAutoCreate() throws Exception {
- ManagedLedgerConfig config = new ManagedLedgerConfig().setCreateIfMissing(true);
- ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("test", config);
- assertNotNull(ledger);
- }
-
- @Test
- public void testManagedLedgerWithoutAutoCreate() throws Exception {
- ManagedLedgerConfig config = new ManagedLedgerConfig().setCreateIfMissing(false);
-
- try {
- factory.open("testManagedLedgerWithoutAutoCreate", config);
- fail("should have thrown ManagedLedgerNotFoundException");
- } catch (ManagedLedgerNotFoundException e) {
- // Expected
- }
-
- assertFalse(factory.getManagedLedgers().containsKey("testManagedLedgerWithoutAutoCreate"));
- }
-
- @Test
- public void testManagedLedgerWithCreateLedgerTimeOut() throws Exception {
- ManagedLedgerConfig config = new ManagedLedgerConfig().setMetadataOperationsTimeoutSeconds(3);
- ManagedLedgerImpl ledger = (ManagedLedgerImpl) factory.open("timeout_ledger_test", config);
-
- BookKeeper bk = mock(BookKeeper.class);
- doNothing().when(bk).asyncCreateLedger(anyInt(), anyInt(), anyInt(), any(), any(), any(), any(), any());
- AtomicInteger response = new AtomicInteger(0);
- CountDownLatch latch = new CountDownLatch(1);
- ledger.asyncCreateLedger(bk, config, null, new CreateCallback() {
- @Override
- public void createComplete(int rc, LedgerHandle lh, Object ctx) {
- response.set(rc);
- latch.countDown();
- }
- }, Collections.emptyMap());
-
- latch.await(config.getMetadataOperationsTimeoutSeconds() + 2, TimeUnit.SECONDS);
- assertEquals(response.get(), BKException.Code.TimeoutException);
-
- ledger.close();
- }
-}
diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/MetaStoreImplZookeeperTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/MetaStoreImplZookeeperTest.java
deleted file mode 100644
index 5f2b89af35..0000000000
--- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/MetaStoreImplZookeeperTest.java
+++ /dev/null
@@ -1,269 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.bookkeeper.mledger.impl;
-
-import static org.testng.Assert.assertEquals;
-import static org.testng.Assert.assertNotNull;
-import static org.testng.Assert.fail;
-
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.atomic.AtomicReference;
-import org.apache.bookkeeper.mledger.ManagedLedgerException.MetaStoreException;
-import org.apache.bookkeeper.mledger.impl.MetaStore.MetaStoreCallback;
-import org.apache.bookkeeper.mledger.impl.MetaStore.Stat;
-import org.apache.bookkeeper.mledger.proto.MLDataFormats;
-import org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedCursorInfo;
-import org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedLedgerInfo;
-import org.apache.bookkeeper.test.MockedBookKeeperTestCase;
-import org.apache.zookeeper.CreateMode;
-import org.apache.zookeeper.KeeperException;
-import org.apache.zookeeper.KeeperException.Code;
-import org.apache.zookeeper.ZooDefs;
-import org.testng.annotations.Test;
-
-public class MetaStoreImplZookeeperTest extends MockedBookKeeperTestCase {
-
- @Test
- void getMLList() throws Exception {
- MetaStore store = new MetaStoreImplZookeeper(zkc, executor);
-
- zkc.failNow(Code.CONNECTIONLOSS);
-
- try {
- store.getManagedLedgers();
- fail("should fail in getting the list");
- } catch (MetaStoreException e) {
- // ok
- }
- }
-
- @Test
- void deleteNonExistingML() throws Exception {
- MetaStore store = new MetaStoreImplZookeeper(zkc, executor);
-
- AtomicReference<MetaStoreException> exception = new AtomicReference<>();
- CountDownLatch counter = new CountDownLatch(1);
-
- store.removeManagedLedger("non-existing", new MetaStoreCallback<Void>() {
- @Override
- public void operationComplete(Void result, Stat version) {
- counter.countDown();
- }
-
- @Override
- public void operationFailed(MetaStoreException e) {
- exception.set(e);
- counter.countDown();
- }
-
- });
-
- counter.await();
- assertNotNull(exception.get());
- }
-
- @Test(timeOut = 20000)
- void readMalformedML() throws Exception {
- MetaStore store = new MetaStoreImplZookeeper(zkc, executor);
-
- zkc.create("/managed-ledgers/my_test", "non-valid".getBytes(), ZooDefs.Ids.OPEN_ACL_UNSAFE,
- CreateMode.PERSISTENT);
-
- final CountDownLatch latch = new CountDownLatch(1);
-
- store.getManagedLedgerInfo("my_test", false, new MetaStoreCallback<MLDataFormats.ManagedLedgerInfo>() {
- public void operationFailed(MetaStoreException e) {
- // Ok
- latch.countDown();
- }
-
- public void operationComplete(ManagedLedgerInfo result, Stat version) {
- fail("Operation should have failed");
- }
- });
-
- latch.await();
- }
-
- @Test(timeOut = 20000)
- void readMalformedCursorNode() throws Exception {
- MetaStore store = new MetaStoreImplZookeeper(zkc, executor);
-
- zkc.create("/managed-ledgers/my_test", "".getBytes(), ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
- zkc.create("/managed-ledgers/my_test/c1", "non-valid".getBytes(), ZooDefs.Ids.OPEN_ACL_UNSAFE,
- CreateMode.PERSISTENT);
-
- final CountDownLatch latch = new CountDownLatch(1);
- store.asyncGetCursorInfo("my_test", "c1", new MetaStoreCallback<MLDataFormats.ManagedCursorInfo>() {
-
- public void operationFailed(MetaStoreException e) {
- // Ok
- latch.countDown();
- }
-
- public void operationComplete(ManagedCursorInfo result, Stat version) {
- fail("Operation should have failed");
- }
- });
-
- latch.await();
- }
-
- @Test(timeOut = 20000)
- void failInCreatingMLnode() throws Exception {
- MetaStore store = new MetaStoreImplZookeeper(zkc, executor);
-
- final CountDownLatch latch = new CountDownLatch(1);
-
- zkc.failAfter(1, Code.CONNECTIONLOSS);
-
- store.getManagedLedgerInfo("my_test", false, new MetaStoreCallback<MLDataFormats.ManagedLedgerInfo>() {
- public void operationFailed(MetaStoreException e) {
- // Ok
- latch.countDown();
- }
-
- public void operationComplete(ManagedLedgerInfo result, Stat version) {
- fail("Operation should have failed");
- }
- });
-
- latch.await();
- }
-
- @Test(timeOut = 20000)
- void updatingCursorNode() throws Exception {
- final MetaStore store = new MetaStoreImplZookeeper(zkc, executor);
-
- zkc.create("/managed-ledgers/my_test", "".getBytes(), ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
-
- final CountDownLatch latch = new CountDownLatch(1);
-
- ManagedCursorInfo info = ManagedCursorInfo.newBuilder().setCursorsLedgerId(1).build();
- store.asyncUpdateCursorInfo("my_test", "c1", info, null, new MetaStoreCallback<Void>() {
- public void operationFailed(MetaStoreException e) {
- fail("should have succeeded");
- }
-
- public void operationComplete(Void result, Stat version) {
- // Update again using the version
- zkc.failNow(Code.CONNECTIONLOSS);
-
- ManagedCursorInfo info = ManagedCursorInfo.newBuilder().setCursorsLedgerId(2).build();
- store.asyncUpdateCursorInfo("my_test", "c1", info, version, new MetaStoreCallback<Void>() {
- public void operationFailed(MetaStoreException e) {
- // ok
- latch.countDown();
- }
-
- @Override
- public void operationComplete(Void result, Stat version) {
- fail("should have failed");
- }
- });
- }
- });
-
- latch.await();
- }
-
- @Test(timeOut = 20000)
- void updatingMLNode() throws Exception {
- final MetaStore store = new MetaStoreImplZookeeper(zkc, executor);
-
- zkc.create("/managed-ledgers/my_test", "".getBytes(), ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
-
- final CountDownLatch latch = new CountDownLatch(1);
-
- store.getManagedLedgerInfo("my_test", false, new MetaStoreCallback<ManagedLedgerInfo>() {
- public void operationFailed(MetaStoreException e) {
- fail("should have succeeded");
- }
-
- public void operationComplete(ManagedLedgerInfo mlInfo, Stat version) {
- // Update again using the version
- zkc.failNow(Code.BADVERSION);
-
- store.asyncUpdateLedgerIds("my_test", mlInfo, version, new MetaStoreCallback<Void>() {
- public void operationFailed(MetaStoreException e) {
- // ok
- latch.countDown();
- }
-
- @Override
- public void operationComplete(Void result, Stat version) {
- fail("should have failed");
- }
- });
- }
- });
-
- latch.await();
- }
-
- @Test(timeOut = 20000)
- public void createOptimisticBaseNotExist() throws Exception {
- CompletableFuture<Void> promise = new CompletableFuture<>();
- MetaStoreImplZookeeper.asyncCreateFullPathOptimistic(
- zkc, "/foo", "bar/zar/gar", new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT,
- (rc, path, ctx, name) -> {
- if (rc != KeeperException.Code.OK.intValue()) {
- promise.completeExceptionally(KeeperException.create(rc));
- } else {
- promise.complete(null);
- }
- });
- try {
- promise.get();
- fail("should have failed");
- } catch (ExecutionException ee) {
- assertEquals(ee.getCause().getClass(), KeeperException.NoNodeException.class);
- }
- }
-
- @Test(timeOut = 20000)
- public void createOptimisticBaseExists() throws Exception {
- zkc.create("/foo", new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
- CompletableFuture<Void> promise = new CompletableFuture<>();
- MetaStoreImplZookeeper.asyncCreateFullPathOptimistic(
- zkc, "/foo", "bar/zar/gar", new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT,
- (rc, path, ctx, name) -> {
- if (rc != KeeperException.Code.OK.intValue()) {
- promise.completeExceptionally(KeeperException.create(rc));
- } else {
- promise.complete(null);
- }
- });
- promise.get();
-
- CompletableFuture<Void> promise2 = new CompletableFuture<>();
- MetaStoreImplZookeeper.asyncCreateFullPathOptimistic(
- zkc, "/foo", "blah", new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT,
- (rc, path, ctx, name) -> {
- if (rc != KeeperException.Code.OK.intValue()) {
- promise2.completeExceptionally(KeeperException.create(rc));
- } else {
- promise2.complete(null);
- }
- });
- promise2.get();
- }
-}
diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/NonDurableCursorTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/NonDurableCursorTest.java
deleted file mode 100644
index ff99da3903..0000000000
--- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/NonDurableCursorTest.java
+++ /dev/null
@@ -1,580 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.bookkeeper.mledger.impl;
-
-import static org.testng.Assert.*;
-
-import com.google.common.base.Charsets;
-import com.google.common.collect.Iterables;
-import java.nio.charset.Charset;
-import java.util.List;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.TimeUnit;
-import java.util.concurrent.atomic.AtomicBoolean;
-import java.util.concurrent.atomic.AtomicReference;
-import org.apache.bookkeeper.mledger.AsyncCallbacks;
-import org.apache.bookkeeper.mledger.AsyncCallbacks.AddEntryCallback;
-import org.apache.bookkeeper.mledger.AsyncCallbacks.MarkDeleteCallback;
-import org.apache.bookkeeper.mledger.Entry;
-import org.apache.bookkeeper.mledger.ManagedCursor;
-import org.apache.bookkeeper.mledger.ManagedLedger;
-import org.apache.bookkeeper.mledger.ManagedLedgerConfig;
-import org.apache.bookkeeper.mledger.ManagedLedgerException;
-import org.apache.bookkeeper.mledger.ManagedLedgerFactory;
-import org.apache.bookkeeper.mledger.ManagedLedgerFactoryConfig;
-import org.apache.bookkeeper.mledger.Position;
-import org.apache.bookkeeper.test.MockedBookKeeperTestCase;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.annotations.Test;
-
-public class NonDurableCursorTest extends MockedBookKeeperTestCase {
-
- private static final Charset Encoding = Charsets.UTF_8;
-
- @Test(timeOut = 20000)
- void readFromEmptyLedger() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger");
-
- ManagedCursor c1 = ledger.newNonDurableCursor(PositionImpl.earliest);
- List<Entry> entries = c1.readEntries(10);
- assertEquals(entries.size(), 0);
- entries.forEach(e -> e.release());
-
- ledger.addEntry("test".getBytes(Encoding));
- entries = c1.readEntries(10);
- assertEquals(entries.size(), 1);
- entries.forEach(e -> e.release());
-
- entries = c1.readEntries(10);
- assertEquals(entries.size(), 0);
- entries.forEach(e -> e.release());
-
- // Test string representation
- assertEquals(c1.toString(), "NonDurableCursorImpl{ledger=my_test_ledger, ackPos=3:-1, readPos=3:1}");
- }
-
- @Test(timeOut = 20000)
- void testZNodeBypassed() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger");
-
- ManagedCursor c1 = ledger.newNonDurableCursor(PositionImpl.earliest);
- assertTrue(Iterables.isEmpty(ledger.getCursors()));
-
- c1.close();
- ledger.close();
-
- // Re-open
- ManagedLedger ledger2 = factory.open("my_test_ledger");
- assertTrue(Iterables.isEmpty(ledger2.getCursors()));
- }
-
- @Test(timeOut = 20000)
- void readTwice() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger",
- new ManagedLedgerConfig().setRetentionTime(1, TimeUnit.HOURS).setRetentionSizeInMB(1));
-
- ManagedCursor c1 = ledger.newNonDurableCursor(PositionImpl.latest);
- ManagedCursor c2 = ledger.newNonDurableCursor(PositionImpl.latest);
-
- ledger.addEntry("entry-1".getBytes(Encoding));
- ledger.addEntry("entry-2".getBytes(Encoding));
-
- List<Entry> entries = c1.readEntries(2);
- assertEquals(entries.size(), 2);
- entries.forEach(e -> e.release());
-
- entries = c1.readEntries(2);
- assertEquals(entries.size(), 0);
- entries.forEach(e -> e.release());
-
- entries = c2.readEntries(2);
- assertEquals(entries.size(), 2);
- entries.forEach(e -> e.release());
-
- entries = c2.readEntries(2);
- assertEquals(entries.size(), 0);
- entries.forEach(e -> e.release());
- }
-
- @Test(timeOut = 20000)
- void readWithCacheDisabled() throws Exception {
- ManagedLedgerFactoryConfig config = new ManagedLedgerFactoryConfig();
- config.setMaxCacheSize(0);
- factory = new ManagedLedgerFactoryImpl(bkc, bkc.getZkHandle(), config);
- ManagedLedger ledger = factory.open("my_test_ledger", new ManagedLedgerConfig().setMaxEntriesPerLedger(1)
- .setRetentionTime(1, TimeUnit.HOURS).setRetentionSizeInMB(1));
-
- ManagedCursor c1 = ledger.newNonDurableCursor(PositionImpl.latest);
- ManagedCursor c2 = ledger.newNonDurableCursor(PositionImpl.latest);
-
- ledger.addEntry("entry-1".getBytes(Encoding));
- ledger.addEntry("entry-2".getBytes(Encoding));
-
- List<Entry> entries = c1.readEntries(2);
- assertEquals(entries.size(), 2);
- assertEquals(new String(entries.get(0).getData(), Encoding), "entry-1");
- assertEquals(new String(entries.get(1).getData(), Encoding), "entry-2");
- entries.forEach(e -> e.release());
-
- entries = c1.readEntries(2);
- assertEquals(entries.size(), 0);
- entries.forEach(e -> e.release());
-
- entries = c2.readEntries(2);
- assertEquals(entries.size(), 2);
- entries.forEach(e -> e.release());
-
- entries = c2.readEntries(2);
- assertEquals(entries.size(), 0);
- entries.forEach(e -> e.release());
- }
-
- @Test(timeOut = 20000)
- void readFromClosedLedger() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger", new ManagedLedgerConfig().setMaxEntriesPerLedger(1)
- .setRetentionTime(1, TimeUnit.HOURS).setRetentionSizeInMB(1));
-
- ManagedCursor c1 = ledger.newNonDurableCursor(PositionImpl.latest);
-
- ledger.close();
-
- try {
- c1.readEntries(2);
- fail("ledger is closed, should fail");
- } catch (ManagedLedgerException e) {
- // ok
- }
- }
-
- @Test(timeOut = 20000)
- void testNumberOfEntries() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger", new ManagedLedgerConfig().setMaxEntriesPerLedger(2)
- .setRetentionTime(1, TimeUnit.HOURS).setRetentionSizeInMB(1));
-
- ManagedCursor c1 = ledger.newNonDurableCursor(PositionImpl.latest);
- ledger.addEntry("dummy-entry-1".getBytes(Encoding));
- ManagedCursor c2 = ledger.newNonDurableCursor(PositionImpl.latest);
- ledger.addEntry("dummy-entry-2".getBytes(Encoding));
- ManagedCursor c3 = ledger.newNonDurableCursor(PositionImpl.latest);
- ledger.addEntry("dummy-entry-3".getBytes(Encoding));
- ManagedCursor c4 = ledger.newNonDurableCursor(PositionImpl.latest);
- ledger.addEntry("dummy-entry-4".getBytes(Encoding));
- ManagedCursor c5 = ledger.newNonDurableCursor(PositionImpl.latest);
-
- assertEquals(c1.getNumberOfEntries(), 4);
- assertEquals(c1.hasMoreEntries(), true);
-
- assertEquals(c2.getNumberOfEntries(), 3);
- assertEquals(c2.hasMoreEntries(), true);
-
- assertEquals(c3.getNumberOfEntries(), 2);
- assertEquals(c3.hasMoreEntries(), true);
-
- assertEquals(c4.getNumberOfEntries(), 1);
- assertEquals(c4.hasMoreEntries(), true);
-
- assertEquals(c5.getNumberOfEntries(), 0);
- assertEquals(c5.hasMoreEntries(), false);
-
- List<Entry> entries = c1.readEntries(2);
- assertEquals(entries.size(), 2);
- c1.markDelete(entries.get(1).getPosition());
- assertEquals(c1.getNumberOfEntries(), 2);
- entries.forEach(e -> e.release());
- }
-
- @Test(timeOut = 20000)
- void testNumberOfEntriesInBacklog() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger", new ManagedLedgerConfig().setMaxEntriesPerLedger(2)
- .setRetentionTime(1, TimeUnit.HOURS).setRetentionSizeInMB(1));
-
- ManagedCursor c1 = ledger.newNonDurableCursor(PositionImpl.latest);
- Position p1 = ledger.addEntry("dummy-entry-1".getBytes(Encoding));
- ManagedCursor c2 = ledger.newNonDurableCursor(PositionImpl.latest);
- ledger.addEntry("dummy-entry-2".getBytes(Encoding));
- ManagedCursor c3 = ledger.newNonDurableCursor(PositionImpl.latest);
- Position p3 = ledger.addEntry("dummy-entry-3".getBytes(Encoding));
- ManagedCursor c4 = ledger.newNonDurableCursor(PositionImpl.latest);
- Position p4 = ledger.addEntry("dummy-entry-4".getBytes(Encoding));
- ManagedCursor c5 = ledger.newNonDurableCursor(PositionImpl.latest);
-
- assertEquals(c1.getNumberOfEntriesInBacklog(), 4);
- assertEquals(c2.getNumberOfEntriesInBacklog(), 3);
- assertEquals(c3.getNumberOfEntriesInBacklog(), 2);
- assertEquals(c4.getNumberOfEntriesInBacklog(), 1);
- assertEquals(c5.getNumberOfEntriesInBacklog(), 0);
-
- List<Entry> entries = c1.readEntries(2);
- assertEquals(entries.size(), 2);
- entries.forEach(e -> e.release());
-
- assertEquals(c1.getNumberOfEntries(), 2);
- assertEquals(c1.getNumberOfEntriesInBacklog(), 4);
-
- c1.markDelete(p1);
- assertEquals(c1.getNumberOfEntries(), 2);
- assertEquals(c1.getNumberOfEntriesInBacklog(), 3);
-
- c1.delete(p3);
-
- assertEquals(c1.getNumberOfEntries(), 1);
- assertEquals(c1.getNumberOfEntriesInBacklog(), 2);
-
- c1.markDelete(p4);
- assertEquals(c1.getNumberOfEntries(), 0);
- assertEquals(c1.getNumberOfEntriesInBacklog(), 0);
- }
-
- @Test(timeOut = 20000)
- void markDeleteWithErrors() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger");
- ManagedCursor cursor = ledger.openCursor("c1");
- ledger.addEntry("dummy-entry-1".getBytes(Encoding));
- List<Entry> entries = cursor.readEntries(100);
-
- stopBookKeeper();
- assertEquals(entries.size(), 1);
-
- // Mark-delete should succeed if BK is down
- cursor.markDelete(entries.get(0).getPosition());
-
- entries.forEach(e -> e.release());
- }
-
- @Test(timeOut = 20000)
- void markDeleteAcrossLedgers() throws Exception {
- ManagedLedger ml1 = factory.open("my_test_ledger");
- ManagedCursor mc1 = ml1.openCursor("c1");
-
- // open ledger id 3 for ml1
- // markDeletePosition for mc1 is 3:-1
- // readPosition is 3:0
-
- ml1.close();
- mc1.close();
-
- // force removal of this ledger from the cache
- factory.close(ml1);
-
- ManagedLedger ml2 = factory.open("my_test_ledger");
- ManagedCursor mc2 = ml2.openCursor("c1");
-
- // open ledger id 5 for ml2
- // this entry is written at 5:0
- Position pos = ml2.addEntry("dummy-entry-1".getBytes(Encoding));
-
- List<Entry> entries = mc2.readEntries(1);
- assertEquals(entries.size(), 1);
- assertEquals(new String(entries.get(0).getData(), Encoding), "dummy-entry-1");
- entries.forEach(e -> e.release());
-
- mc2.delete(pos);
-
- // verify if the markDeletePosition moves from 3:-1 to 5:0
- assertEquals(mc2.getMarkDeletedPosition(), pos);
- assertEquals(mc2.getMarkDeletedPosition().getNext(), mc2.getReadPosition());
- }
-
- @Test(timeOut = 20000)
- void testResetCursor() throws Exception {
- ManagedLedger ledger = factory.open("my_test_move_cursor_ledger",
- new ManagedLedgerConfig().setMaxEntriesPerLedger(10));
- ManagedCursor cursor = ledger.newNonDurableCursor(PositionImpl.latest);
- ledger.addEntry("dummy-entry-1".getBytes(Encoding));
- ledger.addEntry("dummy-entry-2".getBytes(Encoding));
- ledger.addEntry("dummy-entry-3".getBytes(Encoding));
- PositionImpl lastPosition = (PositionImpl) ledger.addEntry("dummy-entry-4".getBytes(Encoding));
- final AtomicBoolean moveStatus = new AtomicBoolean(false);
- PositionImpl resetPosition = new PositionImpl(lastPosition.getLedgerId(), lastPosition.getEntryId() - 2);
- try {
- cursor.resetCursor(resetPosition);
- moveStatus.set(true);
- } catch (Exception e) {
- log.warn("error in reset cursor", e.getCause());
- }
-
- assertTrue(moveStatus.get());
- assertTrue(cursor.getReadPosition().equals(resetPosition));
- cursor.close();
- ledger.close();
- }
-
- @Test(timeOut = 20000)
- void testasyncResetCursor() throws Exception {
- ManagedLedger ledger = factory.open("my_test_move_cursor_ledger",
- new ManagedLedgerConfig().setMaxEntriesPerLedger(10));
- ManagedCursor cursor = ledger.newNonDurableCursor(PositionImpl.latest);
- ledger.addEntry("dummy-entry-1".getBytes(Encoding));
- ledger.addEntry("dummy-entry-2".getBytes(Encoding));
- ledger.addEntry("dummy-entry-3".getBytes(Encoding));
- PositionImpl lastPosition = (PositionImpl) ledger.addEntry("dummy-entry-4".getBytes(Encoding));
- final AtomicBoolean moveStatus = new AtomicBoolean(false);
- CountDownLatch countDownLatch = new CountDownLatch(1);
- PositionImpl resetPosition = new PositionImpl(lastPosition.getLedgerId(), lastPosition.getEntryId() - 2);
-
- cursor.asyncResetCursor(resetPosition, new AsyncCallbacks.ResetCursorCallback() {
- @Override
- public void resetComplete(Object ctx) {
- moveStatus.set(true);
- countDownLatch.countDown();
- }
-
- @Override
- public void resetFailed(ManagedLedgerException exception, Object ctx) {
- moveStatus.set(false);
- countDownLatch.countDown();
- }
- });
- countDownLatch.await();
- assertTrue(moveStatus.get());
- assertTrue(cursor.getReadPosition().equals(resetPosition));
- cursor.close();
- ledger.close();
- }
-
- @Test(timeOut = 20000)
- void rewind() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger", new ManagedLedgerConfig().setMaxEntriesPerLedger(2)
- .setRetentionTime(1, TimeUnit.HOURS).setRetentionSizeInMB(1));
- ManagedCursor c1 = ledger.newNonDurableCursor(PositionImpl.earliest);
- Position p1 = ledger.addEntry("dummy-entry-1".getBytes(Encoding));
- Position p2 = ledger.addEntry("dummy-entry-2".getBytes(Encoding));
- Position p3 = ledger.addEntry("dummy-entry-3".getBytes(Encoding));
- Position p4 = ledger.addEntry("dummy-entry-4".getBytes(Encoding));
-
- log.debug("p1: {}", p1);
- log.debug("p2: {}", p2);
- log.debug("p3: {}", p3);
- log.debug("p4: {}", p4);
-
- assertEquals(c1.getNumberOfEntries(), 4);
- assertEquals(c1.getNumberOfEntriesInBacklog(), 4);
- c1.markDelete(p1);
- assertEquals(c1.getNumberOfEntries(), 3);
- assertEquals(c1.getNumberOfEntriesInBacklog(), 3);
- List<Entry> entries = c1.readEntries(10);
- assertEquals(entries.size(), 3);
- entries.forEach(e -> e.release());
-
- assertEquals(c1.getNumberOfEntries(), 0);
- assertEquals(c1.getNumberOfEntriesInBacklog(), 3);
- c1.rewind();
- assertEquals(c1.getNumberOfEntries(), 3);
- assertEquals(c1.getNumberOfEntriesInBacklog(), 3);
- c1.markDelete(p2);
- assertEquals(c1.getNumberOfEntries(), 2);
- assertEquals(c1.getNumberOfEntriesInBacklog(), 2);
-
- entries = c1.readEntries(10);
- assertEquals(entries.size(), 2);
- entries.forEach(e -> e.release());
-
- assertEquals(c1.getNumberOfEntries(), 0);
- assertEquals(c1.getNumberOfEntriesInBacklog(), 2);
- c1.rewind();
- assertEquals(c1.getNumberOfEntries(), 2);
- c1.markDelete(p4);
- assertEquals(c1.getNumberOfEntries(), 0);
- assertEquals(c1.getNumberOfEntriesInBacklog(), 0);
- c1.rewind();
- assertEquals(c1.getNumberOfEntries(), 0);
- ledger.addEntry("dummy-entry-5".getBytes(Encoding));
- assertEquals(c1.getNumberOfEntries(), 1);
- ledger.addEntry("dummy-entry-6".getBytes(Encoding));
- assertEquals(c1.getNumberOfEntries(), 2);
- }
-
- @Test(timeOut = 20000)
- void markDeleteSkippingMessage() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger", new ManagedLedgerConfig().setMaxEntriesPerLedger(10));
- ManagedCursor cursor = ledger.newNonDurableCursor(PositionImpl.earliest);
- Position p1 = ledger.addEntry("dummy-entry-1".getBytes(Encoding));
- Position p2 = ledger.addEntry("dummy-entry-2".getBytes(Encoding));
- ledger.addEntry("dummy-entry-3".getBytes(Encoding));
- PositionImpl p4 = (PositionImpl) ledger.addEntry("dummy-entry-4".getBytes(Encoding));
-
- assertEquals(cursor.getNumberOfEntries(), 4);
-
- cursor.markDelete(p1);
- assertEquals(cursor.hasMoreEntries(), true);
- assertEquals(cursor.getNumberOfEntries(), 3);
-
- assertEquals(cursor.getReadPosition(), p2);
-
- List<Entry> entries = cursor.readEntries(1);
- assertEquals(entries.size(), 1);
- assertEquals(new String(entries.get(0).getData(), Encoding), "dummy-entry-2");
- entries.forEach(e -> e.release());
-
- cursor.markDelete(p4);
- assertEquals(cursor.hasMoreEntries(), false);
- assertEquals(cursor.getNumberOfEntries(), 0);
-
- assertEquals(cursor.getReadPosition(), new PositionImpl(p4.getLedgerId(), p4.getEntryId() + 1));
- }
-
- @Test(timeOut = 20000)
- public void asyncMarkDeleteBlocking() throws Exception {
- ManagedLedgerConfig config = new ManagedLedgerConfig();
- config.setMaxEntriesPerLedger(10);
- config.setMetadataMaxEntriesPerLedger(5);
- ManagedLedger ledger = factory.open("my_test_ledger", config);
- final ManagedCursor c1 = ledger.openCursor("c1");
- final AtomicReference<Position> lastPosition = new AtomicReference<Position>();
-
- final int N = 100;
- final CountDownLatch latch = new CountDownLatch(N);
- for (int i = 0; i < N; i++) {
- ledger.asyncAddEntry("entry".getBytes(Encoding), new AddEntryCallback() {
- @Override
- public void addFailed(ManagedLedgerException exception, Object ctx) {
- }
-
- @Override
- public void addComplete(Position position, Object ctx) {
- lastPosition.set(position);
- c1.asyncMarkDelete(position, new MarkDeleteCallback() {
- @Override
- public void markDeleteFailed(ManagedLedgerException exception, Object ctx) {
- }
-
- @Override
- public void markDeleteComplete(Object ctx) {
- latch.countDown();
- }
- }, null);
- }
- }, null);
- }
-
- latch.await();
-
- assertEquals(c1.getNumberOfEntries(), 0);
-
- // Reopen
- ManagedLedgerFactory factory2 = new ManagedLedgerFactoryImpl(bkc, bkc.getZkHandle());
- ledger = factory2.open("my_test_ledger");
- ManagedCursor c2 = ledger.openCursor("c1");
-
- assertEquals(c2.getMarkDeletedPosition(), lastPosition.get());
- factory2.shutdown();
- }
-
- @Test(timeOut = 20000)
- void unorderedMarkDelete() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger");
- final ManagedCursor c1 = ledger.openCursor("c1");
-
- Position p1 = ledger.addEntry("entry-1".getBytes(Encoding));
- Position p2 = ledger.addEntry("entry-2".getBytes(Encoding));
-
- c1.markDelete(p2);
- try {
- c1.markDelete(p1);
- fail("Should have thrown exception");
- } catch (ManagedLedgerException e) {
- // ok
- }
-
- assertEquals(c1.getMarkDeletedPosition(), p2);
- }
-
- @Test(timeOut = 20000)
- void testSingleDelete() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger", new ManagedLedgerConfig().setMaxEntriesPerLedger(3)
- .setRetentionTime(1, TimeUnit.HOURS).setRetentionSizeInMB(1));
- ManagedCursor cursor = ledger.newNonDurableCursor(PositionImpl.latest);
-
- Position p1 = ledger.addEntry("entry1".getBytes());
- Position p2 = ledger.addEntry("entry2".getBytes());
- Position p3 = ledger.addEntry("entry3".getBytes());
- Position p4 = ledger.addEntry("entry4".getBytes());
- Position p5 = ledger.addEntry("entry5".getBytes());
- Position p6 = ledger.addEntry("entry6".getBytes());
-
- Position p0 = cursor.getMarkDeletedPosition();
-
- cursor.delete(p4);
- assertEquals(cursor.getMarkDeletedPosition(), p0);
-
- cursor.delete(p1);
- assertEquals(cursor.getMarkDeletedPosition(), p1);
-
- cursor.delete(p3);
-
- // Delete will silently succeed
- cursor.delete(p3);
- assertEquals(cursor.getMarkDeletedPosition(), p1);
-
- cursor.delete(p2);
- assertEquals(cursor.getMarkDeletedPosition(), p4);
-
- cursor.delete(p5);
- assertEquals(cursor.getMarkDeletedPosition(), p5);
-
- cursor.close();
- try {
- cursor.delete(p6);
- } catch (ManagedLedgerException e) {
- // Ok
- }
- }
-
- @Test(timeOut = 20000)
- void subscribeToEarliestPositionWithImmediateDeletion() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger", new ManagedLedgerConfig().setMaxEntriesPerLedger(1));
-
- /* Position p1 = */ ledger.addEntry("entry-1".getBytes());
- /* Position p2 = */ ledger.addEntry("entry-2".getBytes());
- Position p3 = ledger.addEntry("entry-3".getBytes());
-
- Thread.sleep(300);
- ManagedCursor c1 = ledger.newNonDurableCursor(PositionImpl.earliest);
- assertEquals(c1.getReadPosition(), p3);
- assertEquals(c1.getMarkDeletedPosition(), new PositionImpl(5, -1));
- }
-
- @Test // (timeOut = 20000)
- void subscribeToEarliestPositionWithDeferredDeletion() throws Exception {
- ManagedLedger ledger = factory.open("my_test_ledger", new ManagedLedgerConfig().setMaxEntriesPerLedger(1)
- .setRetentionTime(1, TimeUnit.HOURS).setRetentionSizeInMB(1));
-
- Position p1 = ledger.addEntry("entry-1".getBytes());
- Position p2 = ledger.addEntry("entry-2".getBytes());
- /* Position p3 = */ ledger.addEntry("entry-3".getBytes());
- /* Position p4 = */ ledger.addEntry("entry-4".getBytes());
- /* Position p5 = */ ledger.addEntry("entry-5".getBytes());
- /* Position p6 = */ ledger.addEntry("entry-6".getBytes());
-
- ManagedCursor c1 = ledger.newNonDurableCursor(PositionImpl.earliest);
- assertEquals(c1.getReadPosition(), p1);
- assertEquals(c1.getMarkDeletedPosition(), new PositionImpl(3, -1));
- assertEquals(c1.getNumberOfEntries(), 6);
- assertEquals(c1.getNumberOfEntriesInBacklog(), 6);
-
- ManagedCursor c2 = ledger.newNonDurableCursor(p1);
- assertEquals(c2.getReadPosition(), p2);
- assertEquals(c2.getMarkDeletedPosition(), p1);
- assertEquals(c2.getNumberOfEntries(), 5);
- assertEquals(c2.getNumberOfEntriesInBacklog(), 5);
- }
-
- private static final Logger log = LoggerFactory.getLogger(NonDurableCursorTest.class);
-}
diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/OffloadLedgerDeleteTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/OffloadLedgerDeleteTest.java
deleted file mode 100644
index b574d90617..0000000000
--- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/OffloadLedgerDeleteTest.java
+++ /dev/null
@@ -1,203 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.bookkeeper.mledger.impl;
-
-import static org.apache.bookkeeper.mledger.impl.OffloadPrefixTest.assertEventuallyTrue;
-
-import java.util.Map;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.TimeUnit;
-import java.util.stream.Collectors;
-
-import org.apache.bookkeeper.mledger.ManagedCursor;
-import org.apache.bookkeeper.mledger.ManagedLedgerConfig;
-import org.apache.bookkeeper.mledger.util.MockClock;
-import org.apache.bookkeeper.test.MockedBookKeeperTestCase;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.testng.Assert;
-import org.testng.annotations.Test;
-
-public class OffloadLedgerDeleteTest extends MockedBookKeeperTestCase {
- private static final Logger log = LoggerFactory.getLogger(OffloadLedgerDeleteTest.class);
-
- @Test
- public void testLaggedDelete() throws Exception {
- OffloadPrefixTest.MockLedgerOffloader offloader = new OffloadPrefixTest.MockLedgerOffloader();
-
- ManagedLedgerConfig config = new ManagedLedgerConfig();
- MockClock clock = new MockClock();
- config.setMaxEntriesPerLedger(10);
- config.setMinimumRolloverTime(0, TimeUnit.SECONDS);
- config.setRetentionTime(10, TimeUnit.MINUTES);
- config.setOffloadLedgerDeletionLag(5, TimeUnit.MINUTES);
- config.setLedgerOffloader(offloader);
- config.setClock(clock);
-
- ManagedLedgerImpl ledger = (ManagedLedgerImpl)factory.open("my_test_ledger", config);
- int i = 0;
- for (; i < 15; i++) {
- String content = "entry-" + i;
- ledger.addEntry(content.getBytes());
- }
- Assert.assertEquals(ledger.getLedgersInfoAsList().size(), 2);
- long firstLedgerId = ledger.getLedgersInfoAsList().get(0).getLedgerId();
-
- ledger.offloadPrefix(ledger.getLastConfirmedEntry());
-
- Assert.assertEquals(ledger.getLedgersInfoAsList().size(), 2);
- Assert.assertEquals(ledger.getLedgersInfoAsList().stream()
- .filter(e -> e.getOffloadContext().getComplete())
- .map(e -> e.getLedgerId()).collect(Collectors.toSet()),
- offloader.offloadedLedgers());
- Assert.assertTrue(bkc.getLedgers().contains(firstLedgerId));
-
- clock.advance(2, TimeUnit.MINUTES);
- CompletableFuture<Void> promise = new CompletableFuture<>();
- ledger.internalTrimConsumedLedgers(promise);
- promise.join();
- Assert.assertTrue(bkc.getLedgers().contains(firstLedgerId));
-
- clock.advance(5, TimeUnit.MINUTES);
- CompletableFuture<Void> promise2 = new CompletableFuture<>();
- ledger.internalTrimConsumedLedgers(promise2);
- promise2.join();
-
- // assert bk ledger is deleted
- assertEventuallyTrue(() -> !bkc.getLedgers().contains(firstLedgerId));
-
- // ledger still exists in list
- Assert.assertEquals(ledger.getLedgersInfoAsList().stream()
- .filter(e -> e.getOffloadContext().getComplete())
- .map(e -> e.getLedgerId()).collect(Collectors.toSet()),
- offloader.offloadedLedgers());
-
- // move past retention, should be deleted from offloaded also
- clock.advance(5, TimeUnit.MINUTES);
- CompletableFuture<Void> promise3 = new CompletableFuture<>();
- ledger.internalTrimConsumedLedgers(promise3);
- promise3.join();
-
- Assert.assertEquals(ledger.getLedgersInfoAsList().size(), 1);
- assertEventuallyTrue(() -> offloader.deletedOffloads().contains(firstLedgerId));
- }
-
- @Test
- public void testLaggedDeleteRetentionSetLower() throws Exception {
- OffloadPrefixTest.MockLedgerOffloader offloader = new OffloadPrefixTest.MockLedgerOffloader();
-
- ManagedLedgerConfig config = new ManagedLedgerConfig();
- MockClock clock = new MockClock();
- config.setMaxEntriesPerLedger(10);
- config.setMinimumRolloverTime(0, TimeUnit.SECONDS);
- config.setRetentionTime(5, TimeUnit.MINUTES);
- config.setOffloadLedgerDeletionLag(10, TimeUnit.MINUTES);
- config.setLedgerOffloader(offloader);
- config.setClock(clock);
-
- ManagedLedgerImpl ledger = (ManagedLedgerImpl)factory.open("my_test_ledger", config);
- int i = 0;
- for (; i < 15; i++) {
- String content = "entry-" + i;
- ledger.addEntry(content.getBytes());
- }
- Assert.assertEquals(ledger.getLedgersInfoAsList().size(), 2);
- long firstLedgerId = ledger.getLedgersInfoAsList().get(0).getLedgerId();
-
- ledger.offloadPrefix(ledger.getLastConfirmedEntry());
-
- Assert.assertEquals(ledger.getLedgersInfoAsList().size(), 2);
- Assert.assertEquals(ledger.getLedgersInfoAsList().stream()
- .filter(e -> e.getOffloadContext().getComplete())
- .map(e -> e.getLedgerId()).collect(Collectors.toSet()),
- offloader.offloadedLedgers());
- Assert.assertTrue(bkc.getLedgers().contains(firstLedgerId));
-
- clock.advance(2, TimeUnit.MINUTES);
- CompletableFuture<Void> promise = new CompletableFuture<>();
- ledger.internalTrimConsumedLedgers(promise);
- promise.join();
- Assert.assertTrue(bkc.getLedgers().contains(firstLedgerId));
-
- clock.advance(5, TimeUnit.MINUTES);
- CompletableFuture<Void> promise2 = new CompletableFuture<>();
- ledger.internalTrimConsumedLedgers(promise2);
- promise2.join();
-
- // ensure it gets deleted from both bookkeeper and offloader
- assertEventuallyTrue(() -> !bkc.getLedgers().contains(firstLedgerId));
- assertEventuallyTrue(() -> offloader.deletedOffloads().contains(firstLedgerId));
- }
-
- @Test
- public void testLaggedDeleteSlowConsumer() throws Exception {
- OffloadPrefixTest.MockLedgerOffloader offloader = new OffloadPrefixTest.MockLedgerOffloader();
-
- ManagedLedgerConfig config = new ManagedLedgerConfig();
- MockClock clock = new MockClock();
- config.setMaxEntriesPerLedger(10);
- config.setMinimumRolloverTime(0, TimeUnit.SECONDS);
- config.setRetentionTime(10, TimeUnit.MINUTES);
- config.setOffloadLedgerDeletionLag(5, TimeUnit.MINUTES);
- config.setLedgerOffloader(offloader);
- config.setClock(clock);
-
- ManagedLedgerImpl ledger = (ManagedLedgerImpl)factory.open("my_test_ledger", config);
- ManagedCursor cursor = ledger.openCursor("sub1");
-
- for (int i = 0; i < 15; i++) {
- String content = "entry-" + i;
- ledger.addEntry(content.getBytes());
- }
- Assert.assertEquals(ledger.getLedgersInfoAsList().size(), 2);
- long firstLedgerId = ledger.getLedgersInfoAsList().get(0).getLedgerId();
-
- ledger.offloadPrefix(ledger.getLastConfirmedEntry());
-
- Assert.assertEquals(ledger.getLedgersInfoAsList().size(), 2);
- Assert.assertEquals(ledger.getLedgersInfoAsList().stream()
- .filter(e -> e.getOffloadContext().getComplete())
- .map(e -> e.getLedgerId()).collect(Collectors.toSet()),
- offloader.offloadedLedgers());
- Assert.assertTrue(bkc.getLedgers().contains(firstLedgerId));
-
- clock.advance(2, TimeUnit.MINUTES);
-
- CompletableFuture<Void> promise = new CompletableFuture<>();
- ledger.internalTrimConsumedLedgers(promise);
- promise.join();
- Assert.assertTrue(bkc.getLedgers().contains(firstLedgerId));
-
- clock.advance(5, TimeUnit.MINUTES);
- CompletableFuture<Void> promise2 = new CompletableFuture<>();
- ledger.internalTrimConsumedLedgers(promise2);
- promise2.join();
-
- // assert bk ledger is deleted
- assertEventuallyTrue(() -> !bkc.getLedgers().contains(firstLedgerId));
-
- // ledger still exists in list
- Assert.assertEquals(ledger.getLedgersInfoAsList().stream()
- .filter(e -> e.getOffloadContext().getComplete())
- .map(e -> e.getLedgerId()).collect(Collectors.toSet()),
- offloader.offloadedLedgers());
- }
-}
diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/OffloadPrefixReadTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/OffloadPrefixReadTest.java
deleted file mode 100644
index 4bf518f3a1..0000000000
--- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/OffloadPrefixReadTest.java
+++ /dev/null
@@ -1,300 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.bookkeeper.mledger.impl;
-
-import static org.mockito.Matchers.anyLong;
-import static org.mockito.Matchers.anyMap;
-import static org.mockito.Matchers.anyObject;
-import static org.mockito.Matchers.eq;
-import static org.mockito.Mockito.spy;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
-
-import com.google.common.collect.ImmutableMap;
-import com.google.common.collect.Lists;
-
-import io.netty.buffer.ByteBuf;
-
-import java.util.Collections;
-import java.util.List;
-import java.util.Map;
-import java.util.NavigableMap;
-import java.util.UUID;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.bookkeeper.client.api.DigestType;
-import org.apache.bookkeeper.client.api.LastConfirmedAndEntry;
-import org.apache.bookkeeper.client.api.LedgerEntries;
-import org.apache.bookkeeper.client.api.LedgerEntry;
-import org.apache.bookkeeper.client.api.LedgerMetadata;
-import org.apache.bookkeeper.client.api.ReadHandle;
-import org.apache.bookkeeper.client.impl.LedgerEntriesImpl;
-import org.apache.bookkeeper.client.impl.LedgerEntryImpl;
-
-import org.apache.bookkeeper.mledger.Entry;
-import org.apache.bookkeeper.mledger.LedgerOffloader;
-import org.apache.bookkeeper.mledger.ManagedCursor;
-import org.apache.bookkeeper.mledger.ManagedLedgerConfig;
-import org.apache.bookkeeper.net.BookieSocketAddress;
-import org.apache.bookkeeper.test.MockedBookKeeperTestCase;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.testng.Assert;
-import org.testng.annotations.Test;
-
-public class OffloadPrefixReadTest extends MockedBookKeeperTestCase {
- private static final Logger log = LoggerFactory.getLogger(OffloadPrefixReadTest.class);
-
- @Test
- public void testOffloadRead() throws Exception {
- MockLedgerOffloader offloader = spy(new MockLedgerOffloader());
- ManagedLedgerConfig config = new ManagedLedgerConfig();
- config.setMaxEntriesPerLedger(10);
- config.setMinimumRolloverTime(0, TimeUnit.SECONDS);
- config.setRetentionTime(10, TimeUnit.MINUTES);
- config.setLedgerOffloader(offloader);
- ManagedLedgerImpl ledger = (ManagedLedgerImpl)factory.open("my_test_ledger", config);
-
- for (int i = 0; i < 25; i++) {
- String content = "entry-" + i;
- ledger.addEntry(content.getBytes());
- }
- Assert.assertEquals(ledger.getLedgersInfoAsList().size(), 3);
-
- ledger.offloadPrefix(ledger.getLastConfirmedEntry());
-
- Assert.assertEquals(ledger.getLedgersInfoAsList().size(), 3);
- Assert.assertEquals(ledger.getLedgersInfoAsList().stream()
- .filter(e -> e.getOffloadContext().getComplete()).count(), 2);
- Assert.assertTrue(ledger.getLedgersInfoAsList().get(0).getOffloadContext().getComplete());
- Assert.assertTrue(ledger.getLedgersInfoAsList().get(1).getOffloadContext().getComplete());
-
- UUID firstLedgerUUID = new UUID(ledger.getLedgersInfoAsList().get(0).getOffloadContext().getUidMsb(),
- ledger.getLedgersInfoAsList().get(0).getOffloadContext().getUidLsb());
- UUID secondLedgerUUID = new UUID(ledger.getLedgersInfoAsList().get(1).getOffloadContext().getUidMsb(),
- ledger.getLedgersInfoAsList().get(1).getOffloadContext().getUidLsb());
-
- ManagedCursor cursor = ledger.newNonDurableCursor(PositionImpl.earliest);
- int i = 0;
- for (Entry e : cursor.readEntries(10)) {
- Assert.assertEquals(new String(e.getData()), "entry-" + i++);
- }
- verify(offloader, times(1))
- .readOffloaded(anyLong(), anyObject(), anyMap());
- verify(offloader).readOffloaded(anyLong(), eq(firstLedgerUUID), anyMap());
-
- for (Entry e : cursor.readEntries(10)) {
- Assert.assertEquals(new String(e.getData()), "entry-" + i++);
- }
- verify(offloader, times(2))
- .readOffloaded(anyLong(), anyObject(), anyMap());
- verify(offloader).readOffloaded(anyLong(), eq(secondLedgerUUID), anyMap());
-
- for (Entry e : cursor.readEntries(5)) {
- Assert.assertEquals(new String(e.getData()), "entry-" + i++);
- }
- verify(offloader, times(2))
- .readOffloaded(anyLong(), anyObject(), anyMap());
- }
-
- static class MockLedgerOffloader implements LedgerOffloader {
- ConcurrentHashMap<UUID, ReadHandle> offloads = new ConcurrentHashMap<UUID, ReadHandle>();
-
- @Override
- public String getOffloadDriverName() {
- return "mock";
- }
-
- @Override
- public CompletableFuture<Void> offload(ReadHandle ledger,
- UUID uuid,
- Map<String, String> extraMetadata) {
- CompletableFuture<Void> promise = new CompletableFuture<>();
- try {
- offloads.put(uuid, new MockOffloadReadHandle(ledger));
- promise.complete(null);
- } catch (Exception e) {
- promise.completeExceptionally(e);
- }
- return promise;
- }
-
- @Override
- public CompletableFuture<ReadHandle> readOffloaded(long ledgerId, UUID uuid,
- Map<String, String> offloadDriverMetadata) {
- return CompletableFuture.completedFuture(offloads.get(uuid));
- }
-
- @Override
- public CompletableFuture<Void> deleteOffloaded(long ledgerId, UUID uuid,
- Map<String, String> offloadDriverMetadata) {
- offloads.remove(uuid);
- return CompletableFuture.completedFuture(null);
- };
- }
-
- static class MockOffloadReadHandle implements ReadHandle {
- final long id;
- final List<ByteBuf> entries = Lists.newArrayList();
- final LedgerMetadata metadata;
-
- MockOffloadReadHandle(ReadHandle toCopy) throws Exception {
- id = toCopy.getId();
- long lac = toCopy.getLastAddConfirmed();
- try (LedgerEntries entries = toCopy.read(0, lac)) {
- for (LedgerEntry e : entries) {
- this.entries.add(e.getEntryBuffer().retainedSlice());
- }
- }
- metadata = new MockMetadata(toCopy.getLedgerMetadata());
- }
-
- @Override
- public long getId() { return id; }
-
- @Override
- public LedgerMetadata getLedgerMetadata() {
- return metadata;
- }
-
- @Override
- public CompletableFuture<Void> closeAsync() {
- return CompletableFuture.completedFuture(null);
- }
-
- @Override
- public CompletableFuture<LedgerEntries> readAsync(long firstEntry, long lastEntry) {
- List<LedgerEntry> readEntries = Lists.newArrayList();
- for (long eid = firstEntry; eid <= lastEntry; eid++) {
- ByteBuf buf = entries.get((int)eid).retainedSlice();
- readEntries.add(LedgerEntryImpl.create(id, eid, buf.readableBytes(), buf));
- }
- return CompletableFuture.completedFuture(LedgerEntriesImpl.create(readEntries));
- }
-
- @Override
- public CompletableFuture<LedgerEntries> readUnconfirmedAsync(long firstEntry, long lastEntry) {
- return unsupported();
- }
-
- @Override
- public CompletableFuture<Long> readLastAddConfirmedAsync() {
- return unsupported();
- }
-
- @Override
- public CompletableFuture<Long> tryReadLastAddConfirmedAsync() {
- return unsupported();
- }
-
- @Override
- public long getLastAddConfirmed() {
- return entries.size() - 1;
- }
-
- @Override
- public long getLength() {
- return metadata.getLength();
- }
-
- @Override
- public boolean isClosed() {
- return metadata.isClosed();
- }
-
- @Override
- public CompletableFuture<LastConfirmedAndEntry> readLastAddConfirmedAndEntryAsync(long entryId,
- long timeOutInMillis,
- boolean parallel) {
- return unsupported();
- }
-
- private <T> CompletableFuture<T> unsupported() {
- CompletableFuture<T> future = new CompletableFuture<>();
- future.completeExceptionally(new UnsupportedOperationException());
- return future;
- }
- }
-
- static class MockMetadata implements LedgerMetadata {
- private final int ensembleSize;
- private final int writeQuorumSize;
- private final int ackQuorumSize;
- private final long lastEntryId;
- private final long length;
- private final DigestType digestType;
- private final long ctime;
- private final boolean isClosed;
- private final Map<String, byte[]> customMetadata;
-
- MockMetadata(LedgerMetadata toCopy) {
- ensembleSize = toCopy.getEnsembleSize();
- writeQuorumSize = toCopy.getWriteQuorumSize();
- ackQuorumSize = toCopy.getAckQuorumSize();
- lastEntryId = toCopy.getLastEntryId();
- length = toCopy.getLength();
- digestType = toCopy.getDigestType();
- ctime = toCopy.getCtime();
- isClosed = toCopy.isClosed();
-
- customMetadata = ImmutableMap.copyOf(toCopy.getCustomMetadata());
- }
-
- @Override
- public int getEnsembleSize() { return ensembleSize; }
-
- @Override
- public int getWriteQuorumSize() { return writeQuorumSize; }
-
- @Override
- public int getAckQuorumSize() { return ackQuorumSize; }
-
- @Override
- public long getLastEntryId() { return lastEntryId; }
-
- @Override
- public long getLength() { return length; }
-
- @Override
- public DigestType getDigestType() { return digestType; }
-
- @Override
- public long getCtime() { return ctime; }
-
- @Override
- public boolean isClosed() { return isClosed; }
-
- @Override
- public Map<String, byte[]> getCustomMetadata() { return customMetadata; }
-
- @Override
- public List<BookieSocketAddress> getEnsembleAt(long entryId) {
- throw new UnsupportedOperationException("Pulsar shouldn't look at this");
- }
-
- @Override
- public NavigableMap<Long, ? extends List<BookieSocketAddress>> getAllEnsembles() {
- throw new UnsupportedOperationException("Pulsar shouldn't look at this");
- }
- }
-}
diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/OffloadPrefixTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/OffloadPrefixTest.java
deleted file mode 100644
index 351d86b6c3..0000000000
--- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/OffloadPrefixTest.java
+++ /dev/null
@@ -1,995 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.bookkeeper.mledger.impl;
-
-import com.google.common.collect.ImmutableSet;
-
-import java.lang.reflect.Field;
-import java.util.HashMap;
-import java.util.Map;
-import java.util.Set;
-import java.util.UUID;
-import java.util.concurrent.CompletionException;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.TimeUnit;
-import java.util.function.BooleanSupplier;
-import java.util.stream.Collectors;
-
-import org.apache.bookkeeper.client.BKException;
-import org.apache.bookkeeper.client.api.ReadHandle;
-import org.apache.bookkeeper.mledger.AsyncCallbacks.OffloadCallback;
-import org.apache.bookkeeper.mledger.LedgerOffloader;
-import org.apache.bookkeeper.mledger.ManagedCursor;
-import org.apache.bookkeeper.mledger.ManagedLedgerConfig;
-import org.apache.bookkeeper.mledger.ManagedLedgerException;
-import org.apache.bookkeeper.mledger.Position;
-import org.apache.bookkeeper.mledger.proto.MLDataFormats.ManagedLedgerInfo.LedgerInfo;
-import org.apache.bookkeeper.test.MockedBookKeeperTestCase;
-import org.apache.commons.lang3.tuple.Pair;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-
-import org.testng.Assert;
-import org.testng.annotations.Test;
-
-public class OffloadPrefixTest extends MockedBookKeeperTestCase {
- private static final Logger log = LoggerFactory.getLogger(OffloadPrefixTest.class);
-
- @Test
- public void testNullOffloader() throws Exception {
- ManagedLedgerConfig config = new ManagedLedgerConfig();
- config.setMaxEntriesPerLedger(10);
- config.setMinimumRolloverTime(0, TimeUnit.SECONDS);
- config.setRetentionTime(10, TimeUnit.MINUTES);
- ManagedLedgerImpl ledger = (ManagedLedgerImpl)factory.open("my_test_ledger", config);
-
- int i = 0;
- for (; i < 25; i++) {
- String content = "entry-" + i;
- ledger.addEntry(content.getBytes());
- }
- Assert.assertEquals(ledger.getLedgersInfoAsList().size(), 3);
-
- Position p = ledger.getLastConfirmedEntry();
-
- for (; i < 45; i++) {
- String content = "entry-" + i;
- ledger.addEntry(content.getBytes());
- }
- Assert.assertEquals(ledger.getLedgersInfoAsList().size(), 5);
- Assert.assertEquals(ledger.getLedgersInfoAsList().stream()
- .filter(e -> e.getOffloadContext().getComplete()).count(), 0);
- try {
- ledger.offloadPrefix(p);
- Assert.fail("Should have thrown an exception");
- } catch (ManagedLedgerException e) {
- Assert.assertEquals(e.getCause().getClass(), CompletionException.class);
- }
- Assert.assertEquals(ledger.getLedgersInfoAsList().size(), 5);
- Assert.assertEquals(ledger.getLedgersInfoAsList().stream()
- .filter(e -> e.getOffloadContext().getComplete()).count(), 0);
-
- // add more entries to ensure we can update the ledger list
- for (; i < 55; i++) {
- String content = "entry-" + i;
- ledger.addEntry(content.getBytes());
- }
- Assert.assertEquals(ledger.getLedgersInfoAsList().size(), 6);
- Assert.assertEquals(ledger.getLedgersInfoAsList().stream()
- .filter(e -> e.getOffloadContext().getComplete()).count(), 0);
- }
-
- @Test
- public void testOffload() throws Exception {
- MockLedgerOffloader offloader = new MockLedgerOffloader();
- ManagedLedgerConfig config = new ManagedLedgerConfig();
- config.setMaxEntriesPerLedger(10);
- config.setMinimumRolloverTime(0, TimeUnit.SECONDS);
- config.setRetentionTime(10, TimeUnit.MINUTES);
- config.setLedgerOffloader(offloader);
- ManagedLedgerImpl ledger = (ManagedLedgerImpl)factory.open("my_test_ledger", config);
-
- int i = 0;
- for (; i < 25; i++) {
- String content = "entry-" + i;
- ledger.addEntry(content.getBytes());
- }
- Assert.assertEquals(ledger.getLedgersInfoAsList().size(), 3);
-
- ledger.offloadPrefix(ledger.getLastConfirmedEntry());
-
- Assert.assertEquals(ledger.getLedgersInfoAsList().size(), 3);
- Assert.assertEquals(ledger.getLedgersInfoAsList().stream()
- .filter(e -> e.getOffloadContext().getComplete())
- .map(e -> e.getLedgerId()).collect(Collectors.toSet()),
- offloader.offloadedLedgers());
- }
-
- @Test
- public void testPositionOutOfRange() throws Exception {
- MockLedgerOffloader offloader = new MockLedgerOffloader();
- ManagedLedgerConfig config = new ManagedLedgerConfig();
- config.setMaxEntriesPerLedger(10);
- config.setMinimumRolloverTime(0, TimeUnit.SECONDS);
- config.setRetentionTime(10, TimeUnit.MINUTES);
- config.setLedgerOffloader(offloader);
- ManagedLedgerImpl ledger = (ManagedLedgerImpl)factory.open("my_test_ledger", config);
-
- int i = 0;
- for (; i < 25; i++) {
- String content = "entry-" + i;
- ledger.addEntry(content.getBytes());
- }
- Assert.assertEquals(ledger.getLedgersInfoAsList().size(), 3);
-
- try {
- ledger.offloadPrefix(PositionImpl.earliest);
- Assert.fail("Should have thrown an exception");
- } catch (ManagedLedgerException.InvalidCursorPositionException e) {
- // expected
- }
- try {
- ledger.offloadPrefix(PositionImpl.latest);
- Assert.fail("Should have thrown an exception");
- } catch (ManagedLedgerException.InvalidCursorPositionException e) {
- // expected
- }
-
- Assert.assertEquals(ledger.getLedgersInfoAsList().stream()
- .filter(e -> e.getOffloadContext().getComplete()).count(), 0);
- Assert.assertEquals(offloader.offloadedLedgers().size(), 0);
- }
-
- @Test
- public void testPositionOnEdgeOfLedger() throws Exception {
- MockLedgerOffloader offloader = new MockLedgerOffloader();
- ManagedLedgerConfig config = new ManagedLedgerConfig();
- config.setMaxEntriesPerLedger(10);
- config.setMinimumRolloverTime(0, TimeUnit.SECONDS);
- config.setRetentionTime(10, TimeUnit.MINUTES);
- config.setLedgerOffloader(offloader);
- ManagedLedgerImpl ledger = (ManagedLedgerImpl)factory.open("my_test_ledger", config);
-
- int i = 0;
- for (; i < 20; i++) {
- String content = "entry-" + i;
- ledger.addEntry(content.getBytes());
- }
- Assert.assertEquals(ledger.getLedgersInfoAsList().size(), 2);
-
- Position p = ledger.getLastConfirmedEntry(); // position at end of second ledger
-
- ledger.addEntry("entry-blah".getBytes());
- Assert.assertEquals(ledger.getLedgersInfoAsList().size(), 3);
-
- PositionImpl firstUnoffloaded = (PositionImpl)ledger.offloadPrefix(p);
-
- // only the first ledger should have been offloaded
- Assert.assertEquals(ledger.getLedgersInfoAsList().size(), 3);
- Assert.assertEquals(offloader.offloadedLedgers().size(), 1);
- Assert.assertTrue(offloader.offloadedLedgers().contains(ledger.getLedgersInfoAsList().get(0).getLedgerId()));
- Assert.assertTrue(ledger.getLedgersInfoAsList().get(0).getOffloadContext().getComplete());
- Assert.assertEquals(ledger.getLedgersInfoAsList().stream()
- .filter(e -> e.getOffloadContext().getComplete()).count(), 1);
- Assert.assertEquals(firstUnoffloaded.getLedgerId(), ledger.getLedgersInfoAsList().get(1).getLedgerId());
- Assert.assertEquals(firstUnoffloaded.getEntryId(), 0);
-
- // offload again, with the position in the third ledger
- PositionImpl firstUnoffloaded2 = (PositionImpl)ledger.offloadPrefix(ledger.getLastConfirmedEntry());
- Assert.assertEquals(ledger.getLedgersInfoAsList().size(), 3);
- Assert.assertEquals(offloader.offloadedLedgers().size(), 2);
- Assert.assertTrue(offloader.offloadedLedgers().contains(ledger.getLedgersInfoAsList().get(0).getLedgerId()));
- Assert.assertTrue(offloader.offloadedLedgers().contains(ledger.getLedgersInfoAsList().get(1).getLedgerId()));
- Assert.assertTrue(ledger.getLedgersInfoAsList().get(0).getOffloadContext().getComplete());
- Assert.assertTrue(ledger.getLedgersInfoAsList().get(1).getOffloadContext().getComplete());
- Assert.assertEquals(ledger.getLedgersInfoAsList().stream()
- .filter(e -> e.getOffloadContext().getComplete()).count(), 2);
- Assert.assertEquals(firstUnoffloaded2.getLedgerId(), ledger.getLedgersInfoAsList().get(2).getLedgerId());
- }
-
- @Test
- public void testTrimOccursDuringOffload() throws Exception {
- CountDownLatch offloadStarted = new CountDownLatch(1);
- CompletableFuture<Void> blocker = new CompletableFuture<>();
- MockLedgerOffloader offloader = new MockLedgerOffloader() {
- @Override
- public CompletableFuture<Void> offload(ReadHandle ledger,
- UUID uuid,
- Map<String, String> extraMetadata) {
- offloadStarted.countDown();
- return blocker.thenCompose((f) -> super.offload(ledger, uuid, extraMetadata));
- }
- };
-
- ManagedLedgerConfig config = new ManagedLedgerConfig();
- config.setMaxEntriesPerLedger(10);
- config.setMinimumRolloverTime(0, TimeUnit.SECONDS);
- config.setRetentionTime(0, TimeUnit.MINUTES);
- config.setLedgerOffloader(offloader);
- ManagedLedgerImpl ledger = (ManagedLedgerImpl)factory.open("my_test_ledger", config);
- ManagedCursor cursor = ledger.openCursor("foobar");
-
- // Create 3 ledgers, saving position at start of each
- for (int i = 0; i < 21; i++) {
- String content = "entry-" + i;
- ledger.addEntry(content.getBytes());
- }
- Assert.assertEquals(ledger.getLedgersInfoAsList().size(), 3);
-
- PositionImpl startOfSecondLedger = PositionImpl.get(ledger.getLedgersInfoAsList().get(1).getLedgerId(), 0);
- PositionImpl startOfThirdLedger = PositionImpl.get(ledger.getLedgersInfoAsList().get(2).getLedgerId(), 0);
-
- // trigger an offload which should offload the first two ledgers
- OffloadCallbackPromise cbPromise = new OffloadCallbackPromise();
- ledger.asyncOffloadPrefix(startOfThirdLedger, cbPromise, null);
- offloadStarted.await();
-
- // trim first ledger
- cursor.markDelete(startOfSecondLedger, new HashMap<>());
- assertEventuallyTrue(() -> ledger.getLedgersInfoAsList().size() == 2);
- Assert.assertEquals(ledger.getLedgersInfoAsList().stream()
- .filter(e -> e.getOffloadContext().getComplete()).count(), 0);
-
- // complete offloading
- blocker.complete(null);
- cbPromise.get();
-
- Assert.assertEquals(ledger.getLedgersInfoAsList().size(), 2);
- Assert.assertEquals(ledger.getLedgersInfoAsList().stream()
- .filter(e -> e.getOffloadContext().getComplete()).count(), 1);
- Assert.assertTrue(ledger.getLedgersInfoAsList().get(0).getOffloadContext().getComplete());
- Assert.assertEquals(offloader.offloadedLedgers().size(), 1);
- Assert.assertTrue(offloader.offloadedLedgers().contains(ledger.getLedgersInfoAsList().get(0).getLedgerId()));
- }
-
- @Test
- public void testTrimOccursDuringOffloadLedgerDeletedBeforeOffload() throws Exception {
- CountDownLatch offloadStarted = new CountDownLatch(1);
- CompletableFuture<Long> blocker = new CompletableFuture<>();
- MockLedgerOffloader offloader = new MockLedgerOffloader() {
- @Override
- public CompletableFuture<Void> offload(ReadHandle ledger,
- UUID uuid,
- Map<String, String> extraMetadata) {
- offloadStarted.countDown();
- return blocker.thenCompose(
- (trimmedLedger) -> {
- if (trimmedLedger == ledger.getId()) {
- CompletableFuture<Void> future = new CompletableFuture<>();
- future.completeExceptionally(new BKException.BKNoSuchLedgerExistsException());
- return future;
- } else {
- return super.offload(ledger, uuid, extraMetadata);
- }
- });
- }
- };
-
- ManagedLedgerConfig config = new ManagedLedgerConfig();
- config.setMaxEntriesPerLedger(10);
- config.setMinimumRolloverTime(0, TimeUnit.SECONDS);
- config.setRetentionTime(0, TimeUnit.MINUTES);
- config.setLedgerOffloader(offloader);
- ManagedLedgerImpl ledger = (ManagedLedgerImpl)factory.open("my_test_ledger", config);
- ManagedCursor cursor = ledger.openCursor("foobar");
-
- for (int i = 0; i < 21; i++) {
- String content = "entry-" + i;
- ledger.addEntry(content.getBytes());
- }
- Assert.assertEquals(ledger.getLedgersInfoAsList().size(), 3);
-
- PositionImpl startOfSecondLedger = PositionImpl.get(ledger.getLedgersInfoAsList().get(1).getLedgerId(), 0);
- PositionImpl startOfThirdLedger = PositionImpl.get(ledger.getLedgersInfoAsList().get(2).getLedgerId(), 0);
-
- // trigger an offload which should offload the first two ledgers
- OffloadCallbackPromise cbPromise = new OffloadCallbackPromise();
- ledger.asyncOffloadPrefix(startOfThirdLedger, cbPromise, null);
- offloadStarted.await();
-
- // trim first ledger
- long trimmedLedger = ledger.getLedgersInfoAsList().get(0).getLedgerId();
- cursor.markDelete(startOfSecondLedger, new HashMap<>());
- assertEventuallyTrue(() -> ledger.getLedgersInfoAsList().size() == 2);
- Assert.assertEquals(ledger.getLedgersInfoAsList().stream()
- .filter(e -> e.getLedgerId() == trimmedLedger).count(), 0);
- Assert.assertEquals(ledger.getLedgersInfoAsList().stream()
- .filter(e -> e.getOffloadContext().getComplete()).count(), 0);
-
- // complete offloading
- blocker.complete(trimmedLedger);
- cbPromise.get();
-
- Assert.assertEquals(ledger.getLedgersInfoAsList().size(), 2);
- Assert.assertEquals(ledger.getLedgersInfoAsList().stream()
- .filter(e -> e.getOffloadContext().getComplete()).count(), 1);
- Assert.assertTrue(ledger.getLedgersInfoAsList().get(0).getOffloadContext().getComplete());
- Assert.assertEquals(offloader.offloadedLedgers().size(), 1);
- Assert.assertTrue(offloader.offloadedLedgers().contains(ledger.getLedgersInfoAsList().get(0).getLedgerId()));
- }
-
- @Test
- public void testOffloadClosedManagedLedger() throws Exception {
- MockLedgerOffloader offloader = new MockLedgerOffloader();
- ManagedLedgerConfig config = new ManagedLedgerConfig();
- config.setMaxEntriesPerLedger(10);
- config.setMinimumRolloverTime(0, TimeUnit.SECONDS);
- config.setRetentionTime(10, TimeUnit.MINUTES);
- config.setLedgerOffloader(offloader);
- ManagedLedgerImpl ledger = (ManagedLedgerImpl)factory.open("my_test_ledger", config);
-
- for (int i = 0; i < 21; i++) {
- String content = "entry-" + i;
- ledger.addEntry(content.getBytes());
- }
-
- Position p = ledger.getLastConfirmedEntry();
- ledger.close();
-
- try {
- ledger.offloadPrefix(p);
- Assert.fail("Should fail because ML is closed");
- } catch (ManagedLedgerException.ManagedLedgerAlreadyClosedException e) {
- // expected
- }
-
- Assert.assertEquals(ledger.getLedgersInfoAsList().size(), 3);
- Assert.assertEquals(ledger.getLedgersInfoAsList().stream()
- .filter(e -> e.getOffloadContext().getComplete()).count(), 0);
- Assert.assertEquals(offloader.offloadedLedgers().size(), 0);
- }
-
- @Test
- public void testOffloadSamePositionTwice() throws Exception {
- MockLedgerOffloader offloader = new MockLedgerOffloader();
- ManagedLedgerConfig config = new ManagedLedgerConfig();
- config.setMaxEntriesPerLedger(10);
- config.setMinimumRolloverTime(0, TimeUnit.SECONDS);
- config.setRetentionTime(10, TimeUnit.MINUTES);
- config.setLedgerOffloader(offloader);
- ManagedLedgerImpl ledger = (ManagedLedgerImpl)factory.open("my_test_ledger", config);
-
- int i = 0;
- for (; i < 25; i++) {
- String content = "entry-" + i;
- ledger.addEntry(content.getBytes());
- }
- Assert.assertEquals(ledger.getLedgersInfoAsList().size(), 3);
-
- ledger.offloadPrefix(ledger.getLastConfirmedEntry());
-
- Assert.assertEquals(ledger.getLedgersInfoAsList().size(), 3);
- Assert.assertEquals(ledger.getLedgersInfoAsList().stream()
- .filter(e -> e.getOffloadContext().getComplete())
- .map(e -> e.getLedgerId()).collect(Collectors.toSet()),
- offloader.offloadedLedgers());
-
- ledger.offloadPrefix(ledger.getLastConfirmedEntry());
-
- Assert.assertEquals(ledger.getLedgersInfoAsList().size(), 3);
- Assert.assertEquals(ledger.getLedgersInfoAsList().stream()
- .filter(e -> e.getOffloadContext().getComplete())
- .map(e -> e.getLedgerId()).collect(Collectors.toSet()),
- offloader.offloadedLedgers());
-
- }
-
- public void offloadThreeOneFails(int failIndex) throws Exception {
- CompletableFuture<Set<Long>> promise = new CompletableFuture<>();
- MockLedgerOffloader offloader = new ErroringMockLedgerOffloader(promise);
- ManagedLedgerConfig config = new ManagedLedgerConfig();
- config.setMaxEntriesPerLedger(10);
- config.setMinimumRolloverTime(0, TimeUnit.SECONDS);
- config.setRetentionTime(10, TimeUnit.MINUTES);
- config.setLedgerOffloader(offloader);
- ManagedLedgerImpl ledger = (ManagedLedgerImpl)factory.open("my_test_ledger", config);
-
- int i = 0;
- for (; i < 35; i++) {
- String content = "entry-" + i;
- ledger.addEntry(content.getBytes());
- }
- Assert.assertEquals(ledger.getLedgersInfoAsList().size(), 4);
-
- // mark ledgers to fail
- promise.complete(ImmutableSet.of(ledger.getLedgersInfoAsList().get(failIndex).getLedgerId()));
-
- try {
- ledger.offloadPrefix(ledger.getLastConfirmedEntry());
- } catch (ManagedLedgerException e) {
- Assert.assertEquals(e.getCause().getClass(), CompletionException.class);
- }
-
- Assert.assertEquals(ledger.getLedgersInfoAsList().size(), 4);
- Assert.assertEquals(ledger.getLedgersInfoAsList().stream()
- .filter(e -> e.getOffloadContext().getComplete())
- .map(e -> e.getLedgerId()).collect(Collectors.toSet()),
- offloader.offloadedLedgers());
- Assert.assertEquals(ledger.getLedgersInfoAsList().stream()
- .filter(e -> e.getOffloadContext().getComplete()).count(), 2);
- Assert.assertFalse(ledger.getLedgersInfoAsList().get(failIndex).getOffloadContext().getComplete());
- }
-
- @Test
- public void testOffloadThreeFirstFails() throws Exception {
- offloadThreeOneFails(0);
- }
-
- @Test
- public void testOffloadThreeSecondFails() throws Exception {
- offloadThreeOneFails(1);
- }
-
- @Test
- public void testOffloadThreeThirdFails() throws Exception {
- offloadThreeOneFails(2);
- }
-
- @Test
- public void testOffloadNewML() throws Exception {
- MockLedgerOffloader offloader = new MockLedgerOffloader();
- ManagedLedgerConfig config = new ManagedLedgerConfig();
- config.setMaxEntriesPerLedger(10);
- config.setMinimumRolloverTime(0, TimeUnit.SECONDS);
- config.setRetentionTime(10, TimeUnit.MINUTES);
- config.setLedgerOffloader(offloader);
- ManagedLedgerImpl ledger = (ManagedLedgerImpl)factory.open("my_test_ledger", config);
-
- try {
- ledger.offloadPrefix(ledger.getLastConfirmedEntry());
- } catch (ManagedLedgerException.InvalidCursorPositionException e) {
- // expected
- }
- // add one entry and try again
- ledger.addEntry("foobar".getBytes());
-
- Position p = ledger.getLastConfirmedEntry();
- Assert.assertEquals(p, ledger.offloadPrefix(ledger.getLastConfirmedEntry()));
- Assert.assertEquals(ledger.getLedgersInfoAsList().size(), 1);
- Assert.assertEquals(offloader.offloadedLedgers().size(), 0);
- }
-
- @Test
- public void testOffloadConflict() throws Exception {
- Set<Pair<Long, UUID>> deleted = ConcurrentHashMap.newKeySet();
- CompletableFuture<Set<Long>> errorLedgers = new CompletableFuture<>();
- Set<Pair<Long, UUID>> failedOffloads = ConcurrentHashMap.newKeySet();
-
- MockLedgerOffloader offloader = new MockLedgerOffloader() {
- @Override
- public CompletableFuture<Void> offload(ReadHandle ledger,
- UUID uuid,
- Map<String, String> extraMetadata) {
- return errorLedgers.thenCompose(
- (errors) -> {
- if (errors.remove(ledger.getId())) {
- failedOffloads.add(Pair.of(ledger.getId(), uuid));
- CompletableFuture<Void> future = new CompletableFuture<>();
- future.completeExceptionally(new Exception("Some kind of error"));
- return future;
- } else {
- return super.offload(ledger, uuid, extraMetadata);
- }
- });
- }
-
- @Override
- public CompletableFuture<Void> deleteOffloaded(long ledgerId, UUID uuid,
- Map<String, String> offloadDriverMetadata) {
- deleted.add(Pair.of(ledgerId, uuid));
- return super.deleteOffloaded(ledgerId, uuid, offloadDriverMetadata);
- }
- };
- ManagedLedgerConfig config = new ManagedLedgerConfig();
- config.setMaxEntriesPerLedger(10);
- config.setMinimumRolloverTime(0, TimeUnit.SECONDS);
- config.setRetentionTime(10, TimeUnit.MINUTES);
- config.setLedgerOffloader(offloader);
- ManagedLedgerImpl ledger = (ManagedLedgerImpl)factory.open("my_test_ledger", config);
-
- for (int i = 0; i < 15; i++) {
- String content = "entry-" + i;
- ledger.addEntry(content.getBytes());
- }
-
- Set<Long> errorSet = ConcurrentHashMap.newKeySet();
- errorSet.add(ledger.getLedgersInfoAsList().get(0).getLedgerId());
- errorLedgers.complete(errorSet);
-
- try {
- ledger.offloadPrefix(ledger.getLastConfirmedEntry());
- } catch (ManagedLedgerException e) {
- // expected
- }
- Assert.assertTrue(errorSet.isEmpty());
- Assert.assertEquals(failedOffloads.size(), 1);
- Assert.assertEquals(deleted.size(), 0);
-
- long expectedFailedLedger = ledger.getLedgersInfoAsList().get(0).getLedgerId();
- UUID expectedFailedUUID = new UUID(ledger.getLedgersInfoAsList().get(0).getOffloadContext().getUidMsb(),
- ledger.getLedgersInfoAsList().get(0).getOffloadContext().getUidLsb());
- Assert.assertEquals(failedOffloads.stream().findFirst().get(),
- Pair.of(expectedFailedLedger, expectedFailedUUID));
- Assert.assertFalse(ledger.getLedgersInfoAsList().get(0).getOffloadContext().getComplete());
-
- // try offload again
- ledger.offloadPrefix(ledger.getLastConfirmedEntry());
-
- Assert.assertEquals(failedOffloads.size(), 1);
- Assert.assertEquals(deleted.size(), 1);
- Assert.assertEquals(deleted.stream().findFirst().get(),
- Pair.of(expectedFailedLedger, expectedFailedUUID));
- UUID successUUID = new UUID(ledger.getLedgersInfoAsList().get(0).getOffloadContext().getUidMsb(),
- ledger.getLedgersInfoAsList().get(0).getOffloadContext().getUidLsb());
- Assert.assertFalse(successUUID.equals(expectedFailedUUID));
- Assert.assertTrue(ledger.getLedgersInfoAsList().get(0).getOffloadContext().getComplete());
- }
-
- @Test
- public void testOffloadDelete() throws Exception {
- Set<Pair<Long, UUID>> deleted = ConcurrentHashMap.newKeySet();
- CompletableFuture<Set<Long>> errorLedgers = new CompletableFuture<>();
- Set<Pair<Long, UUID>> failedOffloads = ConcurrentHashMap.newKeySet();
-
- MockLedgerOffloader offloader = new MockLedgerOffloader();
- ManagedLedgerConfig config = new ManagedLedgerConfig();
- config.setMaxEntriesPerLedger(10);
- config.setMinimumRolloverTime(0, TimeUnit.SECONDS);
- config.setRetentionTime(0, TimeUnit.MINUTES);
- config.setLedgerOffloader(offloader);
- ManagedLedgerImpl ledger = (ManagedLedgerImpl)factory.open("my_test_ledger", config);
- ManagedCursor cursor = ledger.openCursor("foobar");
- for (int i = 0; i < 15; i++) {
- String content = "entry-" + i;
- ledger.addEntry(content.getBytes());
- }
-
- Assert.assertEquals(ledger.getLedgersInfoAsList().size(), 2);
- ledger.offloadPrefix(ledger.getLastConfirmedEntry());
- Assert.assertEquals(ledger.getLedgersInfoAsList().size(), 2);
-
- Assert.assertEquals(ledger.getLedgersInfoAsList().stream()
- .filter(e -> e.getOffloadContext().getComplete()).count(), 1);
- Assert.assertTrue(ledger.getLedgersInfoAsList().get(0).getOffloadContext().getComplete());
- long firstLedger = ledger.getLedgersInfoAsList().get(0).getLedgerId();
- long secondLedger = ledger.getLedgersInfoAsList().get(1).getLedgerId();
-
- cursor.markDelete(ledger.getLastConfirmedEntry());
- assertEventuallyTrue(() -> ledger.getLedgersInfoAsList().size() == 1);
- Assert.assertEquals(ledger.getLedgersInfoAsList().get(0).getLedgerId(), secondLedger);
-
- assertEventuallyTrue(() -> offloader.deletedOffloads().contains(firstLedger));
- }
-
- @Test
- public void testOffloadDeleteIncomplete() throws Exception {
- Set<Pair<Long, UUID>> deleted = ConcurrentHashMap.newKeySet();
- CompletableFuture<Set<Long>> errorLedgers = new CompletableFuture<>();
- Set<Pair<Long, UUID>> failedOffloads = ConcurrentHashMap.newKeySet();
-
- MockLedgerOffloader offloader = new MockLedgerOffloader() {
- @Override
- public CompletableFuture<Void> offload(ReadHandle ledger,
- UUID uuid,
- Map<String, String> extraMetadata) {
- return super.offload(ledger, uuid, extraMetadata)
- .thenCompose((res) -> {
- CompletableFuture<Void> f = new CompletableFuture<>();
- f.completeExceptionally(new Exception("Fail after offload occurred"));
- return f;
- });
- }
- };
- ManagedLedgerConfig config = new ManagedLedgerConfig();
- config.setMaxEntriesPerLedger(10);
- config.setMinimumRolloverTime(0, TimeUnit.SECONDS);
- config.setRetentionTime(0, TimeUnit.MINUTES);
- config.setLedgerOffloader(offloader);
- ManagedLedgerImpl ledger = (ManagedLedgerImpl)factory.open("my_test_ledger", config);
- ManagedCursor cursor = ledger.openCursor("foobar");
- for (int i = 0; i < 15; i++) {
- String content = "entry-" + i;
- ledger.addEntry(content.getBytes());
- }
-
- Assert.assertEquals(ledger.getLedgersInfoAsList().size(), 2);
- try {
- ledger.offloadPrefix(ledger.getLastConfirmedEntry());
- } catch (ManagedLedgerException mle) {
- // expected
- }
-
- Assert.assertEquals(ledger.getLedgersInfoAsList().size(), 2);
-
- Assert.assertEquals(ledger.getLedgersInfoAsList().stream()
- .filter(e -> e.getOffloadContext().getComplete()).count(), 0);
- Assert.assertEquals(ledger.getLedgersInfoAsList().stream()
- .filter(e -> e.getOffloadContext().hasUidMsb()).count(), 1);
- Assert.assertTrue(ledger.getLedgersInfoAsList().get(0).getOffloadContext().hasUidMsb());
-
- long firstLedger = ledger.getLedgersInfoAsList().get(0).getLedgerId();
- long secondLedger = ledger.getLedgersInfoAsList().get(1).getLedgerId();
-
- cursor.markDelete(ledger.getLastConfirmedEntry());
- assertEventuallyTrue(() -> ledger.getLedgersInfoAsList().size() == 1);
- Assert.assertEquals(ledger.getLedgersInfoAsList().get(0).getLedgerId(), secondLedger);
-
- assertEventuallyTrue(() -> offloader.deletedOffloads().contains(firstLedger));
- }
-
- @Test
- public void testDontOffloadEmpty() throws Exception {
- MockLedgerOffloader offloader = new MockLedgerOffloader();
- ManagedLedgerConfig config = new ManagedLedgerConfig();
- config.setMaxEntriesPerLedger(10);
- config.setMinimumRolloverTime(0, TimeUnit.SECONDS);
- config.setRetentionTime(10, TimeUnit.MINUTES);
- config.setLedgerOffloader(offloader);
- ManagedLedgerImpl ledger = (ManagedLedgerImpl)factory.open("my_test_ledger", config);
-
- int i = 0;
- for (; i < 35; i++) {
- String content = "entry-" + i;
- ledger.addEntry(content.getBytes());
- }
- Assert.assertEquals(ledger.getLedgersInfoAsList().size(), 4);
-
- long firstLedgerId = ledger.getLedgersInfoAsList().get(0).getLedgerId();
- long secondLedgerId = ledger.getLedgersInfoAsList().get(1).getLedgerId();
- long thirdLedgerId = ledger.getLedgersInfoAsList().get(2).getLedgerId();
- long fourthLedgerId = ledger.getLedgersInfoAsList().get(3).getLedgerId();
-
- // make an ledger empty
- Field ledgersField = ledger.getClass().getDeclaredField("ledgers");
- ledgersField.setAccessible(true);
- Map<Long, LedgerInfo> ledgers = (Map<Long,LedgerInfo>)ledgersField.get(ledger);
- ledgers.put(secondLedgerId,
- ledgers.get(secondLedgerId).toBuilder().setEntries(0).setSize(0).build());
-
- PositionImpl firstUnoffloaded = (PositionImpl)ledger.offloadPrefix(ledger.getLastConfirmedEntry());
- Assert.assertEquals(firstUnoffloaded.getLedgerId(), fourthLedgerId);
- Assert.assertEquals(firstUnoffloaded.getEntryId(), 0);
-
- Assert.assertEquals(ledger.getLedgersInfoAsList().size(), 4);
- Assert.assertEquals(ledger.getLedgersInfoAsList().stream()
- .filter(e -> e.getOffloadContext().getComplete())
- .map(e -> e.getLedgerId()).collect(Collectors.toSet()),
- offloader.offloadedLedgers());
- Assert.assertEquals(offloader.offloadedLedgers(), ImmutableSet.of(firstLedgerId, thirdLedgerId));
- }
-
- private static byte[] buildEntry(int size, String pattern) {
- byte[] entry = new byte[size];
- byte[] patternBytes = pattern.getBytes();
-
- for (int i = 0; i < entry.length; i++) {
- entry[i] = patternBytes[i % patternBytes.length];
- }
- return entry;
- }
-
- @Test
- public void testAutoTriggerOffload() throws Exception {
- MockLedgerOffloader offloader = new MockLedgerOffloader();
- ManagedLedgerConfig config = new ManagedLedgerConfig();
- config.setMaxEntriesPerLedger(10);
- config.setOffloadAutoTriggerSizeThresholdBytes(100);
- config.setRetentionTime(10, TimeUnit.MINUTES);
- config.setLedgerOffloader(offloader);
-
- ManagedLedgerImpl ledger = (ManagedLedgerImpl)factory.open("my_test_ledger", config);
-
- // Ledger will roll twice, offload will run on first ledger after second closed
- for (int i = 0; i < 25; i++) {
- ledger.addEntry(buildEntry(10, "entry-" + i));
- }
-
- Assert.assertEquals(ledger.getLedgersInfoAsList().size(), 3);
-
- // offload should eventually be triggered
- assertEventuallyTrue(() -> offloader.offloadedLedgers().size() == 1);
- Assert.assertEquals(offloader.offloadedLedgers(),
- ImmutableSet.of(ledger.getLedgersInfoAsList().get(0).getLedgerId()));
- }
-
- @Test
- public void manualTriggerWhileAutoInProgress() throws Exception {
- CompletableFuture<Void> slowOffload = new CompletableFuture<>();
- CountDownLatch offloadRunning = new CountDownLatch(1);
- MockLedgerOffloader offloader = new MockLedgerOffloader() {
- @Override
- public CompletableFuture<Void> offload(ReadHandle ledger,
- UUID uuid,
- Map<String, String> extraMetadata) {
- offloadRunning.countDown();
- return slowOffload.thenCompose((res) -> super.offload(ledger, uuid, extraMetadata));
- }
- };
-
- ManagedLedgerConfig config = new ManagedLedgerConfig();
- config.setMaxEntriesPerLedger(10);
- config.setOffloadAutoTriggerSizeThresholdBytes(100);
- config.setRetentionTime(10, TimeUnit.MINUTES);
- config.setLedgerOffloader(offloader);
-
- ManagedLedgerImpl ledger = (ManagedLedgerImpl)factory.open("my_test_ledger", config);
-
- // Ledger will roll twice, offload will run on first ledger after second closed
- for (int i = 0; i < 25; i++) {
- ledger.addEntry(buildEntry(10, "entry-" + i));
- }
- offloadRunning.await();
-
- for (int i = 0; i < 20; i++) {
- ledger.addEntry(buildEntry(10, "entry-" + i));
- }
- Position p = ledger.addEntry(buildEntry(10, "last-entry"));
-
- try {
- ledger.offloadPrefix(p);
- Assert.fail("Shouldn't have succeeded");
- } catch (ManagedLedgerException.OffloadInProgressException e) {
- // expected
- }
-
- slowOffload.complete(null);
-
- // eventually all over threshold will be offloaded
- assertEventuallyTrue(() -> offloader.offloadedLedgers().size() == 3);
- Assert.assertEquals(offloader.offloadedLedgers(),
- ImmutableSet.of(ledger.getLedgersInfoAsList().get(0).getLedgerId(),
- ledger.getLedgersInfoAsList().get(1).getLedgerId(),
- ledger.getLedgersInfoAsList().get(2).getLedgerId()));
-
- // then a manual offload can run and offload the one ledger under the threshold
- ledger.offloadPrefix(p);
-
- Assert.assertEquals(offloader.offloadedLedgers().size(), 4);
- Assert.assertEquals(offloader.offloadedLedgers(),
- ImmutableSet.of(ledger.getLedgersInfoAsList().get(0).getLedgerId(),
- ledger.getLedgersInfoAsList().get(1).getLedgerId(),
- ledger.getLedgersInfoAsList().get(2).getLedgerId(),
- ledger.getLedgersInfoAsList().get(3).getLedgerId()));
- }
-
- @Test
- public void autoTriggerWhileManualInProgress() throws Exception {
- CompletableFuture<Void> slowOffload = new CompletableFuture<>();
- CountDownLatch offloadRunning = new CountDownLatch(1);
- MockLedgerOffloader offloader = new MockLedgerOffloader() {
- @Override
- public CompletableFuture<Void> offload(ReadHandle ledger,
- UUID uuid,
- Map<String, String> extraMetadata) {
- offloadRunning.countDown();
- return slowOffload.thenCompose((res) -> super.offload(ledger, uuid, extraMetadata));
- }
- };
-
- ManagedLedgerConfig config = new ManagedLedgerConfig();
- config.setMaxEntriesPerLedger(10);
- config.setOffloadAutoTriggerSizeThresholdBytes(100);
- config.setRetentionTime(10, TimeUnit.MINUTES);
- config.setLedgerOffloader(offloader);
-
- ManagedLedgerImpl ledger = (ManagedLedgerImpl)factory.open("my_test_ledger", config);
-
- // Ledger rolls once, threshold not hit so auto shouldn't run
- for (int i = 0; i < 14; i++) {
- ledger.addEntry(buildEntry(10, "entry-" + i));
- }
- Position p = ledger.addEntry(buildEntry(10, "trigger-entry"));
-
- OffloadCallbackPromise cbPromise = new OffloadCallbackPromise();
- ledger.asyncOffloadPrefix(p, cbPromise, null);
- offloadRunning.await();
-
- // add enough entries to roll the ledger a couple of times and trigger some offloads
- for (int i = 0; i < 20; i++) {
- ledger.addEntry(buildEntry(10, "entry-" + i));
- }
-
- // allow the manual offload to complete
- slowOffload.complete(null);
-
- Assert.assertEquals(cbPromise.join(),
- PositionImpl.get(ledger.getLedgersInfoAsList().get(1).getLedgerId(), 0));
-
- // auto trigger should eventually offload everything else over threshold
- assertEventuallyTrue(() -> offloader.offloadedLedgers().size() == 2);
- Assert.assertEquals(offloader.offloadedLedgers(),
- ImmutableSet.of(ledger.getLedgersInfoAsList().get(0).getLedgerId(),
- ledger.getLedgersInfoAsList().get(1).getLedgerId()));
- }
-
- @Test
- public void multipleAutoTriggers() throws Exception {
- CompletableFuture<Void> slowOffload = new CompletableFuture<>();
- CountDownLatch offloadRunning = new CountDownLatch(1);
- MockLedgerOffloader offloader = new MockLedgerOffloader() {
- @Override
- public CompletableFuture<Void> offload(ReadHandle ledger,
- UUID uuid,
- Map<String, String> extraMetadata) {
- offloadRunning.countDown();
- return slowOffload.thenCompose((res) -> super.offload(ledger, uuid, extraMetadata));
- }
- };
-
- ManagedLedgerConfig config = new ManagedLedgerConfig();
- config.setMaxEntriesPerLedger(10);
- config.setOffloadAutoTriggerSizeThresholdBytes(100);
- config.setRetentionTime(10, TimeUnit.MINUTES);
- config.setLedgerOffloader(offloader);
-
- ManagedLedgerImpl ledger = (ManagedLedgerImpl)factory.open("my_test_ledger", config);
-
- // Ledger will roll twice, offload will run on first ledger after second closed
- for (int i = 0; i < 25; i++) {
- ledger.addEntry(buildEntry(10, "entry-" + i));
- }
- offloadRunning.await();
-
- // trigger a bunch more rolls. Eventually there will be 5 ledgers.
- // first 3 should be offloaded, 4th is 100bytes, 5th is 0 bytes.
- // 4th and 5th sum to 100 bytes so they're just at edge of threshold
- for (int i = 0; i < 20; i++) {
- ledger.addEntry(buildEntry(10, "entry-" + i));
- }
-
- // allow the first offload to continue
- slowOffload.complete(null);
-
- assertEventuallyTrue(() -> offloader.offloadedLedgers().size() == 3);
- Assert.assertEquals(offloader.offloadedLedgers(),
- ImmutableSet.of(ledger.getLedgersInfoAsList().get(0).getLedgerId(),
- ledger.getLedgersInfoAsList().get(1).getLedgerId(),
- ledger.getLedgersInfoAsList().get(2).getLedgerId()));
- }
-
- @Test
- public void offloadAsSoonAsClosed() throws Exception {
-
- MockLedgerOffloader offloader = new MockLedgerOffloader();
- ManagedLedgerConfig config = new ManagedLedgerConfig();
- config.setMaxEntriesPerLedger(10);
- config.setOffloadAutoTriggerSizeThresholdBytes(0);
- config.setRetentionTime(10, TimeUnit.MINUTES);
- config.setLedgerOffloader(offloader);
-
- ManagedLedgerImpl ledger = (ManagedLedgerImpl)factory.open("my_test_ledger", config);
-
- for (int i = 0; i < 11; i++) {
- ledger.addEntry(buildEntry(10, "entry-" + i));
- }
-
- assertEventuallyTrue(() -> offloader.offloadedLedgers().size() == 1);
- Assert.assertEquals(offloader.offloadedLedgers(),
- ImmutableSet.of(ledger.getLedgersInfoAsList().get(0).getLedgerId()));
-
- for (int i = 0; i < 10; i++) {
- ledger.addEntry(buildEntry(10, "entry-" + i));
- }
-
- assertEventuallyTrue(() -> offloader.offloadedLedgers().size() == 2);
- Assert.assertEquals(offloader.offloadedLedgers(),
- ImmutableSet.of(ledger.getLedgersInfoAsList().get(0).getLedgerId(),
- ledger.getLedgersInfoAsList().get(1).getLedgerId()));
- }
-
-
- static void assertEventuallyTrue(BooleanSupplier predicate) throws Exception {
- // wait up to 3 seconds
- for (int i = 0; i < 30 && !predicate.getAsBoolean(); i++) {
- Thread.sleep(100);
- }
- Assert.assertTrue(predicate.getAsBoolean());
- }
-
- static class OffloadCallbackPromise extends CompletableFuture<Position> implements OffloadCallback {
- @Override
- public void offloadComplete(Position pos, Object ctx) {
- complete(pos);
- }
-
- @Override
- public void offloadFailed(ManagedLedgerException exception, Object ctx) {
- completeExceptionally(exception);
- }
- }
-
- static class MockLedgerOffloader implements LedgerOffloader {
- ConcurrentHashMap<Long, UUID> offloads = new ConcurrentHashMap<Long, UUID>();
- ConcurrentHashMap<Long, UUID> deletes = new ConcurrentHashMap<Long, UUID>();
-
- Set<Long> offloadedLedgers() {
- return offloads.keySet();
- }
-
- Set<Long> deletedOffloads() {
- return deletes.keySet();
- }
-
- @Override
- public String getOffloadDriverName() {
- return "mock";
- }
-
- @Override
- public CompletableFuture<Void> offload(ReadHandle ledger,
- UUID uuid,
- Map<String, String> extraMetadata) {
- CompletableFuture<Void> promise = new CompletableFuture<>();
- if (offloads.putIfAbsent(ledger.getId(), uuid) == null) {
- promise.complete(null);
- } else {
- promise.completeExceptionally(new Exception("Already exists exception"));
- }
- return promise;
- }
-
- @Override
- public CompletableFuture<ReadHandle> readOffloaded(long ledgerId, UUID uuid,
- Map<String, String> offloadDriverMetadata) {
- CompletableFuture<ReadHandle> promise = new CompletableFuture<>();
- promise.completeExceptionally(new UnsupportedOperationException());
- return promise;
- }
-
- @Override
- public CompletableFuture<Void> deleteOffloaded(long ledgerId, UUID uuid,
- Map<String, String> offloadDriverMetadata) {
- CompletableFuture<Void> promise = new CompletableFuture<>();
- if (offloads.remove(ledgerId, uuid)) {
- deletes.put(ledgerId, uuid);
- promise.complete(null);
- } else {
- promise.completeExceptionally(new Exception("Not found"));
- }
- return promise;
- };
- }
-
- static class ErroringMockLedgerOffloader extends MockLedgerOffloader {
- CompletableFuture<Set<Long>> errorLedgers = new CompletableFuture<>();
-
- ErroringMockLedgerOffloader(CompletableFuture<Set<Long>> errorLedgers) {
- this.errorLedgers = errorLedgers;
- }
-
- @Override
- public CompletableFuture<Void> offload(ReadHandle ledger,
- UUID uuid,
- Map<String, String> extraMetadata) {
- return errorLedgers.thenCompose(
- (errors) -> {
- if (errors.contains(ledger.getId())) {
- CompletableFuture<Void> future = new CompletableFuture<>();
- future.completeExceptionally(new Exception("Some kind of error"));
- return future;
- } else {
- return super.offload(ledger, uuid, extraMetadata);
- }
- });
- }
- }
-}
diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/PositionTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/PositionTest.java
deleted file mode 100644
index fe2fcdaf18..0000000000
--- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/PositionTest.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.bookkeeper.mledger.impl;
-
-import static org.testng.Assert.assertEquals;
-import static org.testng.Assert.assertFalse;
-
-import org.apache.bookkeeper.mledger.proto.MLDataFormats.PositionInfo;
-import org.testng.annotations.Test;
-
-public class PositionTest {
- @Test(expectedExceptions = NullPointerException.class)
- public void nullParam() {
- new PositionImpl((PositionInfo) null);
- }
-
- @Test
- public void simpleTest() {
- PositionImpl pos = new PositionImpl(1, 2);
- assertEquals(pos.getLedgerId(), 1);
- assertEquals(pos.getEntryId(), 2);
- assertEquals(pos, new PositionImpl(1, 2));
-
- assertFalse(pos.equals(new PositionImpl(1, 3)));
- assertFalse(pos.equals(new PositionImpl(3, 2)));
- assertFalse(pos.equals("1:2"));
- }
-
- @Test
- public void comparisons() {
- PositionImpl pos1_1 = new PositionImpl(1, 1);
- PositionImpl pos2_5 = new PositionImpl(2, 5);
- PositionImpl pos10_0 = new PositionImpl(10, 0);
- PositionImpl pos10_1 = new PositionImpl(10, 1);
-
- assertEquals(0, pos1_1.compareTo(pos1_1));
- assertEquals(-1, pos1_1.compareTo(pos2_5));
- assertEquals(-1, pos1_1.compareTo(pos10_0));
- assertEquals(-1, pos1_1.compareTo(pos10_1));
-
- assertEquals(+1, pos2_5.compareTo(pos1_1));
- assertEquals(0, pos2_5.compareTo(pos2_5));
- assertEquals(-1, pos2_5.compareTo(pos10_0));
- assertEquals(-1, pos2_5.compareTo(pos10_1));
-
- assertEquals(+1, pos10_0.compareTo(pos1_1));
- assertEquals(+1, pos10_0.compareTo(pos2_5));
- assertEquals(0, pos10_0.compareTo(pos10_0));
- assertEquals(-1, pos10_0.compareTo(pos10_1));
-
- assertEquals(+1, pos10_1.compareTo(pos1_1));
- assertEquals(+1, pos10_1.compareTo(pos2_5));
- assertEquals(+1, pos10_1.compareTo(pos10_0));
- assertEquals(0, pos10_1.compareTo(pos10_1));
- }
-
- @Test
- public void hashes() throws Exception {
- PositionImpl p1 = new PositionImpl(5, 15);
- PositionImpl p2 = new PositionImpl(PositionInfo.parseFrom(p1.getPositionInfo().toByteArray()));
- assertEquals(p2.getLedgerId(), 5);
- assertEquals(p2.getEntryId(), 15);
- assertEquals(new PositionImpl(5, 15).hashCode(), p2.hashCode());
- }
-}
diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ReadOnlyCursorTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ReadOnlyCursorTest.java
deleted file mode 100644
index bcc32751b1..0000000000
--- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/impl/ReadOnlyCursorTest.java
+++ /dev/null
@@ -1,223 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.bookkeeper.mledger.impl;
-
-import static org.testng.Assert.assertEquals;
-import static org.testng.Assert.assertFalse;
-import static org.testng.Assert.assertTrue;
-import static org.testng.Assert.fail;
-
-import java.util.List;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.bookkeeper.mledger.Entry;
-import org.apache.bookkeeper.mledger.ManagedLedger;
-import org.apache.bookkeeper.mledger.ManagedLedgerConfig;
-import org.apache.bookkeeper.mledger.ManagedLedgerException.ManagedLedgerNotFoundException;
-import org.apache.bookkeeper.mledger.Position;
-import org.apache.bookkeeper.mledger.ReadOnlyCursor;
-import org.apache.bookkeeper.test.MockedBookKeeperTestCase;
-import org.testng.annotations.Test;
-
-public class ReadOnlyCursorTest extends MockedBookKeeperTestCase {
-
- @Test
- void notFound() throws Exception {
- try {
- factory.openReadOnlyCursor("notFound", PositionImpl.earliest, new ManagedLedgerConfig());
- fail("Should have failed");
- } catch (ManagedLedgerNotFoundException e) {
- // Expected
- }
-
- factory.shutdown();
- }
-
- @Test
- void simple() throws Exception {
- ManagedLedger ledger = factory.open("simple", new ManagedLedgerConfig().setRetentionTime(1, TimeUnit.HOURS));
-
- int N = 10;
-
- for (int i = 0; i < N; i++) {
- ledger.addEntry(("entry-" + i).getBytes());
- }
-
- ReadOnlyCursor cursor = factory.openReadOnlyCursor("simple", PositionImpl.earliest, new ManagedLedgerConfig());
-
- assertEquals(cursor.getNumberOfEntries(), N);
- assertTrue(cursor.hasMoreEntries());
-
- List<Entry> entries = cursor.readEntries(N);
- assertEquals(entries.size(), N);
-
- assertEquals(cursor.getNumberOfEntries(), 0);
- assertFalse(cursor.hasMoreEntries());
- entries.forEach(Entry::release);
- cursor.close();
-
- // Ensure we can still write to ledger
- for (int i = 0; i < N; i++) {
- ledger.addEntry(("entry-" + i).getBytes());
- }
-
- // Open a new cursor
- cursor = factory.openReadOnlyCursor("simple", PositionImpl.earliest, new ManagedLedgerConfig());
-
- assertEquals(cursor.getNumberOfEntries(), 2 * N);
- assertTrue(cursor.hasMoreEntries());
-
- entries = cursor.readEntries(N);
- assertEquals(entries.size(), N);
-
- assertEquals(cursor.getNumberOfEntries(), N);
- assertTrue(cursor.hasMoreEntries());
- entries.forEach(Entry::release);
-
- entries = cursor.readEntries(N);
- assertEquals(entries.size(), N);
-
- assertEquals(cursor.getNumberOfEntries(), 0);
- assertFalse(cursor.hasMoreEntries());
- entries.forEach(Entry::release);
-
- cursor.close();
- }
-
- @Test
- void skip() throws Exception {
- ManagedLedger ledger = factory.open("skip",
- new ManagedLedgerConfig().setMaxEntriesPerLedger(2).setRetentionTime(1, TimeUnit.HOURS));
-
- int N = 10;
-
- for (int i = 0; i < N; i++) {
- ledger.addEntry(("entry-" + i).getBytes());
- }
-
- ReadOnlyCursor cursor = factory.openReadOnlyCursor("skip", PositionImpl.earliest, new ManagedLedgerConfig());
-
- assertEquals(cursor.getNumberOfEntries(), N);
- assertTrue(cursor.hasMoreEntries());
-
- cursor.skipEntries(5);
-
- assertEquals(cursor.getNumberOfEntries(), N - 5);
- assertTrue(cursor.hasMoreEntries());
-
- cursor.close();
- }
-
- @Test
- void skipAll() throws Exception {
- ManagedLedger ledger = factory.open("skip-all",
- new ManagedLedgerConfig().setMaxEntriesPerLedger(7).setRetentionTime(1, TimeUnit.HOURS));
-
- int N = 10;
-
- for (int i = 0; i < N; i++) {
- ledger.addEntry(("entry-" + i).getBytes());
- }
-
- ReadOnlyCursor cursor = factory.openReadOnlyCursor("skip-all", PositionImpl.earliest,
- new ManagedLedgerConfig());
-
- assertEquals(cursor.getNumberOfEntries(), N);
- assertTrue(cursor.hasMoreEntries());
-
- cursor.skipEntries(N);
-
- assertEquals(cursor.getNumberOfEntries(), 0);
- assertFalse(cursor.hasMoreEntries());
-
- cursor.close();
- }
-
- @Test
- void skipMultiple() throws Exception {
- ManagedLedger ledger = factory.open("skip",
- new ManagedLedgerConfig().setMaxEntriesPerLedger(7).setRetentionTime(1, TimeUnit.HOURS));
-
- int N = 30;
-
- for (int i = 0; i < N; i++) {
- ledger.addEntry(("entry-" + i).getBytes());
- }
-
- ReadOnlyCursor cursor = factory.openReadOnlyCursor("skip", PositionImpl.earliest, new ManagedLedgerConfig());
-
- assertEquals(cursor.getNumberOfEntries(), N);
- assertTrue(cursor.hasMoreEntries());
-
- cursor.skipEntries(25);
-
- assertEquals(cursor.getNumberOfEntries(), 5);
- assertTrue(cursor.hasMoreEntries());
-
- cursor.skipEntries(5);
-
- assertEquals(cursor.getNumberOfEntries(), 0);
- assertFalse(cursor.hasMoreEntries());
-
- cursor.close();
- }
-
- @Test
- void empty() throws Exception {
- factory.open("empty", new ManagedLedgerConfig().setRetentionTime(1, TimeUnit.HOURS));
-
- ReadOnlyCursor cursor = factory.openReadOnlyCursor("empty", PositionImpl.earliest, new ManagedLedgerConfig());
-
- assertEquals(cursor.getNumberOfEntries(), 0);
- assertFalse(cursor.hasMoreEntries());
-
- cursor.close();
- }
-
- @Test
- void specifyStartPosition() throws Exception {
- ManagedLedger ledger = factory.open("simple", new ManagedLedgerConfig().setRetentionTime(1, TimeUnit.HOURS));
-
- int N = 10;
-
- for (int i = 0; i < N; i++) {
- ledger.addEntry(("entry-" + i).getBytes());
- }
-
- ReadOnlyCursor cursor = factory.openReadOnlyCursor("simple", PositionImpl.earliest, new ManagedLedgerConfig());
-
- assertEquals(cursor.getNumberOfEntries(), N);
- assertTrue(cursor.hasMoreEntries());
-
- Position readPosition = cursor.getReadPosition();
- cursor = factory.openReadOnlyCursor("simple", readPosition, new ManagedLedgerConfig());
-
- assertEquals(cursor.getNumberOfEntries(), N);
- assertTrue(cursor.hasMoreEntries());
-
- cursor.skipEntries(5);
-
- Position newReadPosition = cursor.getReadPosition();
- cursor = factory.openReadOnlyCursor("simple", newReadPosition, new ManagedLedgerConfig());
-
- assertEquals(cursor.getNumberOfEntries(), N - 5);
- assertTrue(cursor.hasMoreEntries());
- }
-
-}
diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/util/CallbackMutexTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/util/CallbackMutexTest.java
deleted file mode 100644
index 599458dd26..0000000000
--- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/util/CallbackMutexTest.java
+++ /dev/null
@@ -1,126 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.bookkeeper.mledger.util;
-
-import java.util.concurrent.atomic.AtomicBoolean;
-import org.testng.Assert;
-import org.testng.annotations.Test;
-
-public class CallbackMutexTest {
-
- public final int numberOfThreads = 1000;
- public int counter = 0;
-
- @Test
- public void lock() {
-
- final CallbackMutex cbm = new CallbackMutex();
- final Account salary = new Account();
- salary.add(1000);
- // No thread competition here
- // We will test thread competition in unlock()
- new Thread(new Runnable() {
- @Override
- public void run() {
- cbm.lock();
- if (salary.value() == 1000)
- salary.add(2000);
- cbm.unlock();
- Assert.assertEquals(salary.value(), 3000);
- }
- }).start();
- }
-
- @Test(enabled = false)
- public void unlock() {
- ProtectedCode pc = new ProtectedCode();
- // Spawn many threads and start them all, all executing the same protected code
- // Not entirely sure if this models the main use case or idea.
- int i = 0;
- while (i < numberOfThreads) {
- (new Thread(pc)).start();
- i++;
- }
- try {
- // To ensure that all threads have started
- Thread.sleep(1000);
- } catch (InterruptedException e) {
- e.printStackTrace();
- }
- // All threads did start
- Assert.assertEquals(counter, numberOfThreads);
- }
-
- public class Account {
- int balance = 0;
-
- public int add(int change) {
- balance = balance + change;
- return balance;
- }
-
- public int value() {
- return balance;
- }
-
- public int set(int initial) {
- balance = initial;
- return balance;
- }
- }
-
- public class ProtectedCode implements Runnable {
-
- AtomicBoolean ab = new AtomicBoolean(false);
-
- Account salary = new Account();
- CallbackMutex cbm = new CallbackMutex();
-
- @Override
- public void run() {
- // Ensuring that all threads have started
- counter++;
-
- cbm.lock();
-
- // Protected Code
- salary.set(0);
- int raise = 0;
- if (ab.compareAndSet(false, true))
- raise = 2;
- else
- raise = -2; // Due to protection no thread will ever execute this line
- while (salary.value() < 1000000) {
- try {
- Thread.sleep(10);
- } catch (InterruptedException e) {
- e.printStackTrace();
- }
- if (salary.value() % 2 == 0)
- salary.add(raise);
- }
- Assert.assertEquals(salary.value(), 1000000);
- ab.compareAndSet(true, false);
- // End of Protected Code
-
- cbm.unlock();
- }
- }
-
-}
diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/util/RangeCacheTest.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/util/RangeCacheTest.java
deleted file mode 100644
index d1d2e5d2c6..0000000000
--- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/util/RangeCacheTest.java
+++ /dev/null
@@ -1,250 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.bookkeeper.mledger.util;
-
-import static org.testng.Assert.assertEquals;
-import static org.testng.Assert.fail;
-
-import com.google.common.collect.Lists;
-import io.netty.util.AbstractReferenceCounted;
-import io.netty.util.ReferenceCounted;
-import org.apache.commons.lang3.tuple.Pair;
-import org.testng.annotations.Test;
-
-@Test
-public class RangeCacheTest {
-
- class RefString extends AbstractReferenceCounted implements ReferenceCounted {
- final String s;
-
- RefString(String s) {
- super();
- this.s = s;
- setRefCnt(1);
- }
-
- @Override
- protected void deallocate() {
- // no-op
- }
-
- @Override
- public ReferenceCounted touch(Object hint) {
- return this;
- }
-
- @Override
- public boolean equals(Object obj) {
- if (obj instanceof RefString) {
- return this.s.equals(((RefString) obj).s);
- } else if (obj instanceof String) {
- return this.s.equals((String) obj);
- }
-
- return false;
- }
- }
-
- @Test
- void simple() {
- RangeCache<Integer, RefString> cache = new RangeCache<>();
-
- cache.put(0, new RefString("0"));
- cache.put(1, new RefString("1"));
-
- assertEquals(cache.getSize(), 2);
- assertEquals(cache.getNumberOfEntries(), 2);
-
- RefString s = cache.get(0);
- assertEquals(s.s, "0");
- assertEquals(s.refCnt(), 2);
- s.release();
-
- RefString s1 = cache.get(0);
- RefString s2 = cache.get(0);
- assertEquals(s1, s2);
- assertEquals(s1.refCnt(), 3);
- s1.release();
- s2.release();
-
- assertEquals(cache.get(2), null);
-
- cache.put(2, new RefString("2"));
- cache.put(8, new RefString("8"));
- cache.put(11, new RefString("11"));
-
- assertEquals(cache.getSize(), 5);
- assertEquals(cache.getNumberOfEntries(), 5);
-
- cache.removeRange(1, 5, true);
- assertEquals(cache.getSize(), 3);
- assertEquals(cache.getNumberOfEntries(), 3);
-
- cache.removeRange(2, 8, false);
- assertEquals(cache.getSize(), 3);
- assertEquals(cache.getNumberOfEntries(), 3);
-
- cache.removeRange(0, 100, false);
- assertEquals(cache.getSize(), 0);
- assertEquals(cache.getNumberOfEntries(), 0);
-
- cache.removeRange(0, 100, false);
- assertEquals(cache.getSize(), 0);
- assertEquals(cache.getNumberOfEntries(), 0);
- }
-
- @Test
- void customWeighter() {
- RangeCache<Integer, RefString> cache = new RangeCache<>(value -> value.s.length());
-
- cache.put(0, new RefString("zero"));
- cache.put(1, new RefString("one"));
-
- assertEquals(cache.getSize(), 7);
- assertEquals(cache.getNumberOfEntries(), 2);
- }
-
- @Test
- void doubleInsert() {
- RangeCache<Integer, RefString> cache = new RangeCache<>();
-
- RefString s0 = new RefString("zero");
- assertEquals(s0.refCnt(), 1);
- assertEquals(cache.put(0, s0), true);
- assertEquals(s0.refCnt(), 1);
-
- cache.put(1, new RefString("one"));
-
- assertEquals(cache.getSize(), 2);
- assertEquals(cache.getNumberOfEntries(), 2);
- RefString s = cache.get(1);
- assertEquals(s.s, "one");
- assertEquals(s.refCnt(), 2);
-
- RefString s1 = new RefString("uno");
- assertEquals(s1.refCnt(), 1);
- assertEquals(cache.put(1, s1), false);
- assertEquals(s1.refCnt(), 1);
- s1.release();
-
- // Should not have been overridden in cache
- assertEquals(cache.getSize(), 2);
- assertEquals(cache.getNumberOfEntries(), 2);
- assertEquals(cache.get(1).s, "one");
- }
-
- @Test
- void getRange() {
- RangeCache<Integer, RefString> cache = new RangeCache<>();
-
- cache.put(0, new RefString("0"));
- cache.put(1, new RefString("1"));
- cache.put(3, new RefString("3"));
- cache.put(5, new RefString("5"));
-
- assertEquals(cache.getRange(1, 8),
- Lists.newArrayList(new RefString("1"), new RefString("3"), new RefString("5")));
-
- cache.put(8, new RefString("8"));
- assertEquals(cache.getRange(1, 8),
- Lists.newArrayList(new RefString("1"), new RefString("3"), new RefString("5"), new RefString("8")));
-
- cache.clear();
- assertEquals(cache.getSize(), 0);
- assertEquals(cache.getNumberOfEntries(), 0);
- }
-
- @Test
- void eviction() {
- RangeCache<Integer, RefString> cache = new RangeCache<>(value -> value.s.length());
-
- cache.put(0, new RefString("zero"));
- cache.put(1, new RefString("one"));
- cache.put(2, new RefString("two"));
- cache.put(3, new RefString("three"));
-
- // This should remove the LRU entries: 0, 1 whose combined size is 7
- assertEquals(cache.evictLeastAccessedEntries(5), Pair.of(2, (long) 7));
-
- assertEquals(cache.getNumberOfEntries(), 2);
- assertEquals(cache.getSize(), 8);
- assertEquals(cache.get(0), null);
- assertEquals(cache.get(1), null);
- assertEquals(cache.get(2).s, "two");
- assertEquals(cache.get(3).s, "three");
-
- assertEquals(cache.evictLeastAccessedEntries(100), Pair.of(2, (long) 8));
- assertEquals(cache.getNumberOfEntries(), 0);
- assertEquals(cache.getSize(), 0);
- assertEquals(cache.get(0), null);
- assertEquals(cache.get(1), null);
- assertEquals(cache.get(2), null);
- assertEquals(cache.get(3), null);
-
- try {
- cache.evictLeastAccessedEntries(0);
- fail("should throw exception");
- } catch (IllegalArgumentException e) {
- // ok
- }
-
- try {
- cache.evictLeastAccessedEntries(-1);
- fail("should throw exception");
- } catch (IllegalArgumentException e) {
- // ok
- }
- }
-
- @Test
- void evictions() {
- RangeCache<Integer, RefString> cache = new RangeCache<>();
-
- for (int i = 0; i < 100; i++) {
- cache.put(i, new RefString(Integer.toString(i)));
- }
-
- assertEquals(cache.getSize(), 100);
- Pair<Integer, Long> res = cache.evictLeastAccessedEntries(1);
- assertEquals((int) res.getLeft(), 1);
- assertEquals((long) res.getRight(), 1);
- assertEquals(cache.getSize(), 99);
-
- res = cache.evictLeastAccessedEntries(10);
- assertEquals((int) res.getLeft(), 10);
- assertEquals((long) res.getRight(), 10);
- assertEquals(cache.getSize(), 89);
-
- res = cache.evictLeastAccessedEntries(100);
- assertEquals((int) res.getLeft(), 89);
- assertEquals((long) res.getRight(), 89);
- assertEquals(cache.getSize(), 0);
-
- for (int i = 0; i < 100; i++) {
- cache.put(i, new RefString(Integer.toString(i)));
- }
-
- assertEquals(cache.getSize(), 100);
-
- res = cache.removeRange(10, 20, false);
- assertEquals((int) res.getLeft(), 10);
- assertEquals((long) res.getRight(), 10);
- assertEquals(cache.getSize(), 90);
- }
-}
diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/util/TestStatsBuckets.java b/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/util/TestStatsBuckets.java
deleted file mode 100644
index 02366632ed..0000000000
--- a/managed-ledger/src/test/java/org/apache/bookkeeper/mledger/util/TestStatsBuckets.java
+++ /dev/null
@@ -1,102 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.bookkeeper.mledger.util;
-
-import static org.testng.Assert.assertEquals;
-import static org.testng.Assert.fail;
-
-import org.testng.annotations.Test;
-
-public class TestStatsBuckets {
-
- @Test
- public void testInvalidConstructor() {
- try {
- new StatsBuckets();
- fail("Should have thrown exception");
- } catch (IllegalArgumentException e) {
- // ok
- }
- }
-
- @Test
- public void testUnorderedBoundaries() {
- try {
- new StatsBuckets(2, 1);
- fail("Should have thrown exception");
- } catch (IllegalArgumentException e) {
- // ok
- }
- }
-
- @Test
- public void test() {
- StatsBuckets stats = new StatsBuckets(10, 20, 30);
-
- assertEquals(stats.getAvg(), Double.NaN);
- assertEquals(stats.getSum(), 0);
- assertEquals(stats.getCount(), 0);
- assertEquals(stats.getBuckets(), new long[] { 0, 0, 0, 0 });
-
- stats.addValue(5);
-
- // Before refresh stats should not be updated
- assertEquals(stats.getAvg(), Double.NaN);
- assertEquals(stats.getSum(), 0);
- assertEquals(stats.getCount(), 0);
- assertEquals(stats.getBuckets(), new long[] { 0, 0, 0, 0 });
-
- stats.refresh();
-
- assertEquals(stats.getAvg(), 5.0);
- assertEquals(stats.getSum(), 5);
- assertEquals(stats.getCount(), 1);
- assertEquals(stats.getBuckets(), new long[] { 1, 0, 0, 0 });
-
- stats.addValue(15);
-
- // Before refresh stats should not be updated
- assertEquals(stats.getAvg(), 5.0);
- assertEquals(stats.getSum(), 5);
- assertEquals(stats.getCount(), 1);
- assertEquals(stats.getBuckets(), new long[] { 1, 0, 0, 0 });
-
- stats.refresh();
-
- assertEquals(stats.getAvg(), 15.0);
- assertEquals(stats.getSum(), 15);
- assertEquals(stats.getCount(), 1);
- assertEquals(stats.getBuckets(), new long[] { 0, 1, 0, 0 });
-
- stats.addValue(50);
-
- assertEquals(stats.getSum(), 15);
- assertEquals(stats.getCount(), 1);
-
- stats.addValue(10);
- stats.addValue(30);
-
- stats.refresh();
-
- assertEquals(stats.getAvg(), 30.0);
- assertEquals(stats.getSum(), 90);
- assertEquals(stats.getCount(), 3);
- assertEquals(stats.getBuckets(), new long[] { 1, 0, 1, 1 });
- }
-}
diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/test/BookKeeperClusterTestCase.java b/managed-ledger/src/test/java/org/apache/bookkeeper/test/BookKeeperClusterTestCase.java
deleted file mode 100644
index 7f90042877..0000000000
--- a/managed-ledger/src/test/java/org/apache/bookkeeper/test/BookKeeperClusterTestCase.java
+++ /dev/null
@@ -1,499 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-/**
- * This file is derived from BookKeeperClusterTestCase from Apache BookKeeper
- * http://bookkeeper.apache.org
- */
-
-package org.apache.bookkeeper.test;
-
-import java.io.File;
-import java.io.IOException;
-import java.net.InetAddress;
-import java.net.InetSocketAddress;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.concurrent.CountDownLatch;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import org.apache.bookkeeper.bookie.Bookie;
-import org.apache.bookkeeper.bookie.BookieException;
-import org.apache.bookkeeper.client.BookKeeperTestClient;
-import org.apache.bookkeeper.conf.AbstractConfiguration;
-import org.apache.bookkeeper.conf.ClientConfiguration;
-import org.apache.bookkeeper.conf.ServerConfiguration;
-import org.apache.bookkeeper.metastore.InMemoryMetaStore;
-import org.apache.bookkeeper.net.BookieSocketAddress;
-import org.apache.bookkeeper.proto.BookieServer;
-import org.apache.bookkeeper.replication.AutoRecoveryMain;
-import org.apache.commons.io.FileUtils;
-import org.apache.zookeeper.CreateMode;
-import org.apache.zookeeper.KeeperException;
-import org.apache.zookeeper.ZooDefs;
-import org.apache.zookeeper.ZooKeeper;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.annotations.AfterMethod;
-import org.testng.annotations.BeforeMethod;
-
-/**
- * A class runs several bookie servers for testing.
- */
-public abstract class BookKeeperClusterTestCase {
-
- static final Logger LOG = LoggerFactory.getLogger(BookKeeperClusterTestCase.class);
-
- // ZooKeeper related variables
- protected ZooKeeperUtil zkUtil = new ZooKeeperUtil();
- protected ZooKeeper zkc;
-
- // BookKeeper related variables
- protected List<File> tmpDirs = new LinkedList<File>();
- protected List<BookieServer> bs = new LinkedList<BookieServer>();
- protected List<ServerConfiguration> bsConfs = new LinkedList<ServerConfiguration>();
- protected int numBookies;
- protected BookKeeperTestClient bkc;
-
- protected ServerConfiguration baseConf = new ServerConfiguration();
- protected ClientConfiguration baseClientConf = new ClientConfiguration();
-
- private final Map<BookieServer, AutoRecoveryMain> autoRecoveryProcesses = new HashMap<BookieServer, AutoRecoveryMain>();
-
- private boolean isAutoRecoveryEnabled;
-
- protected ExecutorService executor;
-
- public BookKeeperClusterTestCase(int numBookies) {
- this.numBookies = numBookies;
- }
-
- @BeforeMethod
- public void setUp() throws Exception {
- executor = Executors.newCachedThreadPool();
- InMemoryMetaStore.reset();
- setMetastoreImplClass(baseConf);
- setMetastoreImplClass(baseClientConf);
-
- try {
- // start zookeeper service
- startZKCluster();
- // start bookkeeper service
- startBKCluster();
-
- zkc.create("/managed-ledgers", new byte[0], ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
- } catch (Exception e) {
- LOG.error("Error setting up", e);
- throw e;
- }
- }
-
- @AfterMethod
- public void tearDown() throws Exception {
- // stop bookkeeper service
- stopBKCluster();
- // stop zookeeper service
- stopZKCluster();
- executor.shutdown();
- }
-
- /**
- * Start zookeeper cluster
- *
- * @throws Exception
- */
- protected void startZKCluster() throws Exception {
- zkUtil.startServer();
- zkc = zkUtil.getZooKeeperClient();
- }
-
- /**
- * Stop zookeeper cluster
- *
- * @throws Exception
- */
- protected void stopZKCluster() throws Exception {
- zkUtil.killServer();
- }
-
- /**
- * Start cluster. Also, starts the auto recovery process for each bookie, if isAutoRecoveryEnabled is true.
- *
- * @throws Exception
- */
- protected void startBKCluster() throws Exception {
- baseClientConf.setZkServers(zkUtil.getZooKeeperConnectString());
- baseClientConf.setUseV2WireProtocol(true);
- baseClientConf.setEnableDigestTypeAutodetection(true);
- if (numBookies > 0) {
- bkc = new BookKeeperTestClient(baseClientConf);
- }
-
- // Create Bookie Servers (B1, B2, B3)
- for (int i = 0; i < numBookies; i++) {
- startNewBookie();
- }
- }
-
- /**
- * Stop cluster. Also, stops all the auto recovery processes for the bookie cluster, if isAutoRecoveryEnabled is
- * true.
- *
- * @throws Exception
- */
- protected void stopBKCluster() throws Exception {
- if (bkc != null) {
- bkc.close();
- }
-
- for (BookieServer server : bs) {
- server.shutdown();
- AutoRecoveryMain autoRecovery = autoRecoveryProcesses.get(server);
- if (autoRecovery != null && isAutoRecoveryEnabled()) {
- autoRecovery.shutdown();
- LOG.debug("Shutdown auto recovery for bookieserver:" + server.getLocalAddress());
- }
- }
- bs.clear();
- for (File f : tmpDirs) {
- FileUtils.deleteDirectory(f);
- }
- }
-
- protected ServerConfiguration newServerConfiguration() throws Exception {
- File f = File.createTempFile("bookie", "test");
- tmpDirs.add(f);
- f.delete();
- f.mkdir();
-
- int port = PortManager.nextFreePort();
- return newServerConfiguration(port, zkUtil.getZooKeeperConnectString(), f, new File[] { f });
- }
-
- protected ClientConfiguration newClientConfiguration() {
- return new ClientConfiguration(baseConf);
- }
-
- protected ServerConfiguration newServerConfiguration(int port, String zkServers, File journalDir,
- File[] ledgerDirs) {
- ServerConfiguration conf = new ServerConfiguration(baseConf);
- conf.setBookiePort(port);
- conf.setZkServers(zkServers);
- conf.setJournalDirName(journalDir.getPath());
- conf.setAllowLoopback(true);
- conf.setFlushInterval(60 * 1000);
- conf.setGcWaitTime(60 * 1000);
- String[] ledgerDirNames = new String[ledgerDirs.length];
- for (int i = 0; i < ledgerDirs.length; i++) {
- ledgerDirNames[i] = ledgerDirs[i].getPath();
- }
- conf.setLedgerDirNames(ledgerDirNames);
- conf.setLedgerStorageClass("org.apache.bookkeeper.bookie.storage.ldb.DbLedgerStorage");
- return conf;
- }
-
- /**
- * Get bookie address for bookie at index
- */
- public BookieSocketAddress getBookie(int index) throws Exception {
- if (bs.size() <= index || index < 0) {
- throw new IllegalArgumentException(
- "Invalid index, there are only " + bs.size() + " bookies. Asked for " + index);
- }
- return bs.get(index).getLocalAddress();
- }
-
- /**
- * Kill a bookie by its socket address. Also, stops the autorecovery process for the corresponding bookie server, if
- * isAutoRecoveryEnabled is true.
- *
- * @param addr
- * Socket Address
- * @return the configuration of killed bookie
- * @throws InterruptedException
- */
- public ServerConfiguration killBookie(InetSocketAddress addr) throws Exception {
- BookieServer toRemove = null;
- int toRemoveIndex = 0;
- for (BookieServer server : bs) {
- if (server.getLocalAddress().equals(addr)) {
- server.shutdown();
- toRemove = server;
- break;
- }
- ++toRemoveIndex;
- }
- if (toRemove != null) {
- stopAutoRecoveryService(toRemove);
- bs.remove(toRemove);
- return bsConfs.remove(toRemoveIndex);
- }
- return null;
- }
-
- /**
- * Kill a bookie by index. Also, stops the respective auto recovery process for this bookie, if
- * isAutoRecoveryEnabled is true.
- *
- * @param index
- * Bookie Index
- * @return the configuration of killed bookie
- * @throws InterruptedException
- * @throws IOException
- */
- public ServerConfiguration killBookie(int index) throws Exception {
- if (index >= bs.size()) {
- throw new IOException("Bookie does not exist");
- }
- BookieServer server = bs.get(index);
- server.shutdown();
- stopAutoRecoveryService(server);
- bs.remove(server);
- return bsConfs.remove(index);
- }
-
- /**
- * Sleep a bookie
- *
- * @param addr
- * Socket Address
- * @param seconds
- * Sleep seconds
- * @return Count Down latch which will be counted down when sleep finishes
- * @throws InterruptedException
- * @throws IOException
- */
- public CountDownLatch sleepBookie(InetSocketAddress addr, final int seconds) throws Exception {
- for (final BookieServer bookie : bs) {
- if (bookie.getLocalAddress().equals(addr)) {
- final CountDownLatch l = new CountDownLatch(1);
- Thread sleeper = new Thread() {
- @Override
- public void run() {
- try {
- bookie.suspendProcessing();
- l.countDown();
- Thread.sleep(seconds * 1000);
- bookie.resumeProcessing();
- } catch (Exception e) {
- LOG.error("Error suspending bookie", e);
- }
- }
- };
- sleeper.start();
- return l;
- }
- }
- throw new IOException("Bookie not found");
- }
-
- /**
- * Sleep a bookie until I count down the latch
- *
- * @param addr
- * Socket Address
- * @param latch
- * Latch to wait on
- * @throws InterruptedException
- * @throws IOException
- */
- public void sleepBookie(InetSocketAddress addr, final CountDownLatch l) throws Exception {
- for (final BookieServer bookie : bs) {
- if (bookie.getLocalAddress().equals(addr)) {
- Thread sleeper = new Thread() {
- @Override
- public void run() {
- try {
- bookie.suspendProcessing();
- l.await();
- bookie.resumeProcessing();
- } catch (Exception e) {
- LOG.error("Error suspending bookie", e);
- }
- }
- };
- sleeper.start();
- return;
- }
- }
- throw new IOException("Bookie not found");
- }
-
- /**
- * Restart bookie servers. Also restarts all the respective auto recovery process, if isAutoRecoveryEnabled is true.
- *
- * @throws InterruptedException
- * @throws IOException
- * @throws KeeperException
- * @throws BookieException
- */
- public void restartBookies() throws Exception {
- restartBookies(null);
- }
-
- /**
- * Restart bookie servers using new configuration settings. Also restart the respective auto recovery process, if
- * isAutoRecoveryEnabled is true.
- *
- * @param newConf
- * New Configuration Settings
- * @throws InterruptedException
- * @throws IOException
- * @throws KeeperException
- * @throws BookieException
- */
- public void restartBookies(ServerConfiguration newConf) throws Exception {
- // shut down bookie server
- for (BookieServer server : bs) {
- server.shutdown();
- stopAutoRecoveryService(server);
- }
- bs.clear();
- Thread.sleep(1000);
- // restart them to ensure we can't
-
- List<ServerConfiguration> bsConfsCopy = new ArrayList<ServerConfiguration>(bsConfs);
- bsConfs.clear();
- for (ServerConfiguration conf : bsConfsCopy) {
- if (null != newConf) {
- conf.loadConf(newConf);
- }
- startBookie(conf);
- }
- }
-
- /**
- * Helper method to startup a new bookie server with the indicated port number. Also, starts the auto recovery
- * process, if the isAutoRecoveryEnabled is set true.
- *
- * @param port
- * Port to start the new bookie server on
- * @throws IOException
- */
- public int startNewBookie() throws Exception {
- ServerConfiguration conf = newServerConfiguration();
- startBookie(conf);
-
- return conf.getBookiePort();
- }
-
- /**
- * Helper method to startup a bookie server using a configuration object. Also, starts the auto recovery process if
- * isAutoRecoveryEnabled is true.
- *
- * @param conf
- * Server Configuration Object
- *
- */
- protected BookieServer startBookie(ServerConfiguration conf) throws Exception {
- BookieServer server = new BookieServer(conf);
- bsConfs.add(conf);
- bs.add(server);
-
- server.start();
-
- if (bkc == null) {
- bkc = new BookKeeperTestClient(baseClientConf);
- }
-
- int port = conf.getBookiePort();
- while (bkc.getZkHandle().exists(
- "/ledgers/available/" + InetAddress.getLocalHost().getHostAddress() + ":" + port, false) == null) {
- Thread.sleep(500);
- }
-
- bkc.readBookiesBlocking();
- LOG.info("New bookie on port " + port + " has been created.");
-
- return server;
- }
-
- /**
- * Start a bookie with the given bookie instance. Also, starts the auto recovery for this bookie, if
- * isAutoRecoveryEnabled is true.
- */
- protected BookieServer startBookie(ServerConfiguration conf, final Bookie b) throws Exception {
- BookieServer server = new BookieServer(conf) {
- @Override
- protected Bookie newBookie(ServerConfiguration conf)
- throws IOException, KeeperException, InterruptedException, BookieException {
- return b;
- }
- };
- server.start();
-
- int port = conf.getBookiePort();
- while (bkc.getZkHandle().exists(
- "/ledgers/available/" + InetAddress.getLocalHost().getHostAddress() + ":" + port, false) == null) {
- Thread.sleep(500);
- }
-
- bkc.readBookiesBlocking();
- LOG.info("New bookie on port " + port + " has been created.");
- return server;
- }
-
- public void setMetastoreImplClass(AbstractConfiguration conf) {
- conf.setMetastoreImplClass(InMemoryMetaStore.class.getName());
- }
-
- /**
- * Flags used to enable/disable the auto recovery process. If it is enabled, starting the bookie server will starts
- * the auto recovery process for that bookie. Also, stopping bookie will stops the respective auto recovery process.
- *
- * @param isAutoRecoveryEnabled
- * Value true will enable the auto recovery process. Value false will disable the auto recovery process
- */
- public void setAutoRecoveryEnabled(boolean isAutoRecoveryEnabled) {
- this.isAutoRecoveryEnabled = isAutoRecoveryEnabled;
- }
-
- /**
- * Flag used to check whether auto recovery process is enabled/disabled. By default the flag is false.
- *
- * @return true, if the auto recovery is enabled. Otherwise return false.
- */
- public boolean isAutoRecoveryEnabled() {
- return isAutoRecoveryEnabled;
- }
-
- private void stopAutoRecoveryService(BookieServer toRemove) throws Exception {
- AutoRecoveryMain autoRecoveryMain = autoRecoveryProcesses.remove(toRemove);
- if (null != autoRecoveryMain && isAutoRecoveryEnabled()) {
- autoRecoveryMain.shutdown();
- LOG.debug("Shutdown auto recovery for bookieserver:" + toRemove.getLocalAddress());
- }
- }
-
- /**
- * Will stops all the auto recovery processes for the bookie cluster, if isAutoRecoveryEnabled is true.
- */
- public void stopReplicationService() throws Exception {
- if (false == isAutoRecoveryEnabled()) {
- return;
- }
- for (Entry<BookieServer, AutoRecoveryMain> autoRecoveryProcess : autoRecoveryProcesses.entrySet()) {
- autoRecoveryProcess.getValue().shutdown();
- LOG.debug("Shutdown Auditor Recovery for the bookie:" + autoRecoveryProcess.getKey().getLocalAddress());
- }
- }
-}
diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/test/PortManager.java b/managed-ledger/src/test/java/org/apache/bookkeeper/test/PortManager.java
deleted file mode 100644
index a18cc04d0d..0000000000
--- a/managed-ledger/src/test/java/org/apache/bookkeeper/test/PortManager.java
+++ /dev/null
@@ -1,113 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.bookkeeper.test;
-
-import java.io.FileReader;
-import java.io.FileWriter;
-import java.io.IOException;
-import java.net.ServerSocket;
-import java.nio.CharBuffer;
-import java.nio.channels.FileChannel;
-import java.nio.channels.FileLock;
-import java.nio.file.Path;
-import java.nio.file.Paths;
-import java.nio.file.StandardOpenOption;
-
-/**
- * Port manager allows a base port to be specified on the commandline. Tests will then use ports, counting up from this
- * base port. This allows multiple instances of the bookkeeper tests to run at once.
- */
-public class PortManager {
-
- private static final String lockFilename = System.getProperty("test.lockFilename",
- "/tmp/pulsar-test-port-manager.lock");
- private static final int basePort = Integer.valueOf(System.getProperty("test.basePort", "15000"));
-
- private static final int maxPort = 32000;
-
- /**
- * Return a TCP port that is currently unused.
- *
- * Keeps track of assigned ports and avoid race condition between different processes
- */
- public synchronized static int nextFreePort() {
- Path path = Paths.get(lockFilename);
-
- try {
- FileChannel fileChannel = FileChannel.open(path, StandardOpenOption.CREATE, StandardOpenOption.WRITE);
- FileLock lock = fileChannel.lock();
-
- try {
-
- FileReader reader = new FileReader(lockFilename);
- CharBuffer buffer = CharBuffer.allocate(16);
- int len = reader.read(buffer);
- buffer.flip();
-
- int lastUsedPort = basePort;
- if (len > 0) {
- String lastUsedPortStr = buffer.toString();
- lastUsedPort = Integer.parseInt(lastUsedPortStr);
- }
-
- int freePort = probeFreePort(lastUsedPort + 1);
-
- FileWriter writer = new FileWriter(lockFilename);
- writer.write(Integer.toString(freePort));
-
- reader.close();
- writer.close();
-
- return freePort;
-
- } finally {
- lock.release();
- fileChannel.close();
- }
- } catch (IOException e) {
- throw new RuntimeException(e);
- }
- }
-
- private static final int MAX_PORT_CONFLICTS = 10;
-
- private synchronized static int probeFreePort(int port) {
- int exceptionCount = 0;
- while (true) {
- if (port == maxPort) {
- // Rollover the port probe
- port = basePort;
- }
-
- try (ServerSocket ss = new ServerSocket(port)) {
- ss.close();
- // Give it some time to truly close the connection
- Thread.sleep(100);
- return port;
-
- } catch (Exception e) {
- port++;
- exceptionCount++;
- if (exceptionCount > MAX_PORT_CONFLICTS) {
- throw new RuntimeException(e);
- }
- }
- }
- }
-}
diff --git a/managed-ledger/src/test/java/org/apache/bookkeeper/test/ZooKeeperUtil.java b/managed-ledger/src/test/java/org/apache/bookkeeper/test/ZooKeeperUtil.java
deleted file mode 100644
index 8b44ebb741..0000000000
--- a/managed-ledger/src/test/java/org/apache/bookkeeper/test/ZooKeeperUtil.java
+++ /dev/null
@@ -1,137 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-/**
- * This file is derived from ZooKeeperUtil from Apache BookKeeper
- * http://bookkeeper.apache.org
- */
-
-package org.apache.bookkeeper.test;
-
-import java.io.File;
-import java.io.IOException;
-import java.net.InetSocketAddress;
-import java.util.concurrent.CountDownLatch;
-
-import org.apache.bookkeeper.zookeeper.ZooKeeperClient;
-import org.apache.commons.io.FileUtils;
-import org.apache.zookeeper.CreateMode;
-import org.apache.zookeeper.ZooDefs.Ids;
-import org.apache.zookeeper.ZooKeeper;
-import org.apache.zookeeper.server.NIOServerCnxnFactory;
-import org.apache.zookeeper.server.ZooKeeperServer;
-import org.apache.zookeeper.test.ClientBase;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.Assert;
-
-public class ZooKeeperUtil {
- static final Logger LOG = LoggerFactory.getLogger(ZooKeeperUtil.class);
-
- // ZooKeeper related variables
- protected final static Integer zooKeeperPort = PortManager.nextFreePort();
- private final InetSocketAddress zkaddr;
-
- protected ZooKeeperServer zks;
- protected ZooKeeper zkc; // zookeeper client
- protected NIOServerCnxnFactory serverFactory;
- protected File ZkTmpDir;
- private final String connectString;
-
- public ZooKeeperUtil() {
- zkaddr = new InetSocketAddress(zooKeeperPort);
- connectString = "localhost:" + zooKeeperPort;
- }
-
- public ZooKeeper getZooKeeperClient() {
- return zkc;
- }
-
- public String getZooKeeperConnectString() {
- return connectString;
- }
-
- public void startServer() throws Exception {
- // create a ZooKeeper server(dataDir, dataLogDir, port)
- LOG.debug("Running ZK server");
- // ServerStats.registerAsConcrete();
- ClientBase.setupTestEnv();
- ZkTmpDir = File.createTempFile("zookeeper", "test");
- ZkTmpDir.delete();
- ZkTmpDir.mkdir();
-
- zks = new ZooKeeperServer(ZkTmpDir, ZkTmpDir, ZooKeeperServer.DEFAULT_TICK_TIME);
- serverFactory = new NIOServerCnxnFactory();
- serverFactory.configure(zkaddr, 100);
- serverFactory.startup(zks);
-
- boolean b = ClientBase.waitForServerUp(getZooKeeperConnectString(), ClientBase.CONNECTION_TIMEOUT);
- LOG.debug("Server up: " + b);
-
- // create a zookeeper client
- LOG.debug("Instantiate ZK Client");
- zkc = ZooKeeperClient.newBuilder().connectString(getZooKeeperConnectString()).build();
-
- // initialize the zk client with values
- zkc.create("/ledgers", new byte[0], Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
- zkc.create("/ledgers/available", new byte[0], Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
- }
-
- @SuppressWarnings("deprecation")
- public void sleepServer(final int seconds, final CountDownLatch l) throws InterruptedException, IOException {
- Thread[] allthreads = new Thread[Thread.activeCount()];
- Thread.enumerate(allthreads);
- for (final Thread t : allthreads) {
- if (t.getName().contains("SyncThread:0")) {
- Thread sleeper = new Thread() {
- public void run() {
- try {
- t.suspend();
- l.countDown();
- Thread.sleep(seconds * 1000);
- t.resume();
- } catch (Exception e) {
- LOG.error("Error suspending thread", e);
- }
- }
- };
- sleeper.start();
- return;
- }
- }
- throw new IOException("ZooKeeper thread not found");
- }
-
- public void killServer() throws Exception {
- if (zkc != null) {
- zkc.close();
- }
-
- // shutdown ZK server
- if (serverFactory != null) {
- serverFactory.shutdown();
- Assert.assertTrue(ClientBase.waitForServerDown(getZooKeeperConnectString(), ClientBase.CONNECTION_TIMEOUT),
- "waiting for server down");
- }
- if (zks != null) {
- zks.getTxnLogFactory().close();
- }
- // ServerStats.unregister();
- FileUtils.deleteDirectory(ZkTmpDir);
- }
-}
diff --git a/pulsar-broker-auth-athenz/src/test/java/org/apache/pulsar/broker/authentication/AuthenticationProviderAthenzTest.java b/pulsar-broker-auth-athenz/src/test/java/org/apache/pulsar/broker/authentication/AuthenticationProviderAthenzTest.java
deleted file mode 100644
index 1946a014e7..0000000000
--- a/pulsar-broker-auth-athenz/src/test/java/org/apache/pulsar/broker/authentication/AuthenticationProviderAthenzTest.java
+++ /dev/null
@@ -1,136 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pulsar.broker.authentication;
-
-import static org.testng.Assert.assertEquals;
-import static org.testng.Assert.fail;
-
-import org.testng.annotations.Test;
-import org.testng.annotations.BeforeClass;
-
-import org.apache.bookkeeper.test.PortManager;
-import org.apache.pulsar.broker.authentication.AuthenticationDataCommand;
-import org.apache.pulsar.broker.authentication.AuthenticationDataSource;
-import org.apache.pulsar.broker.authentication.AuthenticationProviderAthenz;
-
-import java.util.ArrayList;
-import java.util.List;
-import java.util.Properties;
-import java.net.InetSocketAddress;
-import java.nio.file.Files;
-import java.nio.file.Paths;
-import javax.naming.AuthenticationException;
-
-import com.yahoo.athenz.zpe.ZpeConsts;
-import com.yahoo.athenz.auth.token.RoleToken;
-import org.apache.pulsar.broker.ServiceConfiguration;
-
-public class AuthenticationProviderAthenzTest {
-
- ServiceConfiguration config;
- Properties properties;
- AuthenticationProviderAthenz provider;
-
- @BeforeClass
- public void setup() throws Exception {
-
- // Set provider domain name
- properties = new Properties();
- properties.setProperty("athenzDomainNames", "test_provider");
- config = new ServiceConfiguration();
- config.setProperties(properties);
-
- // Initialize authentication provider
- provider = new AuthenticationProviderAthenz();
- provider.initialize(config);
-
- // Specify Athenz configuration file for AuthZpeClient which is used in AuthenticationProviderAthenz
- System.setProperty(ZpeConsts.ZPE_PROP_ATHENZ_CONF, "./src/test/resources/athenz.conf.test");
- }
-
- @Test
- public void testInitilizeFromSystemPropeties() {
- System.setProperty("pulsar.athenz.domain.names", "test_provider");
- ServiceConfiguration emptyConf = new ServiceConfiguration();
- Properties emptyProp = new Properties();
- emptyConf.setProperties(emptyProp);
- AuthenticationProviderAthenz sysPropProvider = new AuthenticationProviderAthenz();
- try {
- sysPropProvider.initialize(emptyConf);
- } catch (Exception e) {
- fail("Fail to Read pulsar.athenz.domain.names from System Properties");
- }
- }
-
- @Test
- public void testAuthenticateSignedToken() throws Exception {
-
- List<String> roles = new ArrayList<String>() {
- {
- add("test_role");
- }
- };
- RoleToken token = new RoleToken.Builder("Z1", "test_provider", roles).principal("test_app").build();
- String privateKey = new String(Files.readAllBytes(Paths.get("./src/test/resources/zts_private.pem")));
- token.sign(privateKey);
- AuthenticationDataSource authData = new AuthenticationDataCommand(token.getSignedToken(),
- new InetSocketAddress("localhost", PortManager.nextFreePort()), null);
- assertEquals(provider.authenticate(authData), "test_app");
- }
-
- @Test
- public void testAuthenticateUnsignedToken() throws Exception {
-
- List<String> roles = new ArrayList<String>() {
- {
- add("test_role");
- }
- };
- RoleToken token = new RoleToken.Builder("Z1", "test_provider", roles).principal("test_app").build();
- AuthenticationDataSource authData = new AuthenticationDataCommand(token.getUnsignedToken(),
- new InetSocketAddress("localhost", PortManager.nextFreePort()), null);
- try {
- provider.authenticate(authData);
- fail("Unsigned token should not be authenticated");
- } catch (AuthenticationException e) {
- // OK, expected
- }
- }
-
- @Test
- public void testAuthenticateSignedTokenWithDifferentDomain() throws Exception {
-
- List<String> roles = new ArrayList<String>() {
- {
- add("test_role");
- }
- };
- RoleToken token = new RoleToken.Builder("Z1", "invalid", roles).principal("test_app").build();
- String privateKey = new String(Files.readAllBytes(Paths.get("./src/test/resources/zts_private.pem")));
- token.sign(privateKey);
- AuthenticationDataSource authData = new AuthenticationDataCommand(token.getSignedToken(),
- new InetSocketAddress("localhost", PortManager.nextFreePort()), null);
- try {
- provider.authenticate(authData);
- fail("Token which has different domain should not be authenticated");
- } catch (AuthenticationException e) {
- // OK, expected
- }
- }
-}
\ No newline at end of file
diff --git a/pulsar-broker-common/src/test/java/org/apache/pulsar/common/configuration/PulsarConfigurationLoaderTest.java b/pulsar-broker-common/src/test/java/org/apache/pulsar/common/configuration/PulsarConfigurationLoaderTest.java
deleted file mode 100644
index baa0e7558c..0000000000
--- a/pulsar-broker-common/src/test/java/org/apache/pulsar/common/configuration/PulsarConfigurationLoaderTest.java
+++ /dev/null
@@ -1,218 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pulsar.common.configuration;
-
-import static org.apache.pulsar.common.configuration.PulsarConfigurationLoader.isComplete;
-import static org.testng.Assert.assertEquals;
-import static org.testng.Assert.assertNotNull;
-import static org.testng.Assert.assertTrue;
-import static org.testng.Assert.fail;
-
-import java.io.File;
-import java.io.FileInputStream;
-import java.io.FileOutputStream;
-import java.io.InputStream;
-import java.io.OutputStreamWriter;
-import java.io.PrintWriter;
-import java.util.Properties;
-
-import org.apache.bookkeeper.client.api.DigestType;
-import org.apache.pulsar.broker.ServiceConfiguration;
-import org.testng.annotations.Test;
-
-public class PulsarConfigurationLoaderTest {
- public class MockConfiguration implements PulsarConfiguration {
- private Properties properties = new Properties();
-
- private String zookeeperServers = "localhost:2181";
- private String configurationStoreServers = "localhost:2184";
- private int brokerServicePort = 7650;
- private int brokerServicePortTls = 7651;
- private int webServicePort = 9080;
- private int webServicePortTls = 9443;
- private int notExistFieldInServiceConfig = 0;
-
- @Override
- public Properties getProperties() {
- return properties;
- }
-
- @Override
- public void setProperties(Properties properties) {
- this.properties = properties;
- }
- }
-
- @Test
- public void testConfigurationConverting() throws Exception {
- MockConfiguration mockConfiguration = new MockConfiguration();
- ServiceConfiguration serviceConfiguration = PulsarConfigurationLoader.convertFrom(mockConfiguration);
-
- // check whether converting correctly
- assertEquals(serviceConfiguration.getZookeeperServers(), "localhost:2181");
- assertEquals(serviceConfiguration.getConfigurationStoreServers(), "localhost:2184");
- assertEquals(serviceConfiguration.getBrokerServicePort(), 7650);
- assertEquals(serviceConfiguration.getBrokerServicePortTls(), 7651);
- assertEquals(serviceConfiguration.getWebServicePort(), 9080);
- assertEquals(serviceConfiguration.getWebServicePortTls(), 9443);
-
- // check whether exception causes
- try {
- PulsarConfigurationLoader.convertFrom(mockConfiguration, false);
- fail();
- } catch (Exception e) {
- assertEquals(e.getClass(), IllegalArgumentException.class);
- }
- }
-
- @Test
- public void testPulsarConfiguraitonLoadingStream() throws Exception {
- File testConfigFile = new File("tmp." + System.currentTimeMillis() + ".properties");
- if (testConfigFile.exists()) {
- testConfigFile.delete();
- }
- final String zkServer = "z1.example.com,z2.example.com,z3.example.com";
- PrintWriter printWriter = new PrintWriter(new OutputStreamWriter(new FileOutputStream(testConfigFile)));
- printWriter.println("zookeeperServers=" + zkServer);
- printWriter.println("configurationStoreServers=gz1.example.com,gz2.example.com,gz3.example.com/foo");
- printWriter.println("brokerDeleteInactiveTopicsEnabled=true");
- printWriter.println("statusFilePath=/tmp/status.html");
- printWriter.println("managedLedgerDefaultEnsembleSize=1");
- printWriter.println("backlogQuotaDefaultLimitGB=18");
- printWriter.println("clusterName=usc");
- printWriter.println("brokerClientAuthenticationPlugin=test.xyz.client.auth.plugin");
- printWriter.println("brokerClientAuthenticationParameters=role:my-role");
- printWriter.println("superUserRoles=appid1,appid2");
- printWriter.println("brokerServicePort=7777");
- printWriter.println("managedLedgerDefaultMarkDeleteRateLimit=5.0");
- printWriter.println("managedLedgerDigestType=CRC32C");
- printWriter.close();
- testConfigFile.deleteOnExit();
- InputStream stream = new FileInputStream(testConfigFile);
- final ServiceConfiguration serviceConfig = PulsarConfigurationLoader.create(stream, ServiceConfiguration.class);
- assertNotNull(serviceConfig);
- assertEquals(serviceConfig.getZookeeperServers(), zkServer);
- assertEquals(serviceConfig.isBrokerDeleteInactiveTopicsEnabled(), true);
- assertEquals(serviceConfig.getBacklogQuotaDefaultLimitGB(), 18);
- assertEquals(serviceConfig.getClusterName(), "usc");
- assertEquals(serviceConfig.getBrokerClientAuthenticationParameters(), "role:my-role");
- assertEquals(serviceConfig.getBrokerServicePort(), 7777);
- assertEquals(serviceConfig.getManagedLedgerDigestType(), DigestType.CRC32C);
- }
-
- @Test
- public void testPulsarConfiguraitonLoadingProp() throws Exception {
- final String zk = "localhost:2184";
- final Properties prop = new Properties();
- prop.setProperty("zookeeperServers", zk);
- final ServiceConfiguration serviceConfig = PulsarConfigurationLoader.create(prop, ServiceConfiguration.class);
- assertNotNull(serviceConfig);
- assertEquals(serviceConfig.getZookeeperServers(), zk);
- }
-
- @Test
- public void testPulsarConfiguraitonComplete() throws Exception {
- final String zk = "localhost:2184";
- final Properties prop = new Properties();
- prop.setProperty("zookeeperServers", zk);
- final ServiceConfiguration serviceConfig = PulsarConfigurationLoader.create(prop, ServiceConfiguration.class);
- try {
- isComplete(serviceConfig);
- fail("it should fail as config is not complete");
- } catch (IllegalArgumentException e) {
- // Ok
- }
- }
-
- @Test
- public void testComplete() throws Exception {
- TestCompleteObject complete = this.new TestCompleteObject();
- assertTrue(isComplete(complete));
- }
-
- @Test
- public void testInComplete() throws IllegalAccessException {
-
- try {
- isComplete(this.new TestInCompleteObjectRequired());
- fail("Should fail w/ illegal argument exception");
- } catch (IllegalArgumentException iae) {
- // OK, expected
- }
-
- try {
- isComplete(this.new TestInCompleteObjectMin());
- fail("Should fail w/ illegal argument exception");
- } catch (IllegalArgumentException iae) {
- // OK, expected
- }
-
- try {
- isComplete(this.new TestInCompleteObjectMax());
- fail("Should fail w/ illegal argument exception");
- } catch (IllegalArgumentException iae) {
- // OK, expected
- }
-
- try {
- isComplete(this.new TestInCompleteObjectMix());
- fail("Should fail w/ illegal argument exception");
- } catch (IllegalArgumentException iae) {
- // OK, expected
- }
- }
-
- class TestCompleteObject {
- @FieldContext(required = true)
- String required = "I am not null";
- @FieldContext(required = false)
- String optional;
- @FieldContext
- String optional2;
- @FieldContext(minValue = 1)
- int minValue = 2;
- @FieldContext(minValue = 1, maxValue = 3)
- int minMaxValue = 2;
-
- }
-
- class TestInCompleteObjectRequired {
- @FieldContext(required = true)
- String inValidRequired;
- }
-
- class TestInCompleteObjectMin {
- @FieldContext(minValue = 1, maxValue = 3)
- long inValidMin = 0;
- }
-
- class TestInCompleteObjectMax {
- @FieldContext(minValue = 1, maxValue = 3)
- long inValidMax = 4;
- }
-
- class TestInCompleteObjectMix {
- @FieldContext(required = true)
- String inValidRequired;
- @FieldContext(minValue = 1, maxValue = 3)
- long inValidMin = 0;
- @FieldContext(minValue = 1, maxValue = 3)
- long inValidMax = 4;
- }
-}
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/PulsarBrokerStarterTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/PulsarBrokerStarterTest.java
deleted file mode 100644
index 9cce6db0b0..0000000000
--- a/pulsar-broker/src/test/java/org/apache/pulsar/PulsarBrokerStarterTest.java
+++ /dev/null
@@ -1,344 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pulsar;
-
-import static org.testng.Assert.fail;
-
-import com.google.common.collect.Sets;
-import java.io.File;
-import java.io.FileNotFoundException;
-import java.io.FileOutputStream;
-import java.io.IOException;
-import java.io.OutputStreamWriter;
-import java.io.PrintWriter;
-import java.lang.reflect.InvocationTargetException;
-import java.lang.reflect.Method;
-import org.apache.bookkeeper.bookie.LedgerDirsManager;
-import org.apache.pulsar.broker.ServiceConfiguration;
-import org.testng.Assert;
-import org.testng.annotations.Test;
-
-/**
- * @version $Revision$<br>
- * Created on Sep 6, 2012
- */
-public class PulsarBrokerStarterTest {
-
- private File createValidBrokerConfigFile() throws FileNotFoundException {
- File testConfigFile = new File("tmp." + System.currentTimeMillis() + ".properties");
- if (testConfigFile.exists()) {
- testConfigFile.delete();
- }
- PrintWriter printWriter = new PrintWriter(new OutputStreamWriter(new FileOutputStream(testConfigFile)));
- printWriter.println("zookeeperServers=z1.example.com,z2.example.com,z3.example.com");
- printWriter.println("configurationStoreServers=gz1.example.com,gz2.example.com,gz3.example.com/foo");
- printWriter.println("brokerDeleteInactiveTopicsEnabled=false");
- printWriter.println("statusFilePath=/tmp/status.html");
- printWriter.println("managedLedgerDefaultEnsembleSize=1");
- printWriter.println("managedLedgerDefaultWriteQuorum=1");
- printWriter.println("managedLedgerDefaultAckQuorum=1");
- printWriter.println("managedLedgerMaxEntriesPerLedger=25");
- printWriter.println("managedLedgerCursorMaxEntriesPerLedger=50");
- printWriter.println("managedLedgerCursorRolloverTimeInSeconds=3000");
- printWriter.println("bookkeeperClientHealthCheckEnabled=true");
- printWriter.println("bookkeeperClientHealthCheckErrorThresholdPerInterval=5");
- printWriter.println("bookkeeperClientRackawarePolicyEnabled=true");
- printWriter.println("bookkeeperClientIsolationGroups=group1,group2");
- printWriter.println("backlogQuotaDefaultLimitGB=18");
- printWriter.println("clusterName=usc");
- printWriter.println("brokerClientAuthenticationPlugin=test.xyz.client.auth.plugin");
- printWriter.println("brokerClientAuthenticationParameters=role:my-role");
- printWriter.println("superUserRoles=appid1,appid2");
- printWriter.println("clientLibraryVersionCheckEnabled=true");
- printWriter.println("clientLibraryVersionCheckAllowUnversioned=true");
- printWriter.println("managedLedgerMinLedgerRolloverTimeMinutes=34");
- printWriter.println("managedLedgerMaxLedgerRolloverTimeMinutes=34");
- printWriter.println("brokerServicePort=7777");
- printWriter.println("managedLedgerDefaultMarkDeleteRateLimit=5.0");
- printWriter.println("replicationProducerQueueSize=50");
- printWriter.println("bookkeeperClientTimeoutInSeconds=12345");
- printWriter.println("bookkeeperClientSpeculativeReadTimeoutInMillis=3000");
- printWriter.println("enableRunBookieTogether=true");
-
- printWriter.close();
- testConfigFile.deleteOnExit();
- return testConfigFile;
- }
-
- /**
- * Tests the private static <code>loadConfig</code> method of {@link PulsarBrokerStarter} class: verifies (1) if the
- * method returns a non-null {@link ServiceConfiguration} instance where all required settings are filled in and (2)
- * if the property variables inside the given property file are correctly referred to that returned object.
- */
- @Test
- public void testLoadConfig() throws SecurityException, NoSuchMethodException, IOException, IllegalArgumentException,
- IllegalAccessException, InvocationTargetException {
-
- File testConfigFile = createValidBrokerConfigFile();
- Method targetMethod = PulsarBrokerStarter.class.getDeclaredMethod("loadConfig", String.class);
- targetMethod.setAccessible(true);
-
- Object returnValue = targetMethod.invoke(PulsarBrokerStarter.class, testConfigFile.getAbsolutePath());
-
- Assert.assertTrue(ServiceConfiguration.class.isInstance(returnValue));
- ServiceConfiguration serviceConfig = (ServiceConfiguration) returnValue;
- Assert.assertEquals(serviceConfig.getZookeeperServers(), "z1.example.com,z2.example.com,z3.example.com");
- Assert.assertEquals(serviceConfig.getConfigurationStoreServers(),
- "gz1.example.com,gz2.example.com,gz3.example.com/foo");
- Assert.assertFalse(serviceConfig.isBrokerDeleteInactiveTopicsEnabled());
- Assert.assertEquals(serviceConfig.getStatusFilePath(), "/tmp/status.html");
- Assert.assertEquals(serviceConfig.getBacklogQuotaDefaultLimitGB(), 18);
- Assert.assertEquals(serviceConfig.getClusterName(), "usc");
- Assert.assertEquals(serviceConfig.getBrokerClientAuthenticationPlugin(), "test.xyz.client.auth.plugin");
- Assert.assertEquals(serviceConfig.getBrokerClientAuthenticationParameters(), "role:my-role");
- Assert.assertEquals(serviceConfig.getSuperUserRoles(), Sets.newHashSet("appid1", "appid2"));
- Assert.assertEquals(serviceConfig.getManagedLedgerCursorRolloverTimeInSeconds(), 3000);
- Assert.assertEquals(serviceConfig.getManagedLedgerMaxEntriesPerLedger(), 25);
- Assert.assertEquals(serviceConfig.getManagedLedgerCursorMaxEntriesPerLedger(), 50);
- Assert.assertTrue(serviceConfig.isClientLibraryVersionCheckEnabled());
- Assert.assertEquals(serviceConfig.getManagedLedgerMinLedgerRolloverTimeMinutes(), 34);
- Assert.assertEquals(serviceConfig.isBacklogQuotaCheckEnabled(), true);
- Assert.assertEquals(serviceConfig.getManagedLedgerDefaultMarkDeleteRateLimit(), 5.0);
- Assert.assertEquals(serviceConfig.getReplicationProducerQueueSize(), 50);
- Assert.assertEquals(serviceConfig.isReplicationMetricsEnabled(), false);
- Assert.assertEquals(serviceConfig.isBookkeeperClientHealthCheckEnabled(), true);
- Assert.assertEquals(serviceConfig.getBookkeeperClientHealthCheckErrorThresholdPerInterval(), 5);
- Assert.assertEquals(serviceConfig.isBookkeeperClientRackawarePolicyEnabled(), true);
- Assert.assertEquals(serviceConfig.getBookkeeperClientIsolationGroups(), "group1,group2");
- Assert.assertEquals(serviceConfig.getBookkeeperClientSpeculativeReadTimeoutInMillis(), 3000);
- Assert.assertEquals(serviceConfig.getBookkeeperClientTimeoutInSeconds(), 12345);
- }
-
- @Test
- public void testLoadConfigWithException() throws Exception {
-
- File testConfigFile = new File("tmp." + System.currentTimeMillis() + ".properties");
- if (testConfigFile.exists()) {
- testConfigFile.delete();
- }
- PrintWriter printWriter = new PrintWriter(new OutputStreamWriter(new FileOutputStream(testConfigFile)));
- printWriter.println("brokerDeleteInactiveTopicsEnabled=false");
- printWriter.println("statusFilePath=/tmp/pulsar_broker/status.html");
- printWriter.println("managedLedgerDefaultEnsembleSize01");
- printWriter.println("managedLedgerDefaultWriteQuorum=0");
- printWriter.println("managedLedgerDefaultAckQuorum=0");
-
- printWriter.close();
- testConfigFile.deleteOnExit();
-
- try {
- Method targetMethod = PulsarBrokerStarter.class.getDeclaredMethod("loadConfig", String.class);
- targetMethod.setAccessible(true);
- targetMethod.invoke(PulsarBrokerStarter.class, testConfigFile.getAbsolutePath());
- fail("Should fail w/ illegal argument exception");
- } catch (InvocationTargetException e) {
- // OK, expected
- Assert.assertTrue(e.getTargetException() instanceof IllegalArgumentException);
- }
- }
-
- /**
- * Tests the private static <code>loadConfig</code> method of {@link PulsarBrokerStarter} class: verifies (1) if the
- * method returns a non-null {@link ServiceConfiguration} instance where all required settings are filled in and (2)
- * if the property variables inside the given property file are correctly referred to that returned object.
- */
- @Test
- public void testLoadBalancerConfig() throws SecurityException, NoSuchMethodException, IOException,
- IllegalArgumentException, IllegalAccessException, InvocationTargetException {
-
- File testConfigFile = new File("tmp." + System.currentTimeMillis() + ".properties");
- if (testConfigFile.exists()) {
- testConfigFile.delete();
- }
-
- PrintWriter printWriter = new PrintWriter(new OutputStreamWriter(new FileOutputStream(testConfigFile)));
- printWriter.println("zookeeperServers=z1.example.com,z2.example.com,z3.example.com");
- printWriter.println("statusFilePath=/usr/share/pulsar_broker/status.html");
- printWriter.println("clusterName=test");
- printWriter.println("managedLedgerDefaultEnsembleSize=1");
- printWriter.println("managedLedgerDefaultWriteQuorum=1");
- printWriter.println("managedLedgerDefaultAckQuorum=1");
-
- printWriter.println("loadBalancerEnabled=false");
- printWriter.println("loadBalancerHostUsageCheckIntervalMinutes=4");
- printWriter.println("loadBalancerReportUpdateThresholdPercentage=15");
- printWriter.println("loadBalancerReportUpdateMaxIntervalMinutes=20");
- printWriter.println("loadBalancerBrokerOverloadedThresholdPercentage=80");
- printWriter.println("loadBalancerBrokerUnderloadedThresholdPercentage=40");
- printWriter.println("loadBalancerSheddingIntervalMinutes=8");
- printWriter.println("loadBalancerSheddingGracePeriodMinutes=29");
- printWriter.close();
- testConfigFile.deleteOnExit();
-
- Method targetMethod = PulsarBrokerStarter.class.getDeclaredMethod("loadConfig", String.class);
- targetMethod.setAccessible(true);
- Object returnValue = targetMethod.invoke(PulsarBrokerStarter.class, testConfigFile.getAbsolutePath());
-
- Assert.assertTrue(ServiceConfiguration.class.isInstance(returnValue));
- ServiceConfiguration serviceConfig = (ServiceConfiguration) returnValue;
- Assert.assertEquals(serviceConfig.isLoadBalancerEnabled(), false);
- Assert.assertEquals(serviceConfig.getLoadBalancerHostUsageCheckIntervalMinutes(), 4);
- Assert.assertEquals(serviceConfig.getLoadBalancerReportUpdateThresholdPercentage(), 15);
- Assert.assertEquals(serviceConfig.getLoadBalancerReportUpdateMaxIntervalMinutes(), 20);
- Assert.assertEquals(serviceConfig.getLoadBalancerBrokerOverloadedThresholdPercentage(), 80);
- Assert.assertEquals(serviceConfig.getLoadBalancerBrokerUnderloadedThresholdPercentage(), 40);
- Assert.assertEquals(serviceConfig.getLoadBalancerSheddingIntervalMinutes(), 8);
- Assert.assertEquals(serviceConfig.getLoadBalancerSheddingGracePeriodMinutes(), 29);
- }
-
- /**
- * Tests the private static <code>loadConfig</code> method of {@link PulsarBrokerStarter} class: verifies (1) if the
- * method returns a non-null {@link ServiceConfiguration} instance where all required settings are filled in and (2)
- * if the property variables inside the given property file are correctly referred to that returned object.
- */
- @Test
- public void testGlobalZooKeeperConfig() throws SecurityException, NoSuchMethodException, IOException,
- IllegalArgumentException, IllegalAccessException, InvocationTargetException {
-
- File testConfigFile = new File("tmp." + System.currentTimeMillis() + ".properties");
- if (testConfigFile.exists()) {
- testConfigFile.delete();
- }
- PrintWriter printWriter = new PrintWriter(new OutputStreamWriter(new FileOutputStream(testConfigFile)));
- printWriter.println("zookeeperServers=z1.example.com,z2.example.com,z3.example.com");
- printWriter.println("configurationStoreServers=");
- printWriter.println("brokerDeleteInactiveTopicsEnabled=false");
- printWriter.println("statusFilePath=/tmp/status.html");
- printWriter.println("managedLedgerDefaultEnsembleSize=1");
- printWriter.println("managedLedgerDefaultWriteQuorum=1");
- printWriter.println("managedLedgerDefaultAckQuorum=1");
- printWriter.println("managedLedgerMaxEntriesPerLedger=25");
- printWriter.println("managedLedgerCursorMaxEntriesPerLedger=50");
- printWriter.println("managedLedgerCursorRolloverTimeInSeconds=3000");
- printWriter.println("backlogQuotaDefaultLimitGB=18");
- printWriter.println("clusterName=usc");
- printWriter.println("brokerClientAuthenticationPlugin=test.xyz.client.auth.plugin");
- printWriter.println("brokerClientAuthenticationParameters=role:my-role");
- printWriter.println("superUserRoles=appid1,appid2");
- printWriter.println("pulsar.broker.enableClientVersionCheck=true");
- printWriter.println("pulsar.broker.allowUnversionedClients=true");
- printWriter.println("clientLibraryVersionCheckEnabled=true");
- printWriter.println("clientLibraryVersionCheckAllowUnversioned=true");
- printWriter.println("replicationConnectionsPerBroker=12");
-
- printWriter.close();
- testConfigFile.deleteOnExit();
-
- Method targetMethod = PulsarBrokerStarter.class.getDeclaredMethod("loadConfig", String.class);
- targetMethod.setAccessible(true);
-
- Object returnValue = targetMethod.invoke(PulsarBrokerStarter.class, testConfigFile.getAbsolutePath());
-
- Assert.assertTrue(ServiceConfiguration.class.isInstance(returnValue));
- ServiceConfiguration serviceConfig = (ServiceConfiguration) returnValue;
- Assert.assertEquals(serviceConfig.getZookeeperServers(), "z1.example.com,z2.example.com,z3.example.com");
- Assert.assertEquals(serviceConfig.getConfigurationStoreServers(), "z1.example.com,z2.example.com,z3.example.com");
- Assert.assertFalse(serviceConfig.isBrokerDeleteInactiveTopicsEnabled());
- Assert.assertEquals(serviceConfig.getStatusFilePath(), "/tmp/status.html");
- Assert.assertEquals(serviceConfig.getBacklogQuotaDefaultLimitGB(), 18);
- Assert.assertEquals(serviceConfig.getClusterName(), "usc");
- Assert.assertEquals(serviceConfig.getBrokerClientAuthenticationPlugin(), "test.xyz.client.auth.plugin");
- Assert.assertEquals(serviceConfig.getBrokerClientAuthenticationParameters(), "role:my-role");
- Assert.assertEquals(serviceConfig.getSuperUserRoles(), Sets.newHashSet("appid1", "appid2"));
- Assert.assertEquals(serviceConfig.getManagedLedgerCursorRolloverTimeInSeconds(), 3000);
- Assert.assertEquals(serviceConfig.getManagedLedgerMaxEntriesPerLedger(), 25);
- Assert.assertEquals(serviceConfig.getManagedLedgerCursorMaxEntriesPerLedger(), 50);
- Assert.assertTrue(serviceConfig.isClientLibraryVersionCheckEnabled());
- Assert.assertEquals(serviceConfig.getReplicationConnectionsPerBroker(), 12);
- }
-
- /**
- * Verifies that the main throws {@link FileNotFoundException} when no argument is given.
- */
- @Test
- public void testMainWithNoArgument() throws Exception {
- try {
- PulsarBrokerStarter.main(new String[0]);
- Assert.fail("No argument to main should've raised FileNotFoundException for no broker config!");
- } catch (FileNotFoundException e) {
- // code should reach here.
- }
- }
-
- /**
- * Verifies that the main throws {@link IllegalArgumentException}
- * when no config file for bookie and bookie auto recovery is given.
- */
- @Test
- public void testMainRunBookieAndAutoRecoveryNoConfig() throws Exception {
- try {
- File testConfigFile = createValidBrokerConfigFile();
- String[] args = {"-c", testConfigFile.getAbsolutePath(), "-rb", "-ra", "-bc", ""};
- PulsarBrokerStarter.main(args);
- Assert.fail("No Config file for bookie auto recovery should've raised IllegalArgumentException!");
- } catch (IllegalArgumentException e) {
- // code should reach here.
- e.printStackTrace();
- Assert.assertEquals(e.getMessage(), "No configuration file for Bookie");
- }
- }
-
- /**
- * Verifies that the main throws {@link IllegalArgumentException}
- * when no config file for bookie auto recovery is given.
- */
- @Test
- public void testMainRunBookieRecoveryNoConfig() throws Exception {
- try {
- File testConfigFile = createValidBrokerConfigFile();
- String[] args = {"-c", testConfigFile.getAbsolutePath(), "-ra", "-bc", ""};
- PulsarBrokerStarter.main(args);
- Assert.fail("No Config file for bookie auto recovery should've raised IllegalArgumentException!");
- } catch (IllegalArgumentException e) {
- // code should reach here.
- Assert.assertEquals(e.getMessage(), "No configuration file for Bookie");
- }
- }
-
- /**
- * Verifies that the main throws {@link IllegalArgumentException} when no config file for bookie is given.
- */
- @Test
- public void testMainRunBookieNoConfig() throws Exception {
- try {
- File testConfigFile = createValidBrokerConfigFile();
- String[] args = {"-c", testConfigFile.getAbsolutePath(), "-rb", "-bc", ""};
- PulsarBrokerStarter.main(args);
- Assert.fail("No Config file for bookie should've raised IllegalArgumentException!");
- } catch (IllegalArgumentException e) {
- // code should reach here
- Assert.assertEquals(e.getMessage(), "No configuration file for Bookie");
- }
- }
-
- /**
- * Verifies that the main throws {@link IllegalArgumentException} when no config file for bookie .
- */
- @Test
- public void testMainEnableRunBookieThroughBrokerConfig() throws Exception {
- try {
- File testConfigFile = createValidBrokerConfigFile();
- String[] args = {"-c", testConfigFile.getAbsolutePath()};
- PulsarBrokerStarter.main(args);
- Assert.fail("No argument to main should've raised IllegalArgumentException for no bookie config!");
- } catch (IllegalArgumentException e) {
- // code should reach here.
- }
- }
-}
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/SLAMonitoringTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/SLAMonitoringTest.java
deleted file mode 100644
index 233d6b7b36..0000000000
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/SLAMonitoringTest.java
+++ /dev/null
@@ -1,245 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pulsar.broker;
-
-import static org.testng.Assert.assertEquals;
-import static org.testng.Assert.assertNotEquals;
-import static org.testng.Assert.assertTrue;
-import static org.testng.Assert.fail;
-
-import java.net.MalformedURLException;
-import java.net.URL;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.LinkedBlockingQueue;
-import java.util.concurrent.ThreadPoolExecutor;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.bookkeeper.test.PortManager;
-import org.apache.pulsar.broker.loadbalance.LoadBalancerTest;
-import org.apache.pulsar.broker.namespace.NamespaceService;
-import org.apache.pulsar.client.admin.PulsarAdmin;
-import org.apache.pulsar.client.admin.PulsarAdminException;
-import org.apache.pulsar.client.api.PulsarClientException;
-import org.apache.pulsar.common.policies.data.ClusterData;
-import org.apache.pulsar.common.policies.data.NamespaceOwnershipStatus;
-import org.apache.pulsar.common.policies.data.TenantInfo;
-import org.apache.pulsar.zookeeper.LocalBookkeeperEnsemble;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.Assert;
-import org.testng.annotations.AfterClass;
-import org.testng.annotations.BeforeClass;
-import org.testng.annotations.Test;
-
-public class SLAMonitoringTest {
- LocalBookkeeperEnsemble bkEnsemble;
-
- ExecutorService executor = new ThreadPoolExecutor(5, 20, 30, TimeUnit.SECONDS, new LinkedBlockingQueue<Runnable>());
-
- private static final Logger log = LoggerFactory.getLogger(LoadBalancerTest.class);
-
- private final int ZOOKEEPER_PORT = PortManager.nextFreePort();
-
- private static final int BROKER_COUNT = 5;
- private int[] brokerWebServicePorts = new int[BROKER_COUNT];
- private int[] brokerNativeBrokerPorts = new int[BROKER_COUNT];
- private URL[] brokerUrls = new URL[BROKER_COUNT];
- private PulsarService[] pulsarServices = new PulsarService[BROKER_COUNT];
- private PulsarAdmin[] pulsarAdmins = new PulsarAdmin[BROKER_COUNT];
- private ServiceConfiguration[] configurations = new ServiceConfiguration[BROKER_COUNT];
-
- @BeforeClass
- void setup() throws Exception {
- log.info("---- Initializing SLAMonitoringTest -----");
- // Start local bookkeeper ensemble
- bkEnsemble = new LocalBookkeeperEnsemble(3, ZOOKEEPER_PORT, () -> PortManager.nextFreePort());
- bkEnsemble.start();
-
- // start brokers
- for (int i = 0; i < BROKER_COUNT; i++) {
- brokerWebServicePorts[i] = PortManager.nextFreePort();
- brokerNativeBrokerPorts[i] = PortManager.nextFreePort();
-
- ServiceConfiguration config = new ServiceConfiguration();
- config.setBrokerServicePort(brokerNativeBrokerPorts[i]);
- config.setClusterName("my-cluster");
- config.setAdvertisedAddress("localhost");
- config.setWebServicePort(brokerWebServicePorts[i]);
- config.setZookeeperServers("127.0.0.1" + ":" + ZOOKEEPER_PORT);
- config.setBrokerServicePort(brokerNativeBrokerPorts[i]);
- config.setDefaultNumberOfNamespaceBundles(1);
- config.setLoadBalancerEnabled(false);
- configurations[i] = config;
-
- pulsarServices[i] = new PulsarService(config);
- pulsarServices[i].start();
-
- brokerUrls[i] = new URL("http://127.0.0.1" + ":" + brokerWebServicePorts[i]);
- pulsarAdmins[i] = PulsarAdmin.builder().serviceHttpUrl(brokerUrls[i].toString()).build();
- }
-
- Thread.sleep(100);
-
- createTenant(pulsarAdmins[BROKER_COUNT - 1]);
- for (int i = 0; i < BROKER_COUNT; i++) {
- String topic = String.format("%s/%s/%s:%s", NamespaceService.SLA_NAMESPACE_PROPERTY, "my-cluster",
- pulsarServices[i].getAdvertisedAddress(), brokerWebServicePorts[i]);
- pulsarAdmins[0].namespaces().createNamespace(topic);
- }
- }
-
- private void createTenant(PulsarAdmin pulsarAdmin)
- throws PulsarClientException, MalformedURLException, PulsarAdminException {
- ClusterData clusterData = new ClusterData();
- clusterData.setServiceUrl(pulsarAdmin.getServiceUrl());
- pulsarAdmins[0].clusters().createCluster("my-cluster", clusterData);
- Set<String> allowedClusters = new HashSet<>();
- allowedClusters.add("my-cluster");
- TenantInfo adminConfig = new TenantInfo();
- adminConfig.setAllowedClusters(allowedClusters);
- Set<String> adminRoles = new HashSet<>();
- adminRoles.add("");
- adminConfig.setAdminRoles(adminRoles);
- pulsarAdmin.tenants().createTenant("sla-monitor", adminConfig);
- }
-
- @AfterClass
- void shutdown() throws Exception {
- log.info("--- Shutting down ---");
- executor.shutdown();
-
- for (int i = 0; i < BROKER_COUNT; i++) {
- pulsarAdmins[i].close();
- pulsarServices[i].close();
- }
-
- bkEnsemble.stop();
- }
-
- @Test
- public void testOwnershipAfterSetup() {
- for (int i = 0; i < BROKER_COUNT; i++) {
- try {
- assertTrue(pulsarServices[0].getNamespaceService().registerSLANamespace());
- } catch (PulsarServerException e) {
- e.printStackTrace();
- log.error("Exception occured", e);
- fail("SLA Namespace should have been owned by the broker, Exception.", e);
- }
- }
- }
-
- @Test
- public void testOwnedNamespaces() {
- testOwnershipViaAdminAfterSetup();
- try {
- for (int i = 0; i < BROKER_COUNT; i++) {
- List<String> list = pulsarAdmins[i].brokers().getActiveBrokers("my-cluster");
- Assert.assertNotNull(list);
- Assert.assertEquals(list.size(), BROKER_COUNT);
-
- Map<String, NamespaceOwnershipStatus> nsMap = pulsarAdmins[i].brokers().getOwnedNamespaces("my-cluster",
- list.get(0));
- Assert.assertEquals(2, nsMap.size());
- }
- } catch (Exception e) {
- e.printStackTrace();
- fail("Hearbeat namespace and SLA namespace should be owned by the broker");
- }
- }
-
- @Test
- public void testOwnershipViaAdminAfterSetup() {
- for (int i = 0; i < BROKER_COUNT; i++) {
- try {
- String topic = String.format("persistent://%s/%s/%s:%s/%s",
- NamespaceService.SLA_NAMESPACE_PROPERTY, "my-cluster", pulsarServices[i].getAdvertisedAddress(),
- brokerWebServicePorts[i], "my-topic");
- assertEquals(pulsarAdmins[0].lookups().lookupTopic(topic),
- "pulsar://" + pulsarServices[i].getAdvertisedAddress() + ":" + brokerNativeBrokerPorts[i]);
- } catch (Exception e) {
- e.printStackTrace();
- fail("SLA Namespace should have been owned by the broker(" + "pulsar://" + pulsarServices[i].getAdvertisedAddress()
- + ":" + brokerNativeBrokerPorts[i] + ")");
- }
- }
- }
-
- @Test
- public void testUnloadIfBrokerCrashes() {
- int crashIndex = BROKER_COUNT / 2;
- log.info("Trying to close the broker at index = {}", crashIndex);
-
- try {
- pulsarServices[crashIndex].close();
- } catch (PulsarServerException e) {
- e.printStackTrace();
- fail("Should be a able to close the broker index " + crashIndex + " Exception: " + e);
- }
-
- String topic = String.format("persistent://%s/%s/%s:%s/%s", NamespaceService.SLA_NAMESPACE_PROPERTY,
- "my-cluster", pulsarServices[crashIndex].getAdvertisedAddress(), brokerWebServicePorts[crashIndex], "my-topic");
-
- log.info("Lookup for namespace {}", topic);
-
- String broker = null;
- try {
- broker = pulsarAdmins[BROKER_COUNT - 1].lookups().lookupTopic(topic);
- log.info("{} Namespace is owned by {}", topic, broker);
- assertNotEquals(broker,
- "pulsar://" + pulsarServices[crashIndex].getAdvertisedAddress() + ":" + brokerNativeBrokerPorts[crashIndex]);
- } catch (PulsarAdminException e) {
- e.printStackTrace();
- fail("The SLA Monitor namespace should be owned by some other broker");
- }
-
- // Check if the namespace is properly unloaded and reowned by the broker
- try {
- pulsarServices[crashIndex] = new PulsarService(configurations[crashIndex]);
- pulsarServices[crashIndex].start();
- assertEquals(pulsarServices[crashIndex].getConfiguration().getBrokerServicePort(),
- brokerNativeBrokerPorts[crashIndex]);
- } catch (PulsarServerException e) {
- e.printStackTrace();
- fail("The broker should be able to start without exception");
- }
-
- try {
- broker = pulsarAdmins[0].lookups().lookupTopic(topic);
- log.info("{} Namespace is re-owned by {}", topic, broker);
- assertEquals(broker,
- "pulsar://" + pulsarServices[crashIndex].getAdvertisedAddress() + ":" + brokerNativeBrokerPorts[crashIndex]);
- } catch (PulsarAdminException e) {
- e.printStackTrace();
- fail("The SLA Monitor namespace should be reowned by the broker" + broker);
- }
-
- try {
- pulsarServices[crashIndex].close();
- } catch (PulsarServerException e) {
- e.printStackTrace();
- fail("The broker should be able to stop without exception");
- }
- }
-
-}
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiOffloadTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiOffloadTest.java
deleted file mode 100644
index 63b4d84c1b..0000000000
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiOffloadTest.java
+++ /dev/null
@@ -1,151 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pulsar.broker.admin;
-
-import static org.mockito.Matchers.anyObject;
-import static org.mockito.Mockito.doReturn;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.when;
-
-import com.google.common.collect.Sets;
-
-import java.util.concurrent.CompletableFuture;
-
-import lombok.extern.slf4j.Slf4j;
-
-import org.apache.bookkeeper.mledger.ManagedLedgerInfo;
-import org.apache.bookkeeper.mledger.LedgerOffloader;
-import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest;
-import org.apache.pulsar.client.admin.LongRunningProcessStatus;
-import org.apache.pulsar.client.admin.PulsarAdminException.ConflictException;
-import org.apache.pulsar.client.api.MessageId;
-import org.apache.pulsar.client.api.Producer;
-import org.apache.pulsar.client.api.PulsarClient;
-import org.apache.pulsar.client.impl.MessageIdImpl;
-import org.apache.pulsar.common.naming.TopicName;
-import org.apache.pulsar.common.policies.data.ClusterData;
-import org.apache.pulsar.common.policies.data.TenantInfo;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.Assert;
-import org.testng.annotations.AfterMethod;
-import org.testng.annotations.BeforeMethod;
-import org.testng.annotations.Test;
-
-@Slf4j
-public class AdminApiOffloadTest extends MockedPulsarServiceBaseTest {
-
- private static final Logger LOG = LoggerFactory.getLogger(AdminApiOffloadTest.class);
-
- @BeforeMethod
- @Override
- public void setup() throws Exception {
- conf.setManagedLedgerMaxEntriesPerLedger(10);
- conf.setManagedLedgerMinLedgerRolloverTimeMinutes(0);
-
- super.internalSetup();
-
- // Setup namespaces
- admin.clusters().createCluster("test", new ClusterData("http://127.0.0.1" + ":" + BROKER_WEBSERVICE_PORT));
- TenantInfo tenantInfo = new TenantInfo(Sets.newHashSet("role1", "role2"), Sets.newHashSet("test"));
- admin.tenants().createTenant("prop-xyz", tenantInfo);
- admin.namespaces().createNamespace("prop-xyz/ns1", Sets.newHashSet("test"));
- }
-
- @AfterMethod
- @Override
- public void cleanup() throws Exception {
- super.internalCleanup();
- }
-
- private void testOffload(String topicName, String mlName) throws Exception {
- LedgerOffloader offloader = mock(LedgerOffloader.class);
- when(offloader.getOffloadDriverName()).thenReturn("mock");
-
- doReturn(offloader).when(pulsar).getManagedLedgerOffloader();
-
- CompletableFuture<Void> promise = new CompletableFuture<>();
- doReturn(promise).when(offloader).offload(anyObject(), anyObject(), anyObject());
-
- MessageId currentId = MessageId.latest;
- try (Producer p = pulsarClient.newProducer().topic(topicName).enableBatching(false).create()) {
- for (int i = 0; i < 15; i++) {
- currentId = p.send("Foobar".getBytes());
- }
- }
-
- ManagedLedgerInfo info = pulsar.getManagedLedgerFactory().getManagedLedgerInfo(mlName);
- Assert.assertEquals(info.ledgers.size(), 2);
-
- Assert.assertEquals(admin.persistentTopics().offloadStatus(topicName).status,
- LongRunningProcessStatus.Status.NOT_RUN);
-
- admin.persistentTopics().triggerOffload(topicName, currentId);
-
- Assert.assertEquals(admin.persistentTopics().offloadStatus(topicName).status,
- LongRunningProcessStatus.Status.RUNNING);
-
- try {
- admin.persistentTopics().triggerOffload(topicName, currentId);
- Assert.fail("Should have failed");
- } catch (ConflictException e) {
- // expected
- }
-
- // fail first time
- promise.completeExceptionally(new Exception("Some random failure"));
-
- Assert.assertEquals(admin.persistentTopics().offloadStatus(topicName).status,
- LongRunningProcessStatus.Status.ERROR);
- Assert.assertTrue(admin.persistentTopics().offloadStatus(topicName).lastError.contains("Some random failure"));
-
- // Try again
- doReturn(CompletableFuture.completedFuture(null))
- .when(offloader).offload(anyObject(), anyObject(), anyObject());
-
- admin.persistentTopics().triggerOffload(topicName, currentId);
-
- Assert.assertEquals(admin.persistentTopics().offloadStatus(topicName).status,
- LongRunningProcessStatus.Status.SUCCESS);
- MessageIdImpl firstUnoffloaded = admin.persistentTopics().offloadStatus(topicName).firstUnoffloadedMessage;
- // First unoffloaded is the first entry of current ledger
- Assert.assertEquals(firstUnoffloaded.getLedgerId(), info.ledgers.get(1).ledgerId);
- Assert.assertEquals(firstUnoffloaded.getEntryId(), 0);
-
- verify(offloader, times(2)).offload(anyObject(), anyObject(), anyObject());
- }
-
-
- @Test
- public void testOffloadV2() throws Exception {
- String topicName = "persistent://prop-xyz/ns1/topic1";
- String mlName = "prop-xyz/ns1/persistent/topic1";
- testOffload(topicName, mlName);
- }
-
- @Test
- public void testOffloadV1() throws Exception {
- String topicName = "persistent://prop-xyz/test/ns1/topic2";
- String mlName = "prop-xyz/test/ns1/persistent/topic2";
- testOffload(topicName, mlName);
- }
-}
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiSchemaAutoUpdateTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiSchemaAutoUpdateTest.java
deleted file mode 100644
index 5060c8867f..0000000000
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiSchemaAutoUpdateTest.java
+++ /dev/null
@@ -1,300 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pulsar.broker.admin;
-
-import org.apache.avro.reflect.AvroAlias;
-import org.apache.avro.reflect.AvroDefault;
-import com.google.common.collect.Sets;
-
-import java.lang.reflect.Field;
-
-import lombok.extern.slf4j.Slf4j;
-
-import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest;
-import org.apache.pulsar.broker.service.Topic;
-import org.apache.pulsar.broker.service.schema.SchemaCompatibilityStrategy;
-import org.apache.pulsar.client.api.Producer;
-import org.apache.pulsar.client.api.PulsarClient;
-import org.apache.pulsar.client.api.PulsarClientException;
-import org.apache.pulsar.client.api.Schema;
-import org.apache.pulsar.common.naming.TopicName;
-import org.apache.pulsar.common.policies.data.ClusterData;
-import org.apache.pulsar.common.policies.data.TenantInfo;
-import org.apache.pulsar.common.policies.data.SchemaAutoUpdateCompatibilityStrategy;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.Assert;
-import org.testng.annotations.AfterMethod;
-import org.testng.annotations.BeforeMethod;
-import org.testng.annotations.Test;
-
-@Slf4j
-public class AdminApiSchemaAutoUpdateTest extends MockedPulsarServiceBaseTest {
-
- private static final Logger LOG = LoggerFactory.getLogger(AdminApiSchemaAutoUpdateTest.class);
-
- @BeforeMethod
- @Override
- public void setup() throws Exception {
- super.internalSetup();
-
- // Setup namespaces
- admin.clusters().createCluster("test", new ClusterData("http://127.0.0.1" + ":" + BROKER_WEBSERVICE_PORT));
- TenantInfo tenantInfo = new TenantInfo(Sets.newHashSet("role1", "role2"), Sets.newHashSet("test"));
- admin.tenants().createTenant("prop-xyz", tenantInfo);
- admin.namespaces().createNamespace("prop-xyz/ns1", Sets.newHashSet("test"));
- admin.namespaces().createNamespace("prop-xyz/test/ns1");
- admin.namespaces().createNamespace("prop-xyz/ns2", Sets.newHashSet("test"));
- admin.namespaces().createNamespace("prop-xyz/test/ns2");
- }
-
- @AfterMethod
- @Override
- public void cleanup() throws Exception {
- super.internalCleanup();
- }
-
- private void testAutoUpdateBackward(String namespace, String topicName) throws Exception {
- Assert.assertEquals(admin.namespaces().getSchemaAutoUpdateCompatibilityStrategy(namespace),
- SchemaAutoUpdateCompatibilityStrategy.Full);
- admin.namespaces().setSchemaAutoUpdateCompatibilityStrategy(namespace,
- SchemaAutoUpdateCompatibilityStrategy.Backward);
-
- try (Producer<V1Data> p = pulsarClient.newProducer(Schema.AVRO(V1Data.class)).topic(topicName).create()) {
- p.send(new V1Data("test1", 1));
- }
-
- log.info("try with forward compat, should fail");
- try (Producer<V3Data> p = pulsarClient.newProducer(Schema.AVRO(V3Data.class)).topic(topicName).create()) {
- Assert.fail("Forward compat schema should be rejected");
- } catch (PulsarClientException e) {
- Assert.assertTrue(e.getMessage().contains("IncompatibleSchemaException"));
- }
-
- log.info("try with backward compat, should succeed");
- try (Producer<V2Data> p = pulsarClient.newProducer(Schema.AVRO(V2Data.class)).topic(topicName).create()) {
- p.send(new V2Data("test2"));
- }
-
- }
-
- private void testAutoUpdateForward(String namespace, String topicName) throws Exception {
- Assert.assertEquals(admin.namespaces().getSchemaAutoUpdateCompatibilityStrategy(namespace),
- SchemaAutoUpdateCompatibilityStrategy.Full);
- admin.namespaces().setSchemaAutoUpdateCompatibilityStrategy(namespace,
- SchemaAutoUpdateCompatibilityStrategy.Forward);
-
- try (Producer<V1Data> p = pulsarClient.newProducer(Schema.AVRO(V1Data.class)).topic(topicName).create()) {
- p.send(new V1Data("test1", 1));
- }
-
- log.info("try with backward compat, should fail");
- try (Producer<V2Data> p = pulsarClient.newProducer(Schema.AVRO(V2Data.class)).topic(topicName).create()) {
- Assert.fail("Backward compat schema should be rejected");
- } catch (PulsarClientException e) {
- Assert.assertTrue(e.getMessage().contains("IncompatibleSchemaException"));
- }
-
- log.info("try with forward compat, should succeed");
- try (Producer<V3Data> p = pulsarClient.newProducer(Schema.AVRO(V3Data.class)).topic(topicName).create()) {
- p.send(new V3Data("test2", 1, 2));
- }
- }
-
- private void testAutoUpdateFull(String namespace, String topicName) throws Exception {
- Assert.assertEquals(admin.namespaces().getSchemaAutoUpdateCompatibilityStrategy(namespace),
- SchemaAutoUpdateCompatibilityStrategy.Full);
-
- try (Producer<V1Data> p = pulsarClient.newProducer(Schema.AVRO(V1Data.class)).topic(topicName).create()) {
- p.send(new V1Data("test1", 1));
- }
-
- log.info("try with backward compat only, should fail");
- try (Producer<V2Data> p = pulsarClient.newProducer(Schema.AVRO(V2Data.class)).topic(topicName).create()) {
- Assert.fail("Backward compat only schema should fail");
- } catch (PulsarClientException e) {
- Assert.assertTrue(e.getMessage().contains("IncompatibleSchemaException"));
- }
-
- log.info("try with forward compat only, should fail");
- try (Producer<V3Data> p = pulsarClient.newProducer(Schema.AVRO(V3Data.class)).topic(topicName).create()) {
- Assert.fail("Forward compat only schema should fail");
- } catch (PulsarClientException e) {
- Assert.assertTrue(e.getMessage().contains("IncompatibleSchemaException"));
- }
-
- log.info("try with fully compat");
- try (Producer<V4Data> p = pulsarClient.newProducer(Schema.AVRO(V4Data.class)).topic(topicName).create()) {
- p.send(new V4Data("test2", 1, (short)100));
- }
- }
-
- private void testAutoUpdateDisabled(String namespace, String topicName) throws Exception {
- Assert.assertEquals(admin.namespaces().getSchemaAutoUpdateCompatibilityStrategy(namespace),
- SchemaAutoUpdateCompatibilityStrategy.Full);
- admin.namespaces().setSchemaAutoUpdateCompatibilityStrategy(namespace,
- SchemaAutoUpdateCompatibilityStrategy.AutoUpdateDisabled);
-
- try (Producer<V1Data> p = pulsarClient.newProducer(Schema.AVRO(V1Data.class)).topic(topicName).create()) {
- p.send(new V1Data("test1", 1));
- }
- log.info("try with backward compat only, should fail");
- try (Producer<V2Data> p = pulsarClient.newProducer(Schema.AVRO(V2Data.class)).topic(topicName).create()) {
- Assert.fail("Backward compat only schema should fail");
- } catch (PulsarClientException e) {
- Assert.assertTrue(e.getMessage().contains("IncompatibleSchemaException"));
- }
-
- log.info("try with forward compat only, should fail");
- try (Producer<V3Data> p = pulsarClient.newProducer(Schema.AVRO(V3Data.class)).topic(topicName).create()) {
- Assert.fail("Forward compat only schema should fail");
- } catch (PulsarClientException e) {
- Assert.assertTrue(e.getMessage().contains("IncompatibleSchemaException"));
- }
-
- log.info("try with fully compat, should fail");
- try (Producer<V4Data> p = pulsarClient.newProducer(Schema.AVRO(V4Data.class)).topic(topicName).create()) {
- Assert.fail("Fully compat schema should fail, autoupdate disabled");
- } catch (PulsarClientException e) {
- Assert.assertTrue(e.getMessage().contains("IncompatibleSchemaException"));
- }
-
- log.info("Should still be able to connect with original schema");
- try (Producer<V1Data> p = pulsarClient.newProducer(Schema.AVRO(V1Data.class)).topic(topicName).create()) {
- p.send(new V1Data("test2", 2));
- }
-
- admin.namespaces().setSchemaAutoUpdateCompatibilityStrategy(namespace,
- SchemaAutoUpdateCompatibilityStrategy.Full);
-
- for (int i = 0; i < 100; i++) {
- Topic t = pulsar.getBrokerService().getTopicIfExists(topicName).get().get();
- // get around fact that field is private and topic can be persisent or non-persistent
- Field strategy = t.getClass().getDeclaredField("schemaCompatibilityStrategy");
- strategy.setAccessible(true);
- if (((SchemaCompatibilityStrategy)strategy.get(t)) == SchemaCompatibilityStrategy.FULL) {
- break;
- }
- Thread.sleep(100);
- }
- log.info("try with fully compat, again");
- try (Producer<V4Data> p = pulsarClient.newProducer(Schema.AVRO(V4Data.class)).topic(topicName).create()) {
- p.send(new V4Data("test2", 1, (short)100));
- }
- }
-
- @AvroAlias(space="blah", alias="data")
- static class V1Data {
- String foo;
- int bar;
-
- V1Data(String foo, int bar) {
- this.foo = foo;
- this.bar = bar;
- }
- }
-
- // backward compatible with V1Data
- @AvroAlias(space="blah", alias="data")
- static class V2Data {
- String foo;
-
- V2Data(String foo) {
- this.foo = foo;
- }
- }
-
- // forward compatible with V1Data
- @AvroAlias(space="blah", alias="data")
- static class V3Data {
- String foo;
- int bar;
- long baz;
-
- V3Data(String foo, int bar, long baz) {
- this.foo = foo;
- this.bar = bar;
- this.baz = baz;
- }
- }
-
- // fully compatible with V1Data
- @AvroAlias(space="blah", alias="data")
- static class V4Data {
- String foo;
- int bar;
- @AvroDefault(value = "10")
- short blah;
-
- V4Data(String foo, int bar, short blah) {
- this.foo = foo;
- this.bar = bar;
- this.blah = blah;
- }
- }
-
- @Test
- public void testBackwardV2() throws Exception {
- testAutoUpdateBackward("prop-xyz/ns1", "persistent://prop-xyz/ns1/backward");
- testAutoUpdateBackward("prop-xyz/ns2", "non-persistent://prop-xyz/ns2/backward-np");
- }
-
- @Test
- public void testForwardV2() throws Exception {
- testAutoUpdateForward("prop-xyz/ns1", "persistent://prop-xyz/ns1/forward");
- testAutoUpdateForward("prop-xyz/ns2", "non-persistent://prop-xyz/ns2/forward-np");
- }
-
- @Test
- public void testFullV2() throws Exception {
- testAutoUpdateFull("prop-xyz/ns1", "persistent://prop-xyz/ns1/full");
- testAutoUpdateFull("prop-xyz/ns2", "non-persistent://prop-xyz/ns2/full-np");
- }
-
- @Test
- public void testDisabledV2() throws Exception {
- testAutoUpdateDisabled("prop-xyz/ns1", "persistent://prop-xyz/ns1/disabled");
- testAutoUpdateDisabled("prop-xyz/ns2", "non-persistent://prop-xyz/ns2/disabled-np");
- }
-
- @Test
- public void testBackwardV1() throws Exception {
- testAutoUpdateBackward("prop-xyz/test/ns1", "persistent://prop-xyz/test/ns1/backward");
- testAutoUpdateBackward("prop-xyz/test/ns2", "non-persistent://prop-xyz/test/ns2/backward-np");
- }
-
- @Test
- public void testForwardV1() throws Exception {
- testAutoUpdateForward("prop-xyz/test/ns1", "persistent://prop-xyz/test/ns1/forward");
- testAutoUpdateForward("prop-xyz/test/ns2", "non-persistent://prop-xyz/test/ns2/forward-np");
- }
-
- @Test
- public void testFullV1() throws Exception {
- testAutoUpdateFull("prop-xyz/test/ns1", "persistent://prop-xyz/test/ns1/full");
- testAutoUpdateFull("prop-xyz/test/ns2", "non-persistent://prop-xyz/test/ns2/full-np");
- }
-
- @Test
- public void testDisabledV1() throws Exception {
- testAutoUpdateDisabled("prop-xyz/test/ns1", "persistent://prop-xyz/test/ns1/disabled");
- testAutoUpdateDisabled("prop-xyz/test/ns2", "non-persistent://prop-xyz/test/ns2/disabled-np");
- }
-}
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest.java
deleted file mode 100644
index 5c18dfe2fb..0000000000
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest.java
+++ /dev/null
@@ -1,2016 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pulsar.broker.admin;
-
-import static org.mockito.Mockito.doReturn;
-import static org.mockito.Mockito.spy;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
-import static org.testng.Assert.assertEquals;
-import static org.testng.Assert.assertFalse;
-import static org.testng.Assert.assertNotEquals;
-import static org.testng.Assert.assertNotNull;
-import static org.testng.Assert.assertTrue;
-import static org.testng.Assert.fail;
-
-import com.google.common.collect.BoundType;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-import com.google.common.collect.Range;
-import com.google.common.collect.Sets;
-import com.google.common.hash.Hashing;
-
-import java.lang.reflect.Field;
-import java.net.URL;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.EnumSet;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.Set;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
-
-import javax.ws.rs.client.InvocationCallback;
-import javax.ws.rs.client.WebTarget;
-import javax.ws.rs.core.Response.Status;
-
-import lombok.extern.slf4j.Slf4j;
-
-import org.apache.pulsar.broker.PulsarServerException;
-import org.apache.pulsar.broker.PulsarService;
-import org.apache.pulsar.broker.ServiceConfiguration;
-import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest;
-import org.apache.pulsar.broker.namespace.NamespaceEphemeralData;
-import org.apache.pulsar.broker.namespace.NamespaceService;
-import org.apache.pulsar.broker.service.BrokerService;
-import org.apache.pulsar.client.admin.LongRunningProcessStatus;
-import org.apache.pulsar.client.admin.PulsarAdmin;
-import org.apache.pulsar.client.admin.PulsarAdminException;
-import org.apache.pulsar.client.admin.PulsarAdminException.ConflictException;
-import org.apache.pulsar.client.admin.PulsarAdminException.NotAuthorizedException;
-import org.apache.pulsar.client.admin.PulsarAdminException.NotFoundException;
-import org.apache.pulsar.client.admin.PulsarAdminException.PreconditionFailedException;
-import org.apache.pulsar.client.admin.internal.LookupImpl;
-import org.apache.pulsar.client.admin.internal.TopicsImpl;
-import org.apache.pulsar.client.admin.internal.TenantsImpl;
-import org.apache.pulsar.client.api.Consumer;
-import org.apache.pulsar.client.api.ConsumerBuilder;
-import org.apache.pulsar.client.api.Message;
-import org.apache.pulsar.client.api.MessageRoutingMode;
-import org.apache.pulsar.client.api.Producer;
-import org.apache.pulsar.client.api.PulsarClient;
-import org.apache.pulsar.client.api.Schema;
-import org.apache.pulsar.client.api.SubscriptionType;
-import org.apache.pulsar.common.lookup.data.LookupData;
-import org.apache.pulsar.common.naming.NamespaceBundle;
-import org.apache.pulsar.common.naming.NamespaceBundleFactory;
-import org.apache.pulsar.common.naming.NamespaceBundles;
-import org.apache.pulsar.common.naming.NamespaceName;
-import org.apache.pulsar.common.naming.TopicDomain;
-import org.apache.pulsar.common.naming.TopicName;
-import org.apache.pulsar.common.partition.PartitionedTopicMetadata;
-import org.apache.pulsar.common.policies.data.AuthAction;
-import org.apache.pulsar.common.policies.data.AutoFailoverPolicyData;
-import org.apache.pulsar.common.policies.data.AutoFailoverPolicyType;
-import org.apache.pulsar.common.policies.data.BacklogQuota;
-import org.apache.pulsar.common.policies.data.BacklogQuota.BacklogQuotaType;
-import org.apache.pulsar.common.policies.data.BacklogQuota.RetentionPolicy;
-import org.apache.pulsar.common.policies.data.BrokerAssignment;
-import org.apache.pulsar.common.policies.data.ClusterData;
-import org.apache.pulsar.common.policies.data.NamespaceIsolationData;
-import org.apache.pulsar.common.policies.data.NamespaceOwnershipStatus;
-import org.apache.pulsar.common.policies.data.PartitionedTopicStats;
-import org.apache.pulsar.common.policies.data.PersistencePolicies;
-import org.apache.pulsar.common.policies.data.PersistentTopicInternalStats;
-import org.apache.pulsar.common.policies.data.TopicStats;
-import org.apache.pulsar.common.policies.data.Policies;
-import org.apache.pulsar.common.policies.data.RetentionPolicies;
-import org.apache.pulsar.common.policies.data.TenantInfo;
-import org.apache.pulsar.common.util.Codec;
-import org.apache.pulsar.common.util.ObjectMapperFactory;
-import org.apache.pulsar.compaction.Compactor;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.Assert;
-import org.testng.annotations.AfterMethod;
-import org.testng.annotations.BeforeMethod;
-import org.testng.annotations.DataProvider;
-import org.testng.annotations.Test;
-
-@Slf4j
-public class AdminApiTest extends MockedPulsarServiceBaseTest {
-
- private static final Logger LOG = LoggerFactory.getLogger(AdminApiTest.class);
-
- private final String TLS_SERVER_CERT_FILE_PATH = "./src/test/resources/certificate/server.crt";
- private final String TLS_SERVER_KEY_FILE_PATH = "./src/test/resources/certificate/server.key";
-
- private MockedPulsarService mockPulsarSetup;
-
- private PulsarService otherPulsar;
-
- private PulsarAdmin adminTls;
- private PulsarAdmin otheradmin;
-
- private NamespaceBundleFactory bundleFactory;
-
- @BeforeMethod
- @Override
- public void setup() throws Exception {
- conf.setLoadBalancerEnabled(true);
- conf.setTlsEnabled(true);
- conf.setTlsCertificateFilePath(TLS_SERVER_CERT_FILE_PATH);
- conf.setTlsKeyFilePath(TLS_SERVER_KEY_FILE_PATH);
-
- super.internalSetup();
-
- bundleFactory = new NamespaceBundleFactory(pulsar, Hashing.crc32());
-
- adminTls = spy(PulsarAdmin.builder().tlsTrustCertsFilePath(TLS_SERVER_CERT_FILE_PATH)
- .serviceHttpUrl(brokerUrlTls.toString()).build());
-
- // create otherbroker to test redirect on calls that need
- // namespace ownership
- mockPulsarSetup = new MockedPulsarService(this.conf);
- mockPulsarSetup.setup();
- otherPulsar = mockPulsarSetup.getPulsar();
- otheradmin = mockPulsarSetup.getAdmin();
-
- // Setup namespaces
- admin.clusters().createCluster("test", new ClusterData("http://127.0.0.1" + ":" + BROKER_WEBSERVICE_PORT));
- TenantInfo tenantInfo = new TenantInfo(Sets.newHashSet("role1", "role2"), Sets.newHashSet("test"));
- admin.tenants().createTenant("prop-xyz", tenantInfo);
- admin.namespaces().createNamespace("prop-xyz/ns1", Sets.newHashSet("test"));
- }
-
- @AfterMethod
- @Override
- public void cleanup() throws Exception {
- adminTls.close();
- super.internalCleanup();
- mockPulsarSetup.cleanup();
- }
-
- @DataProvider(name = "numBundles")
- public static Object[][] numBundles() {
- return new Object[][] { { 1 }, { 4 } };
- }
-
- @DataProvider(name = "bundling")
- public static Object[][] bundling() {
- return new Object[][] { { 0 }, { 4 } };
- }
-
- @DataProvider(name = "topicName")
- public Object[][] topicNamesProvider() {
- return new Object[][] { { "topic_+&*%{}() \\$@#^%" }, { "simple-topicName" } };
- }
-
- @DataProvider(name = "topicType")
- public Object[][] topicTypeProvider() {
- return new Object[][] { { TopicDomain.persistent.value() }, { TopicDomain.non_persistent.value() } };
- }
-
- @Test
- public void clusters() throws Exception {
- admin.clusters().createCluster("usw",
- new ClusterData("http://broker.messaging.use.example.com" + ":" + BROKER_WEBSERVICE_PORT));
- // "test" cluster is part of config-default cluster and it's znode gets created when PulsarService creates
- // failure-domain znode of this default cluster
- assertEquals(admin.clusters().getClusters(), Lists.newArrayList("test", "usw"));
-
- assertEquals(admin.clusters().getCluster("test"),
- new ClusterData("http://127.0.0.1" + ":" + BROKER_WEBSERVICE_PORT));
-
- admin.clusters().updateCluster("usw",
- new ClusterData("http://new-broker.messaging.usw.example.com" + ":" + BROKER_WEBSERVICE_PORT));
- assertEquals(admin.clusters().getClusters(), Lists.newArrayList("test", "usw"));
- assertEquals(admin.clusters().getCluster("usw"),
- new ClusterData("http://new-broker.messaging.usw.example.com" + ":" + BROKER_WEBSERVICE_PORT));
-
- admin.clusters().updateCluster("usw",
- new ClusterData("http://new-broker.messaging.usw.example.com" + ":" + BROKER_WEBSERVICE_PORT,
- "https://new-broker.messaging.usw.example.com" + ":" + BROKER_WEBSERVICE_PORT_TLS));
- assertEquals(admin.clusters().getClusters(), Lists.newArrayList("test", "usw"));
- assertEquals(admin.clusters().getCluster("usw"),
- new ClusterData("http://new-broker.messaging.usw.example.com" + ":" + BROKER_WEBSERVICE_PORT,
- "https://new-broker.messaging.usw.example.com" + ":" + BROKER_WEBSERVICE_PORT_TLS));
-
- admin.clusters().deleteCluster("usw");
- Thread.sleep(300);
-
- assertEquals(admin.clusters().getClusters(), Lists.newArrayList("test"));
-
- admin.namespaces().deleteNamespace("prop-xyz/ns1");
- admin.clusters().deleteCluster("test");
- assertEquals(admin.clusters().getClusters(), Lists.newArrayList());
-
- // Check name validation
- try {
- admin.clusters().createCluster("bf!", new ClusterData("http://dummy.messaging.example.com"));
- fail("should have failed");
- } catch (PulsarAdminException e) {
- assertTrue(e instanceof PreconditionFailedException);
- }
- }
-
- @Test
- public void clusterNamespaceIsolationPolicies() throws PulsarAdminException {
- try {
- // create
- String policyName1 = "policy-1";
- NamespaceIsolationData nsPolicyData1 = new NamespaceIsolationData();
- nsPolicyData1.namespaces = new ArrayList<String>();
- nsPolicyData1.namespaces.add("other/use/other.*");
- nsPolicyData1.primary = new ArrayList<String>();
- nsPolicyData1.primary.add("prod1-broker[4-6].messaging.use.example.com");
- nsPolicyData1.secondary = new ArrayList<String>();
- nsPolicyData1.secondary.add("prod1-broker.*.messaging.use.example.com");
- nsPolicyData1.auto_failover_policy = new AutoFailoverPolicyData();
- nsPolicyData1.auto_failover_policy.policy_type = AutoFailoverPolicyType.min_available;
- nsPolicyData1.auto_failover_policy.parameters = new HashMap<String, String>();
- nsPolicyData1.auto_failover_policy.parameters.put("min_limit", "1");
- nsPolicyData1.auto_failover_policy.parameters.put("usage_threshold", "100");
- admin.clusters().createNamespaceIsolationPolicy("test", policyName1, nsPolicyData1);
-
- String policyName2 = "policy-2";
- NamespaceIsolationData nsPolicyData2 = new NamespaceIsolationData();
- nsPolicyData2.namespaces = new ArrayList<String>();
- nsPolicyData2.namespaces.add("other/use/other.*");
- nsPolicyData2.primary = new ArrayList<String>();
- nsPolicyData2.primary.add("prod1-broker[4-6].messaging.use.example.com");
- nsPolicyData2.secondary = new ArrayList<String>();
- nsPolicyData2.secondary.add("prod1-broker.*.messaging.use.example.com");
- nsPolicyData2.auto_failover_policy = new AutoFailoverPolicyData();
- nsPolicyData2.auto_failover_policy.policy_type = AutoFailoverPolicyType.min_available;
- nsPolicyData2.auto_failover_policy.parameters = new HashMap<String, String>();
- nsPolicyData2.auto_failover_policy.parameters.put("min_limit", "1");
- nsPolicyData2.auto_failover_policy.parameters.put("usage_threshold", "100");
- admin.clusters().createNamespaceIsolationPolicy("test", policyName2, nsPolicyData2);
-
- // verify create indirectly with get
- Map<String, NamespaceIsolationData> policiesMap = admin.clusters().getNamespaceIsolationPolicies("test");
- assertEquals(policiesMap.get(policyName1), nsPolicyData1);
- assertEquals(policiesMap.get(policyName2), nsPolicyData2);
-
- // verify update of primary
- nsPolicyData1.primary.remove(0);
- nsPolicyData1.primary.add("prod1-broker[1-2].messaging.use.example.com");
- admin.clusters().updateNamespaceIsolationPolicy("test", policyName1, nsPolicyData1);
-
- // verify primary change
- policiesMap = admin.clusters().getNamespaceIsolationPolicies("test");
- assertEquals(policiesMap.get(policyName1), nsPolicyData1);
-
- // verify update of secondary
- nsPolicyData1.secondary.remove(0);
- nsPolicyData1.secondary.add("prod1-broker[3-4].messaging.use.example.com");
- admin.clusters().updateNamespaceIsolationPolicy("test", policyName1, nsPolicyData1);
-
- // verify secondary change
- policiesMap = admin.clusters().getNamespaceIsolationPolicies("test");
- assertEquals(policiesMap.get(policyName1), nsPolicyData1);
-
- // verify update of failover policy limit
- nsPolicyData1.auto_failover_policy.parameters.put("min_limit", "10");
- admin.clusters().updateNamespaceIsolationPolicy("test", policyName1, nsPolicyData1);
-
- // verify min_limit change
- policiesMap = admin.clusters().getNamespaceIsolationPolicies("test");
- assertEquals(policiesMap.get(policyName1), nsPolicyData1);
-
- // verify update of failover usage_threshold limit
- nsPolicyData1.auto_failover_policy.parameters.put("usage_threshold", "80");
- admin.clusters().updateNamespaceIsolationPolicy("test", policyName1, nsPolicyData1);
-
- // verify usage_threshold change
- policiesMap = admin.clusters().getNamespaceIsolationPolicies("test");
- assertEquals(policiesMap.get(policyName1), nsPolicyData1);
-
- // verify single get
- NamespaceIsolationData policy1Data = admin.clusters().getNamespaceIsolationPolicy("test", policyName1);
- assertEquals(policy1Data, nsPolicyData1);
-
- // verify creation of more than one policy
- admin.clusters().createNamespaceIsolationPolicy("test", policyName2, nsPolicyData1);
-
- try {
- admin.clusters().getNamespaceIsolationPolicy("test", "no-such-policy");
- fail("should have raised exception");
- } catch (PulsarAdminException e) {
- assertTrue(e instanceof NotFoundException);
- }
-
- // verify delete cluster failed
- try {
- admin.clusters().deleteCluster("test");
- fail("should have raised exception");
- } catch (PulsarAdminException e) {
- assertTrue(e instanceof PreconditionFailedException);
- }
-
- // verify delete
- admin.clusters().deleteNamespaceIsolationPolicy("test", policyName1);
- admin.clusters().deleteNamespaceIsolationPolicy("test", policyName2);
-
- try {
- admin.clusters().getNamespaceIsolationPolicy("test", policyName1);
- fail("should have raised exception");
- } catch (PulsarAdminException e) {
- assertTrue(e instanceof NotFoundException);
- }
-
- try {
- admin.clusters().getNamespaceIsolationPolicy("test", policyName2);
- fail("should have raised exception");
- } catch (PulsarAdminException e) {
- assertTrue(e instanceof NotFoundException);
- }
-
- try {
- admin.clusters().getNamespaceIsolationPolicies("usc");
- fail("should have raised exception");
- } catch (PulsarAdminException e) {
- assertTrue(e instanceof NotFoundException);
- }
-
- try {
- admin.clusters().getNamespaceIsolationPolicy("usc", "no-such-cluster");
- fail("should have raised exception");
- } catch (PulsarAdminException e) {
- assertTrue(e instanceof PreconditionFailedException);
- }
-
- try {
- admin.clusters().createNamespaceIsolationPolicy("usc", "no-such-cluster", nsPolicyData1);
- fail("should have raised exception");
- } catch (PulsarAdminException e) {
- assertTrue(e instanceof PreconditionFailedException);
- }
-
- try {
- admin.clusters().updateNamespaceIsolationPolicy("usc", "no-such-cluster", policy1Data);
- fail("should have raised exception");
- } catch (PulsarAdminException e) {
- assertTrue(e instanceof PreconditionFailedException);
- }
-
- } catch (PulsarAdminException e) {
- LOG.warn("TEST FAILED [{}]", e.getMessage());
- throw e;
- }
- }
-
- @Test
- public void brokers() throws Exception {
- List<String> list = admin.brokers().getActiveBrokers("test");
- Assert.assertNotNull(list);
- Assert.assertEquals(list.size(), 1);
-
- List<String> list2 = otheradmin.brokers().getActiveBrokers("test");
- Assert.assertNotNull(list2);
- Assert.assertEquals(list2.size(), 1);
-
- Map<String, NamespaceOwnershipStatus> nsMap = admin.brokers().getOwnedNamespaces("test", list.get(0));
- // since sla-monitor ns is not created nsMap.size() == 1 (for HeartBeat Namespace)
- Assert.assertEquals(1, nsMap.size());
- for (String ns : nsMap.keySet()) {
- NamespaceOwnershipStatus nsStatus = nsMap.get(ns);
- if (ns.equals(
- NamespaceService.getHeartbeatNamespace(pulsar.getAdvertisedAddress(), pulsar.getConfiguration())
- + "/0x00000000_0xffffffff")) {
- assertEquals(nsStatus.broker_assignment, BrokerAssignment.shared);
- assertFalse(nsStatus.is_controlled);
- assertTrue(nsStatus.is_active);
- }
- }
-
- String[] parts = list.get(0).split(":");
- Assert.assertEquals(parts.length, 2);
- Map<String, NamespaceOwnershipStatus> nsMap2 = adminTls.brokers().getOwnedNamespaces("test",
- String.format("%s:%d", parts[0], BROKER_WEBSERVICE_PORT_TLS));
- Assert.assertEquals(nsMap2.size(), 1);
-
- admin.namespaces().deleteNamespace("prop-xyz/ns1");
- admin.clusters().deleteCluster("test");
- assertEquals(admin.clusters().getClusters(), Lists.newArrayList());
- }
-
- /**
- * <pre>
- * Verifies: zk-update configuration updates service-config
- * 1. create znode for dynamic-config
- * 2. start pulsar service so, pulsar can set the watch on that znode
- * 3. update the configuration with new value
- * 4. wait and verify that new value has been updated
- * </pre>
- *
- * @throws Exception
- */
- @Test
- public void testUpdateDynamicConfigurationWithZkWatch() throws Exception {
- final int initValue = 30000;
- pulsar.getConfiguration().setBrokerShutdownTimeoutMs(initValue);
- // (1) try to update dynamic field
- final long shutdownTime = 10;
- // update configuration
- admin.brokers().updateDynamicConfiguration("brokerShutdownTimeoutMs", Long.toString(shutdownTime));
- // sleep incrementally as zk-watch notification is async and may take some time
- for (int i = 0; i < 5; i++) {
- if (pulsar.getConfiguration().getBrokerShutdownTimeoutMs() != initValue) {
- Thread.sleep(50 + (i * 10));
- }
- }
- // wait config to be updated
- for (int i = 0; i < 5; i++) {
- if (pulsar.getConfiguration().getBrokerShutdownTimeoutMs() != shutdownTime) {
- Thread.sleep(100 + (i * 10));
- } else {
- break;
- }
- }
- // verify value is updated
- assertEquals(pulsar.getConfiguration().getBrokerShutdownTimeoutMs(), shutdownTime);
-
- // (2) try to update non-dynamic field
- try {
- admin.brokers().updateDynamicConfiguration("zookeeperServers", "test-zk:1234");
- } catch (Exception e) {
- assertTrue(e instanceof PreconditionFailedException);
- }
-
- // (3) try to update non-existent field
- try {
- admin.brokers().updateDynamicConfiguration("test", Long.toString(shutdownTime));
- } catch (Exception e) {
- assertTrue(e instanceof PreconditionFailedException);
- }
-
- }
-
- /**
- * Verifies broker sets watch on dynamic-configuration map even with invalid init json data
- *
- * <pre>
- * 1. Set invalid json at dynamic-config znode
- * 2. Broker fails to deserialize znode content but sets the watch on znode
- * 3. Update znode with valid json map
- * 4. Broker should get watch and update the dynamic-config map
- * </pre>
- *
- * @throws Exception
- */
- @Test
- public void testInvalidDynamicConfigContentInZK() throws Exception {
- final int newValue = 10;
- stopBroker();
- // set invalid data into dynamic-config znode so, broker startup fail to deserialize data
- mockZookKeeper.setData(BrokerService.BROKER_SERVICE_CONFIGURATION_PATH, "$".getBytes(), -1);
- // start broker: it should have set watch even if with failure of deserialization
- startBroker();
- Assert.assertNotEquals(pulsar.getConfiguration().getBrokerShutdownTimeoutMs(), newValue);
- // update zk with config-value which should fire watch and broker should update the config value
- Map<String, String> configMap = Maps.newHashMap();
- configMap.put("brokerShutdownTimeoutMs", Integer.toString(newValue));
- mockZookKeeper.setData(BrokerService.BROKER_SERVICE_CONFIGURATION_PATH,
- ObjectMapperFactory.getThreadLocal().writeValueAsBytes(configMap), -1);
- // wait config to be updated
- for (int i = 0; i < 5; i++) {
- if (pulsar.getConfiguration().getBrokerShutdownTimeoutMs() != newValue) {
- Thread.sleep(100 + (i * 10));
- } else {
- break;
- }
- }
- // verify value is updated
- assertEquals(pulsar.getConfiguration().getBrokerShutdownTimeoutMs(), newValue);
- }
-
- /**
- * <pre>
- * verifies: that registerListener updates pulsar.config value with newly updated zk-dynamic config
- * 1.start pulsar
- * 2.update zk-config with admin api
- * 3. trigger watch and listener
- * 4. verify that config is updated
- * </pre>
- *
- * @throws Exception
- */
- @Test
- public void testUpdateDynamicLocalConfiguration() throws Exception {
- // (1) try to update dynamic field
- final long initValue = 30000;
- final long shutdownTime = 10;
- pulsar.getConfiguration().setBrokerShutdownTimeoutMs(initValue);
- // update configuration
- admin.brokers().updateDynamicConfiguration("brokerShutdownTimeoutMs", Long.toString(shutdownTime));
- // sleep incrementally as zk-watch notification is async and may take some time
- for (int i = 0; i < 5; i++) {
- if (pulsar.getConfiguration().getBrokerShutdownTimeoutMs() == initValue) {
- Thread.sleep(50 + (i * 10));
- }
- }
-
- // verify value is updated
- assertEquals(pulsar.getConfiguration().getBrokerShutdownTimeoutMs(), shutdownTime);
- }
-
- @Test
- public void testUpdatableConfigurationName() throws Exception {
- // (1) try to update dynamic field
- final String configName = "brokerShutdownTimeoutMs";
- assertTrue(admin.brokers().getDynamicConfigurationNames().contains(configName));
- }
-
- @Test
- public void testGetDynamicLocalConfiguration() throws Exception {
- // (1) try to update dynamic field
- final String configName = "brokerShutdownTimeoutMs";
- final long shutdownTime = 10;
- pulsar.getConfiguration().setBrokerShutdownTimeoutMs(30000);
- Map<String, String> configs = admin.brokers().getAllDynamicConfigurations();
- assertTrue(configs.isEmpty());
- assertNotEquals(pulsar.getConfiguration().getBrokerShutdownTimeoutMs(), shutdownTime);
- // update configuration
- admin.brokers().updateDynamicConfiguration(configName, Long.toString(shutdownTime));
- // Now, znode is created: updateConfigurationAndregisterListeners and check if configuration updated
- assertEquals(Long.parseLong(admin.brokers().getAllDynamicConfigurations().get(configName)), shutdownTime);
- }
-
- @Test(enabled = true)
- public void properties() throws PulsarAdminException {
- Set<String> allowedClusters = Sets.newHashSet("test");
- TenantInfo tenantInfo = new TenantInfo(Sets.newHashSet("role1", "role2"), allowedClusters);
- admin.tenants().updateTenant("prop-xyz", tenantInfo);
-
- assertEquals(admin.tenants().getTenants(), Lists.newArrayList("prop-xyz"));
-
- assertEquals(admin.tenants().getTenantInfo("prop-xyz"), tenantInfo);
-
- TenantInfo newTenantAdmin = new TenantInfo(Sets.newHashSet("role3", "role4"), allowedClusters);
- admin.tenants().updateTenant("prop-xyz", newTenantAdmin);
-
- assertEquals(admin.tenants().getTenantInfo("prop-xyz"), newTenantAdmin);
-
- admin.namespaces().deleteNamespace("prop-xyz/ns1");
- admin.tenants().deleteTenant("prop-xyz");
- assertEquals(admin.tenants().getTenants(), Lists.newArrayList());
-
- // Check name validation
- try {
- admin.tenants().createTenant("prop-xyz&", tenantInfo);
- fail("should have failed");
- } catch (PulsarAdminException e) {
- assertTrue(e instanceof PreconditionFailedException);
- }
- }
-
- @Test(invocationCount = 1)
- public void namespaces() throws PulsarAdminException, PulsarServerException, Exception {
- admin.clusters().createCluster("usw", new ClusterData());
- TenantInfo tenantInfo = new TenantInfo(Sets.newHashSet("role1", "role2"),
- Sets.newHashSet("test", "usw"));
- admin.tenants().updateTenant("prop-xyz", tenantInfo);
-
- assertEquals(admin.namespaces().getPolicies("prop-xyz/ns1").bundles, Policies.defaultBundle());
-
- admin.namespaces().createNamespace("prop-xyz/ns2", Sets.newHashSet("test"));
-
- admin.namespaces().createNamespace("prop-xyz/ns3", 4);
- admin.namespaces().setNamespaceReplicationClusters("prop-xyz/ns3", Sets.newHashSet("test"));
- assertEquals(admin.namespaces().getPolicies("prop-xyz/ns3").bundles.numBundles, 4);
- assertEquals(admin.namespaces().getPolicies("prop-xyz/ns3").bundles.boundaries.size(), 5);
-
- admin.namespaces().deleteNamespace("prop-xyz/ns3");
-
- try {
- admin.namespaces().createNamespace("non-existing/ns1");
- fail("Should not have passed");
- } catch (NotFoundException e) {
- // Ok
- }
-
- assertEquals(admin.namespaces().getNamespaces("prop-xyz"), Lists.newArrayList("prop-xyz/ns1", "prop-xyz/ns2"));
- assertEquals(admin.namespaces().getNamespaces("prop-xyz"), Lists.newArrayList("prop-xyz/ns1", "prop-xyz/ns2"));
-
- try {
- admin.namespaces().createNamespace("prop-xyz/ns4", Sets.newHashSet("usc"));
- fail("Should not have passed");
- } catch (NotAuthorizedException e) {
- // Ok, got the non authorized exception since usc cluster is not in the allowed clusters list.
- }
-
- // test with url style role.
- admin.namespaces().grantPermissionOnNamespace("prop-xyz/ns1",
- "spiffe://developer/passport-role", EnumSet.allOf(AuthAction.class));
- admin.namespaces().grantPermissionOnNamespace("prop-xyz/ns1", "my-role", EnumSet.allOf(AuthAction.class));
-
- Policies policies = new Policies();
- policies.replication_clusters = Sets.newHashSet("test");
- policies.bundles = Policies.defaultBundle();
- policies.auth_policies.namespace_auth.put("spiffe://developer/passport-role", EnumSet.allOf(AuthAction.class));
- policies.auth_policies.namespace_auth.put("my-role", EnumSet.allOf(AuthAction.class));
-
- assertEquals(admin.namespaces().getPolicies("prop-xyz/ns1"), policies);
- assertEquals(admin.namespaces().getPermissions("prop-xyz/ns1"), policies.auth_policies.namespace_auth);
-
- assertEquals(admin.namespaces().getTopics("prop-xyz/ns1"), Lists.newArrayList());
-
- admin.namespaces().revokePermissionsOnNamespace("prop-xyz/ns1", "spiffe://developer/passport-role");
- admin.namespaces().revokePermissionsOnNamespace("prop-xyz/ns1", "my-role");
- policies.auth_policies.namespace_auth.remove("spiffe://developer/passport-role");
- policies.auth_policies.namespace_auth.remove("my-role");
- assertEquals(admin.namespaces().getPolicies("prop-xyz/ns1"), policies);
-
- assertEquals(admin.namespaces().getPersistence("prop-xyz/ns1"), new PersistencePolicies(1, 1, 1, 0.0));
- admin.namespaces().setPersistence("prop-xyz/ns1", new PersistencePolicies(3, 2, 1, 10.0));
- assertEquals(admin.namespaces().getPersistence("prop-xyz/ns1"), new PersistencePolicies(3, 2, 1, 10.0));
-
- // Force topic creation and namespace being loaded
- Producer<byte[]> producer = pulsarClient.newProducer(Schema.BYTES)
- .topic("persistent://prop-xyz/ns1/my-topic")
- .enableBatching(false)
- .messageRoutingMode(MessageRoutingMode.SinglePartition)
- .create();
- producer.close();
- admin.topics().delete("persistent://prop-xyz/ns1/my-topic");
-
- admin.namespaces().unloadNamespaceBundle("prop-xyz/ns1", "0x00000000_0xffffffff");
- NamespaceName ns = NamespaceName.get("prop-xyz/ns1");
- // Now, w/ bundle policies, we will use default bundle
- NamespaceBundle defaultBundle = bundleFactory.getFullBundle(ns);
- int i = 0;
- for (; i < 10; i++) {
- Optional<NamespaceEphemeralData> data1 = pulsar.getNamespaceService().getOwnershipCache()
- .getOwnerAsync(defaultBundle).get();
- if (!data1.isPresent()) {
- // Already unloaded
- break;
- }
- LOG.info("Waiting for unload namespace {} to complete. Current service unit isDisabled: {}", defaultBundle,
- data1.get().isDisabled());
- Thread.sleep(1000);
- }
- assertTrue(i < 10);
-
- admin.namespaces().deleteNamespace("prop-xyz/ns1");
- assertEquals(admin.namespaces().getNamespaces("prop-xyz"), Lists.newArrayList("prop-xyz/ns2"));
-
- try {
- admin.namespaces().unload("prop-xyz/ns1");
- fail("should have raised exception");
- } catch (Exception e) {
- // OK excepted
- }
-
- // Force topic creation and namespace being loaded
- producer = pulsarClient.newProducer(Schema.BYTES).topic("persistent://prop-xyz/use/ns2/my-topic").create();
- producer.close();
- admin.topics().delete("persistent://prop-xyz/use/ns2/my-topic");
-
- // both unload and delete should succeed for ns2 on other broker with a redirect
- // otheradmin.namespaces().unload("prop-xyz/use/ns2");
- }
-
- @Test(dataProvider = "topicName")
- public void persistentTopics(String topicName) throws Exception {
- assertEquals(admin.topics().getList("prop-xyz/ns1"), Lists.newArrayList());
-
- final String persistentTopicName = "persistent://prop-xyz/ns1/" + topicName;
- // Force to create a topic
- publishMessagesOnPersistentTopic("persistent://prop-xyz/ns1/" + topicName, 0);
- assertEquals(admin.topics().getList("prop-xyz/ns1"),
- Lists.newArrayList("persistent://prop-xyz/ns1/" + topicName));
-
- // create consumer and subscription
- URL pulsarUrl = new URL("http://127.0.0.1" + ":" + BROKER_WEBSERVICE_PORT);
- PulsarClient client = PulsarClient.builder().serviceUrl(pulsarUrl.toString()).statsInterval(0, TimeUnit.SECONDS)
- .build();
- Consumer<byte[]> consumer = client.newConsumer().topic(persistentTopicName).subscriptionName("my-sub")
- .subscriptionType(SubscriptionType.Exclusive).subscribe();
-
- assertEquals(admin.topics().getSubscriptions(persistentTopicName), Lists.newArrayList("my-sub"));
-
- publishMessagesOnPersistentTopic("persistent://prop-xyz/ns1/" + topicName, 10);
-
- TopicStats topicStats = admin.topics().getStats(persistentTopicName);
- assertEquals(topicStats.subscriptions.keySet(), Sets.newTreeSet(Lists.newArrayList("my-sub")));
- assertEquals(topicStats.subscriptions.get("my-sub").consumers.size(), 1);
- assertEquals(topicStats.subscriptions.get("my-sub").msgBacklog, 10);
- assertEquals(topicStats.publishers.size(), 0);
-
- PersistentTopicInternalStats internalStats = admin.topics().getInternalStats(persistentTopicName);
- assertEquals(internalStats.cursors.keySet(), Sets.newTreeSet(Lists.newArrayList("my-sub")));
-
- List<Message<byte[]>> messages = admin.topics().peekMessages(persistentTopicName, "my-sub", 3);
- assertEquals(messages.size(), 3);
- for (int i = 0; i < 3; i++) {
- String expectedMessage = "message-" + i;
- assertEquals(messages.get(i).getData(), expectedMessage.getBytes());
- }
-
- messages = admin.topics().peekMessages(persistentTopicName, "my-sub", 15);
- assertEquals(messages.size(), 10);
- for (int i = 0; i < 10; i++) {
- String expectedMessage = "message-" + i;
- assertEquals(messages.get(i).getData(), expectedMessage.getBytes());
- }
-
- admin.topics().skipMessages(persistentTopicName, "my-sub", 5);
- topicStats = admin.topics().getStats(persistentTopicName);
- assertEquals(topicStats.subscriptions.get("my-sub").msgBacklog, 5);
-
- admin.topics().skipAllMessages(persistentTopicName, "my-sub");
- topicStats = admin.topics().getStats(persistentTopicName);
- assertEquals(topicStats.subscriptions.get("my-sub").msgBacklog, 0);
-
- consumer.close();
- client.close();
-
- admin.topics().deleteSubscription(persistentTopicName, "my-sub");
-
- assertEquals(admin.topics().getSubscriptions(persistentTopicName), Lists.newArrayList());
- topicStats = admin.topics().getStats(persistentTopicName);
- assertEquals(topicStats.subscriptions.keySet(), Sets.newTreeSet());
- assertEquals(topicStats.publishers.size(), 0);
-
- try {
- admin.topics().skipAllMessages(persistentTopicName, "my-sub");
- } catch (NotFoundException e) {
- }
-
- admin.topics().delete(persistentTopicName);
-
- try {
- admin.topics().delete(persistentTopicName);
- fail("Should have received 404");
- } catch (NotFoundException e) {
- }
-
- assertEquals(admin.topics().getList("prop-xyz/ns1"), Lists.newArrayList());
- }
-
- @Test(dataProvider = "topicName")
- public void partitionedTopics(String topicName) throws Exception {
- assertEquals(admin.topics().getPartitionedTopicList("prop-xyz/ns1"), Lists.newArrayList());
- final String partitionedTopicName = "persistent://prop-xyz/ns1/" + topicName;
- admin.topics().createPartitionedTopic(partitionedTopicName, 4);
- assertEquals(admin.topics().getPartitionedTopicList("prop-xyz/ns1"),
- Lists.newArrayList(partitionedTopicName));
-
- assertEquals(admin.topics().getPartitionedTopicMetadata(partitionedTopicName).partitions, 4);
-
- // check if the virtual topic doesn't get created
- List<String> topics = admin.topics().getList("prop-xyz/ns1");
- assertEquals(topics.size(), 0);
-
- assertEquals(admin.topics().getPartitionedTopicMetadata("persistent://prop-xyz/ns1/ds2").partitions,
- 0);
-
- try {
- admin.topics().getPartitionedStats(partitionedTopicName, false);
- fail("should have failed");
- } catch (PulsarAdminException e) {
- // ok
- assertEquals(e.getStatusCode(), Status.NOT_FOUND.getStatusCode());
- } catch (Exception e) {
- fail(e.getMessage());
- }
-
- try {
- admin.topics().getSubscriptions(partitionedTopicName);
- fail("should have failed");
- } catch (PulsarAdminException e) {
- // ok
- assertEquals(e.getStatusCode(), Status.NOT_FOUND.getStatusCode());
- } catch (Exception e) {
- fail(e.getMessage());
- }
-
- // create consumer and subscription
- URL pulsarUrl = new URL("http://127.0.0.1" + ":" + BROKER_WEBSERVICE_PORT);
- PulsarClient client = PulsarClient.builder().serviceUrl(pulsarUrl.toString()).statsInterval(0, TimeUnit.SECONDS)
- .build();
- Consumer<byte[]> consumer = client.newConsumer().topic(partitionedTopicName).subscriptionName("my-sub")
- .subscriptionType(SubscriptionType.Exclusive).subscribe();
-
- assertEquals(admin.topics().getSubscriptions(partitionedTopicName), Lists.newArrayList("my-sub"));
-
- try {
- admin.topics().deleteSubscription(partitionedTopicName, "my-sub");
- fail("should have failed");
- } catch (PulsarAdminException.PreconditionFailedException e) {
- // ok
- } catch (Exception e) {
- fail(e.getMessage());
- }
-
- Consumer<byte[]> consumer1 = client.newConsumer().topic(partitionedTopicName).subscriptionName("my-sub-1")
- .subscribe();
-
- assertEquals(Sets.newHashSet(admin.topics().getSubscriptions(partitionedTopicName)),
- Sets.newHashSet("my-sub", "my-sub-1"));
-
- consumer1.close();
- admin.topics().deleteSubscription(partitionedTopicName, "my-sub-1");
- assertEquals(admin.topics().getSubscriptions(partitionedTopicName), Lists.newArrayList("my-sub"));
-
- Producer<byte[]> producer = client.newProducer(Schema.BYTES)
- .topic(partitionedTopicName)
- .enableBatching(false)
- .messageRoutingMode(MessageRoutingMode.RoundRobinPartition)
- .create();
-
- for (int i = 0; i < 10; i++) {
- String message = "message-" + i;
- producer.send(message.getBytes());
- }
-
- assertEquals(Sets.newHashSet(admin.topics().getList("prop-xyz/ns1")),
- Sets.newHashSet(partitionedTopicName + "-partition-0", partitionedTopicName + "-partition-1",
- partitionedTopicName + "-partition-2", partitionedTopicName + "-partition-3"));
-
- // test cumulative stats for partitioned topic
- PartitionedTopicStats topicStats = admin.topics().getPartitionedStats(partitionedTopicName, false);
- assertEquals(topicStats.subscriptions.keySet(), Sets.newTreeSet(Lists.newArrayList("my-sub")));
- assertEquals(topicStats.subscriptions.get("my-sub").consumers.size(), 1);
- assertEquals(topicStats.subscriptions.get("my-sub").msgBacklog, 10);
- assertEquals(topicStats.publishers.size(), 1);
- assertEquals(topicStats.partitions, Maps.newHashMap());
-
- // test per partition stats for partitioned topic
- topicStats = admin.topics().getPartitionedStats(partitionedTopicName, true);
- assertEquals(topicStats.metadata.partitions, 4);
- assertEquals(topicStats.partitions.keySet(),
- Sets.newHashSet(partitionedTopicName + "-partition-0", partitionedTopicName + "-partition-1",
- partitionedTopicName + "-partition-2", partitionedTopicName + "-partition-3"));
- TopicStats partitionStats = topicStats.partitions.get(partitionedTopicName + "-partition-0");
- assertEquals(partitionStats.publishers.size(), 1);
- assertEquals(partitionStats.subscriptions.get("my-sub").consumers.size(), 1);
- assertEquals(partitionStats.subscriptions.get("my-sub").msgBacklog, 3, 1);
-
- try {
- admin.topics().skipMessages(partitionedTopicName, "my-sub", 5);
- fail("skip messages for partitioned topics should fail");
- } catch (Exception e) {
- // ok
- }
-
- admin.topics().skipAllMessages(partitionedTopicName, "my-sub");
- topicStats = admin.topics().getPartitionedStats(partitionedTopicName, false);
- assertEquals(topicStats.subscriptions.get("my-sub").msgBacklog, 0);
-
- producer.close();
- consumer.close();
-
- admin.topics().deleteSubscription(partitionedTopicName, "my-sub");
-
- assertEquals(admin.topics().getSubscriptions(partitionedTopicName), Lists.newArrayList());
-
- try {
- admin.topics().createPartitionedTopic(partitionedTopicName, 32);
- fail("Should have failed as the partitioned topic already exists");
- } catch (ConflictException ce) {
- }
-
- producer = client.newProducer(Schema.BYTES)
- .topic(partitionedTopicName)
- .enableBatching(false)
- .messageRoutingMode(MessageRoutingMode.SinglePartition)
- .create();
-
- topics = admin.topics().getList("prop-xyz/ns1");
- assertEquals(topics.size(), 4);
-
- try {
- admin.topics().deletePartitionedTopic(partitionedTopicName);
- fail("The topic is busy");
- } catch (PreconditionFailedException pfe) {
- // ok
- }
-
- producer.close();
- client.close();
-
- admin.topics().deletePartitionedTopic(partitionedTopicName);
-
- assertEquals(admin.topics().getPartitionedTopicMetadata(partitionedTopicName).partitions, 0);
-
- admin.topics().createPartitionedTopic(partitionedTopicName, 32);
-
- assertEquals(admin.topics().getPartitionedTopicMetadata(partitionedTopicName).partitions, 32);
-
- try {
- admin.topics().deletePartitionedTopic("persistent://prop-xyz/ns1/ds2");
- fail("Should have failed as the partitioned topic was not created");
- } catch (NotFoundException nfe) {
- }
-
- admin.topics().deletePartitionedTopic(partitionedTopicName);
-
- // delete a partitioned topic in a global namespace
- admin.topics().createPartitionedTopic(partitionedTopicName, 4);
- admin.topics().deletePartitionedTopic(partitionedTopicName);
- }
-
- @Test(dataProvider = "numBundles")
- public void testDeleteNamespaceBundle(Integer numBundles) throws Exception {
- admin.namespaces().deleteNamespace("prop-xyz/ns1");
- admin.namespaces().createNamespace("prop-xyz/ns1-bundles", numBundles);
- admin.namespaces().setNamespaceReplicationClusters("prop-xyz/ns1-bundles", Sets.newHashSet("test"));
-
- // since we have 2 brokers running, we try to let both of them acquire bundle ownership
- admin.lookups().lookupTopic("persistent://prop-xyz/ns1-bundles/ds1");
- admin.lookups().lookupTopic("persistent://prop-xyz/ns1-bundles/ds2");
- admin.lookups().lookupTopic("persistent://prop-xyz/ns1-bundles/ds3");
- admin.lookups().lookupTopic("persistent://prop-xyz/ns1-bundles/ds4");
-
- assertEquals(admin.namespaces().getTopics("prop-xyz/ns1-bundles"), Lists.newArrayList());
-
- admin.namespaces().deleteNamespace("prop-xyz/ns1-bundles");
- assertEquals(admin.namespaces().getNamespaces("prop-xyz", "test"), Lists.newArrayList());
- }
-
- @Test
- public void testNamespaceSplitBundle() throws Exception {
- // Force to create a topic
- final String namespace = "prop-xyz/ns1";
- final String topicName = (new StringBuilder("persistent://")).append(namespace).append("/ds2").toString();
- Producer<byte[]> producer = pulsarClient.newProducer(Schema.BYTES)
- .topic(topicName)
- .enableBatching(false)
- .messageRoutingMode(MessageRoutingMode.SinglePartition)
- .create();
- producer.send("message".getBytes());
- publishMessagesOnPersistentTopic(topicName, 0);
- assertEquals(admin.topics().getList(namespace), Lists.newArrayList(topicName));
-
- try {
- admin.namespaces().splitNamespaceBundle(namespace, "0x00000000_0xffffffff", true);
- } catch (Exception e) {
- fail("split bundle shouldn't have thrown exception");
- }
-
- // bundle-factory cache must have updated split bundles
- NamespaceBundles bundles = bundleFactory.getBundles(NamespaceName.get(namespace));
- String[] splitRange = { namespace + "/0x00000000_0x7fffffff", namespace + "/0x7fffffff_0xffffffff" };
- for (int i = 0; i < bundles.getBundles().size(); i++) {
- assertEquals(bundles.getBundles().get(i).toString(), splitRange[i]);
- }
-
- producer.close();
- }
-
- @Test
- public void testNamespaceSplitBundleConcurrent() throws Exception {
- // Force to create a topic
- final String namespace = "prop-xyz/ns1";
- final String topicName = (new StringBuilder("persistent://")).append(namespace).append("/ds2").toString();
- Producer<byte[]> producer = pulsarClient.newProducer(Schema.BYTES)
- .topic(topicName)
- .enableBatching(false)
- .messageRoutingMode(MessageRoutingMode.SinglePartition)
- .create();
- producer.send("message".getBytes());
- publishMessagesOnPersistentTopic(topicName, 0);
- assertEquals(admin.topics().getList(namespace), Lists.newArrayList(topicName));
-
- try {
- admin.namespaces().splitNamespaceBundle(namespace, "0x00000000_0xffffffff", false);
- } catch (Exception e) {
- fail("split bundle shouldn't have thrown exception");
- }
-
- // bundle-factory cache must have updated split bundles
- NamespaceBundles bundles = bundleFactory.getBundles(NamespaceName.get(namespace));
- String[] splitRange = { namespace + "/0x00000000_0x7fffffff", namespace + "/0x7fffffff_0xffffffff" };
- for (int i = 0; i < bundles.getBundles().size(); i++) {
- assertEquals(bundles.getBundles().get(i).toString(), splitRange[i]);
- }
-
- ExecutorService executorService = Executors.newCachedThreadPool();
-
- try {
- executorService.invokeAll(Arrays.asList(() -> {
- log.info("split 2 bundles at the same time. spilt: 0x00000000_0x7fffffff ");
- admin.namespaces().splitNamespaceBundle(namespace, "0x00000000_0x7fffffff", false);
- return null;
- }, () -> {
- log.info("split 2 bundles at the same time. spilt: 0x7fffffff_0xffffffff ");
- admin.namespaces().splitNamespaceBundle(namespace, "0x7fffffff_0xffffffff", false);
- return null;
- }));
- } catch (Exception e) {
- fail("split bundle shouldn't have thrown exception");
- }
-
- String[] splitRange4 = { namespace + "/0x00000000_0x3fffffff", namespace + "/0x3fffffff_0x7fffffff",
- namespace + "/0x7fffffff_0xbfffffff", namespace + "/0xbfffffff_0xffffffff" };
- bundles = bundleFactory.getBundles(NamespaceName.get(namespace));
- assertEquals(bundles.getBundles().size(), 4);
- for (int i = 0; i < bundles.getBundles().size(); i++) {
- assertEquals(bundles.getBundles().get(i).toString(), splitRange4[i]);
- }
-
- try {
- executorService.invokeAll(Arrays.asList(() -> {
- log.info("split 4 bundles at the same time. spilt: 0x00000000_0x3fffffff ");
- admin.namespaces().splitNamespaceBundle(namespace, "0x00000000_0x3fffffff", false);
- return null;
- }, () -> {
- log.info("split 4 bundles at the same time. spilt: 0x3fffffff_0x7fffffff ");
- admin.namespaces().splitNamespaceBundle(namespace, "0x3fffffff_0x7fffffff", false);
- return null;
- }, () -> {
- log.info("split 4 bundles at the same time. spilt: 0x7fffffff_0xbfffffff ");
- admin.namespaces().splitNamespaceBundle(namespace, "0x7fffffff_0xbfffffff", false);
- return null;
- }, () -> {
- log.info("split 4 bundles at the same time. spilt: 0xbfffffff_0xffffffff ");
- admin.namespaces().splitNamespaceBundle(namespace, "0xbfffffff_0xffffffff", false);
- return null;
- }));
- } catch (Exception e) {
- fail("split bundle shouldn't have thrown exception");
- }
-
- String[] splitRange8 = { namespace + "/0x00000000_0x1fffffff", namespace + "/0x1fffffff_0x3fffffff",
- namespace + "/0x3fffffff_0x5fffffff", namespace + "/0x5fffffff_0x7fffffff",
- namespace + "/0x7fffffff_0x9fffffff", namespace + "/0x9fffffff_0xbfffffff",
- namespace + "/0xbfffffff_0xdfffffff", namespace + "/0xdfffffff_0xffffffff" };
- bundles = bundleFactory.getBundles(NamespaceName.get(namespace));
- assertEquals(bundles.getBundles().size(), 8);
- for (int i = 0; i < bundles.getBundles().size(); i++) {
- assertEquals(bundles.getBundles().get(i).toString(), splitRange8[i]);
- }
-
- producer.close();
- }
-
- @Test
- public void testNamespaceUnloadBundle() throws Exception {
- assertEquals(admin.topics().getList("prop-xyz/ns1"), Lists.newArrayList());
-
- // Force to create a topic
- publishMessagesOnPersistentTopic("persistent://prop-xyz/ns1/ds2", 0);
- assertEquals(admin.topics().getList("prop-xyz/ns1"),
- Lists.newArrayList("persistent://prop-xyz/ns1/ds2"));
-
- // create consumer and subscription
- Consumer<byte[]> consumer = pulsarClient.newConsumer().topic("persistent://prop-xyz/ns1/ds2")
- .subscriptionName("my-sub").subscribe();
- assertEquals(admin.topics().getSubscriptions("persistent://prop-xyz/ns1/ds2"),
- Lists.newArrayList("my-sub"));
-
- // Create producer
- Producer<byte[]> producer = pulsarClient.newProducer(Schema.BYTES)
- .topic("persistent://prop-xyz/ns1/ds2")
- .enableBatching(false)
- .messageRoutingMode(MessageRoutingMode.SinglePartition)
- .create();
- for (int i = 0; i < 10; i++) {
- String message = "message-" + i;
- producer.send(message.getBytes());
- }
-
- consumer.close();
- producer.close();
-
- try {
- admin.namespaces().unloadNamespaceBundle("prop-xyz/ns1", "0x00000000_0xffffffff");
- } catch (Exception e) {
- fail("Unload shouldn't have throw exception");
- }
-
- // check that no one owns the namespace
- NamespaceBundle bundle = bundleFactory.getBundle(NamespaceName.get("prop-xyz/ns1"),
- Range.range(0L, BoundType.CLOSED, 0xffffffffL, BoundType.CLOSED));
- assertFalse(pulsar.getNamespaceService().isServiceUnitOwned(bundle));
- assertFalse(otherPulsar.getNamespaceService().isServiceUnitOwned(bundle));
- pulsarClient.shutdown();
-
- LOG.info("--- RELOAD ---");
-
- // Force reload of namespace and wait for topic to be ready
- for (int i = 0; i < 30; i++) {
- try {
- admin.topics().getStats("persistent://prop-xyz/ns1/ds2");
- break;
- } catch (PulsarAdminException e) {
- LOG.warn("Failed to get topic stats.. {}", e.getMessage());
- Thread.sleep(1000);
- }
- }
-
- admin.topics().deleteSubscription("persistent://prop-xyz/ns1/ds2", "my-sub");
- admin.topics().delete("persistent://prop-xyz/ns1/ds2");
- }
-
- @Test(dataProvider = "numBundles")
- public void testNamespaceBundleUnload(Integer numBundles) throws Exception {
- admin.namespaces().createNamespace("prop-xyz/ns1-bundles", numBundles);
- admin.namespaces().setNamespaceReplicationClusters("prop-xyz/ns1-bundles", Sets.newHashSet("test"));
-
- assertEquals(admin.topics().getList("prop-xyz/ns1-bundles"), Lists.newArrayList());
-
- // Force to create a topic
- publishMessagesOnPersistentTopic("persistent://prop-xyz/ns1-bundles/ds2", 0);
- assertEquals(admin.topics().getList("prop-xyz/ns1-bundles"),
- Lists.newArrayList("persistent://prop-xyz/ns1-bundles/ds2"));
-
- // create consumer and subscription
- Consumer<byte[]> consumer = pulsarClient.newConsumer().topic("persistent://prop-xyz/ns1-bundles/ds2")
- .subscriptionName("my-sub").subscribe();
- assertEquals(admin.topics().getSubscriptions("persistent://prop-xyz/ns1-bundles/ds2"),
- Lists.newArrayList("my-sub"));
-
- // Create producer
- Producer<byte[]> producer = pulsarClient.newProducer(Schema.BYTES)
- .topic("persistent://prop-xyz/ns1-bundles/ds2")
- .enableBatching(false)
- .messageRoutingMode(MessageRoutingMode.SinglePartition)
- .create();
- for (int i = 0; i < 10; i++) {
- String message = "message-" + i;
- producer.send(message.getBytes());
- }
-
- NamespaceBundle bundle = (NamespaceBundle) pulsar.getNamespaceService()
- .getBundle(TopicName.get("persistent://prop-xyz/ns1-bundles/ds2"));
-
- consumer.close();
- producer.close();
-
- admin.namespaces().unloadNamespaceBundle("prop-xyz/ns1-bundles", bundle.getBundleRange());
-
- // check that no one owns the namespace bundle
- assertFalse(pulsar.getNamespaceService().isServiceUnitOwned(bundle));
- assertFalse(otherPulsar.getNamespaceService().isServiceUnitOwned(bundle));
-
- LOG.info("--- RELOAD ---");
-
- // Force reload of namespace and wait for topic to be ready
- for (int i = 0; i < 30; i++) {
- try {
- admin.topics().getStats("persistent://prop-xyz/ns1-bundles/ds2");
- break;
- } catch (PulsarAdminException e) {
- LOG.warn("Failed to get topic stats.. {}", e.getMessage());
- Thread.sleep(1000);
- }
- }
-
- admin.topics().deleteSubscription("persistent://prop-xyz/ns1-bundles/ds2", "my-sub");
- admin.topics().delete("persistent://prop-xyz/ns1-bundles/ds2");
- }
-
- @Test(dataProvider = "bundling")
- public void testClearBacklogOnNamespace(Integer numBundles) throws Exception {
- admin.namespaces().createNamespace("prop-xyz/ns1-bundles", numBundles);
- admin.namespaces().setNamespaceReplicationClusters("prop-xyz/ns1-bundles", Sets.newHashSet("test"));
-
- // create consumer and subscription
- pulsarClient.newConsumer().topic("persistent://prop-xyz/ns1-bundles/ds2").subscriptionName("my-sub")
- .subscribe();
- pulsarClient.newConsumer().topic("persistent://prop-xyz/ns1-bundles/ds2").subscriptionName("my-sub-1")
- .subscribe();
- pulsarClient.newConsumer().topic("persistent://prop-xyz/ns1-bundles/ds2").subscriptionName("my-sub-2")
- .subscribe();
- pulsarClient.newConsumer().topic("persistent://prop-xyz/ns1-bundles/ds1").subscriptionName("my-sub")
- .subscribe();
- pulsarClient.newConsumer().topic("persistent://prop-xyz/ns1-bundles/ds1").subscriptionName("my-sub-1")
- .subscribe();
-
- // Create producer
- Producer<byte[]> producer = pulsarClient.newProducer(Schema.BYTES)
- .topic("persistent://prop-xyz/ns1-bundles/ds2")
- .enableBatching(false)
- .messageRoutingMode(MessageRoutingMode.SinglePartition)
- .create();
- for (int i = 0; i < 10; i++) {
- String message = "message-" + i;
- producer.send(message.getBytes());
- }
-
- producer.close();
-
- // Create producer
- Producer<byte[]> producer1 = pulsarClient.newProducer(Schema.BYTES)
- .topic("persistent://prop-xyz/ns1-bundles/ds1")
- .enableBatching(false)
- .messageRoutingMode(MessageRoutingMode.SinglePartition)
- .create();
- for (int i = 0; i < 10; i++) {
- String message = "message-" + i;
- producer1.send(message.getBytes());
- }
-
- producer1.close();
-
- admin.namespaces().clearNamespaceBacklogForSubscription("prop-xyz/ns1-bundles", "my-sub");
-
- long backlog = admin.topics().getStats("persistent://prop-xyz/ns1-bundles/ds2").subscriptions
- .get("my-sub").msgBacklog;
- assertEquals(backlog, 0);
- backlog = admin.topics().getStats("persistent://prop-xyz/ns1-bundles/ds1").subscriptions
- .get("my-sub").msgBacklog;
- assertEquals(backlog, 0);
- backlog = admin.topics().getStats("persistent://prop-xyz/ns1-bundles/ds1").subscriptions
- .get("my-sub-1").msgBacklog;
- assertEquals(backlog, 10);
-
- admin.namespaces().clearNamespaceBacklog("prop-xyz/ns1-bundles");
-
- backlog = admin.topics().getStats("persistent://prop-xyz/ns1-bundles/ds1").subscriptions
- .get("my-sub-1").msgBacklog;
- assertEquals(backlog, 0);
- backlog = admin.topics().getStats("persistent://prop-xyz/ns1-bundles/ds2").subscriptions
- .get("my-sub-1").msgBacklog;
- assertEquals(backlog, 0);
- backlog = admin.topics().getStats("persistent://prop-xyz/ns1-bundles/ds2").subscriptions
- .get("my-sub-2").msgBacklog;
- assertEquals(backlog, 0);
- }
-
- @Test(dataProvider = "bundling")
- public void testUnsubscribeOnNamespace(Integer numBundles) throws Exception {
- admin.namespaces().createNamespace("prop-xyz/ns1-bundles", numBundles);
- admin.namespaces().setNamespaceReplicationClusters("prop-xyz/ns1-bundles", Sets.newHashSet("test"));
-
- // create consumer and subscription
- Consumer<byte[]> consumer1 = pulsarClient.newConsumer().topic("persistent://prop-xyz/ns1-bundles/ds2")
- .subscriptionName("my-sub").subscribe();
- Consumer<byte[]> consumer2 = pulsarClient.newConsumer().topic("persistent://prop-xyz/ns1-bundles/ds2")
- .subscriptionName("my-sub-1").subscribe();
- /* Consumer consumer3 = */ pulsarClient.newConsumer().topic("persistent://prop-xyz/ns1-bundles/ds2")
- .subscriptionName("my-sub-2").subscribe();
- Consumer<byte[]> consumer4 = pulsarClient.newConsumer().topic("persistent://prop-xyz/ns1-bundles/ds1")
- .subscriptionName("my-sub").subscribe();
- Consumer<byte[]> consumer5 = pulsarClient.newConsumer().topic("persistent://prop-xyz/ns1-bundles/ds1")
- .subscriptionName("my-sub-1").subscribe();
-
- try {
- admin.namespaces().unsubscribeNamespace("prop-xyz/ns1-bundles", "my-sub");
- fail("should have failed");
- } catch (PulsarAdminException.PreconditionFailedException e) {
- // ok
- }
-
- consumer1.close();
-
- try {
- admin.namespaces().unsubscribeNamespace("prop-xyz/ns1-bundles", "my-sub");
- fail("should have failed");
- } catch (PulsarAdminException.PreconditionFailedException e) {
- // ok
- }
-
- consumer4.close();
-
- admin.namespaces().unsubscribeNamespace("prop-xyz/ns1-bundles", "my-sub");
-
- assertEquals(admin.topics().getSubscriptions("persistent://prop-xyz/ns1-bundles/ds2"),
- Lists.newArrayList("my-sub-1", "my-sub-2"));
- assertEquals(admin.topics().getSubscriptions("persistent://prop-xyz/ns1-bundles/ds1"),
- Lists.newArrayList("my-sub-1"));
-
- consumer2.close();
- consumer5.close();
-
- admin.namespaces().unsubscribeNamespace("prop-xyz/ns1-bundles", "my-sub-1");
-
- assertEquals(admin.topics().getSubscriptions("persistent://prop-xyz/ns1-bundles/ds2"),
- Lists.newArrayList("my-sub-2"));
- assertEquals(admin.topics().getSubscriptions("persistent://prop-xyz/ns1-bundles/ds1"),
- Lists.newArrayList());
- }
-
- long messageTimestamp = System.currentTimeMillis();
- long secondTimestamp = System.currentTimeMillis();
-
- private void publishMessagesOnPersistentTopic(String topicName, int messages) throws Exception {
- publishMessagesOnPersistentTopic(topicName, messages, 0);
- }
-
- private void publishMessagesOnPersistentTopic(String topicName, int messages, int startIdx) throws Exception {
- Producer<byte[]> producer = pulsarClient.newProducer(Schema.BYTES)
- .topic(topicName)
- .enableBatching(false)
- .messageRoutingMode(MessageRoutingMode.SinglePartition)
- .create();
-
- for (int i = startIdx; i < (messages + startIdx); i++) {
- String message = "message-" + i;
- producer.send(message.getBytes());
- }
-
- producer.close();
- }
-
- @Test
- public void backlogQuotas() throws Exception {
- assertEquals(admin.namespaces().getBacklogQuotaMap("prop-xyz/ns1"), Maps.newTreeMap());
-
- Map<BacklogQuotaType, BacklogQuota> quotaMap = admin.namespaces().getBacklogQuotaMap("prop-xyz/ns1");
- assertEquals(quotaMap.size(), 0);
- assertEquals(quotaMap.get(BacklogQuotaType.destination_storage), null);
-
- admin.namespaces().setBacklogQuota("prop-xyz/ns1",
- new BacklogQuota(1 * 1024 * 1024 * 1024, RetentionPolicy.producer_exception));
- quotaMap = admin.namespaces().getBacklogQuotaMap("prop-xyz/ns1");
- assertEquals(quotaMap.size(), 1);
- assertEquals(quotaMap.get(BacklogQuotaType.destination_storage),
- new BacklogQuota(1 * 1024 * 1024 * 1024, RetentionPolicy.producer_exception));
-
- admin.namespaces().removeBacklogQuota("prop-xyz/ns1");
-
- quotaMap = admin.namespaces().getBacklogQuotaMap("prop-xyz/ns1");
- assertEquals(quotaMap.size(), 0);
- assertEquals(quotaMap.get(BacklogQuotaType.destination_storage), null);
- }
-
- @Test
- public void statsOnNonExistingTopics() throws Exception {
- try {
- admin.topics().getStats("persistent://prop-xyz/ns1/ghostTopic");
- fail("The topic doesn't exist");
- } catch (NotFoundException e) {
- // OK
- }
- }
-
- @Test
- public void testDeleteFailedReturnCode() throws Exception {
- String topicName = "persistent://prop-xyz/ns1/my-topic";
- Producer<byte[]> producer = pulsarClient.newProducer(Schema.BYTES)
- .topic(topicName)
- .enableBatching(false)
- .messageRoutingMode(MessageRoutingMode.SinglePartition)
- .create();
-
- try {
- admin.topics().delete(topicName);
- fail("The topic is busy");
- } catch (PreconditionFailedException e) {
- // OK
- }
-
- producer.close();
-
- Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("sub").subscribe();
-
- try {
- admin.topics().delete(topicName);
- fail("The topic is busy");
- } catch (PreconditionFailedException e) {
- // OK
- }
-
- try {
- admin.topics().deleteSubscription(topicName, "sub");
- fail("The topic is busy");
- } catch (PreconditionFailedException e) {
- // Ok
- }
-
- consumer.close();
-
- // Now should succeed
- admin.topics().delete(topicName);
- }
-
- private static class IncompatibleTenantAdmin {
- public Set<String> allowedClusters;
- public int someNewIntField;
- public String someNewString;
- }
-
- @Test
- public void testJacksonWithTypeDifferencies() throws Exception {
- String expectedJson = "{\"adminRoles\":[\"role1\",\"role2\"],\"allowedClusters\":[\"usw\",\"test\"]}";
- IncompatibleTenantAdmin r1 = ObjectMapperFactory.getThreadLocal().readerFor(IncompatibleTenantAdmin.class)
- .readValue(expectedJson);
- assertEquals(r1.allowedClusters, Sets.newHashSet("test", "usw"));
- assertEquals(r1.someNewIntField, 0);
- assertEquals(r1.someNewString, null);
- }
-
- @Test
- public void testBackwardCompatiblity() throws Exception {
- assertEquals(admin.tenants().getTenants(), Lists.newArrayList("prop-xyz"));
- assertEquals(admin.tenants().getTenantInfo("prop-xyz").getAdminRoles(),
- Lists.newArrayList("role1", "role2"));
- assertEquals(admin.tenants().getTenantInfo("prop-xyz").getAllowedClusters(), Sets.newHashSet("test"));
-
- // Try to deserialize property JSON with IncompatibleTenantAdmin format
- // it should succeed ignoring missing fields
- TenantsImpl properties = (TenantsImpl) admin.tenants();
- IncompatibleTenantAdmin result = properties.request(properties.getWebTarget().path("prop-xyz"))
- .get(IncompatibleTenantAdmin.class);
-
- assertEquals(result.allowedClusters, Sets.newHashSet("test"));
- assertEquals(result.someNewIntField, 0);
- assertEquals(result.someNewString, null);
-
- admin.namespaces().deleteNamespace("prop-xyz/ns1");
- admin.tenants().deleteTenant("prop-xyz");
- assertEquals(admin.tenants().getTenants(), Lists.newArrayList());
- }
-
- @Test(dataProvider = "topicName")
- public void persistentTopicsCursorReset(String topicName) throws Exception {
- admin.namespaces().setRetention("prop-xyz/ns1", new RetentionPolicies(10, 10));
-
- assertEquals(admin.topics().getList("prop-xyz/ns1"), Lists.newArrayList());
-
- topicName = "persistent://prop-xyz/ns1/" + topicName;
-
- // create consumer and subscription
- Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("my-sub")
- .subscriptionType(SubscriptionType.Exclusive).acknowledgmentGroupTime(0, TimeUnit.SECONDS).subscribe();
-
- assertEquals(admin.topics().getSubscriptions(topicName), Lists.newArrayList("my-sub"));
-
- publishMessagesOnPersistentTopic(topicName, 5, 0);
-
- // Allow at least 1ms for messages to have different timestamps
- Thread.sleep(1);
- long messageTimestamp = System.currentTimeMillis();
-
- publishMessagesOnPersistentTopic(topicName, 5, 5);
-
- List<Message<byte[]>> messages = admin.topics().peekMessages(topicName, "my-sub", 10);
- assertEquals(messages.size(), 10);
-
- for (int i = 0; i < 10; i++) {
- Message<byte[]> message = consumer.receive();
- consumer.acknowledge(message);
- }
- // messages should still be available due to retention
-
- admin.topics().resetCursor(topicName, "my-sub", messageTimestamp);
-
- int receivedAfterReset = 0;
-
- for (int i = 4; i < 10; i++) {
- Message<byte[]> message = consumer.receive();
- consumer.acknowledge(message);
- ++receivedAfterReset;
- String expected = "message-" + i;
- assertEquals(message.getData(), expected.getBytes());
- }
- assertEquals(receivedAfterReset, 6);
-
- consumer.close();
-
- admin.topics().deleteSubscription(topicName, "my-sub");
-
- assertEquals(admin.topics().getSubscriptions(topicName), Lists.newArrayList());
- admin.topics().delete(topicName);
- }
-
- @Test(dataProvider = "topicName")
- public void persistentTopicsCursorResetAfterReset(String topicName) throws Exception {
- admin.namespaces().setRetention("prop-xyz/ns1", new RetentionPolicies(10, 10));
- assertEquals(admin.topics().getList("prop-xyz/ns1"), Lists.newArrayList());
-
- topicName = "persistent://prop-xyz/ns1/" + topicName;
-
- // create consumer and subscription
- Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("my-sub")
- .subscriptionType(SubscriptionType.Exclusive).acknowledgmentGroupTime(0, TimeUnit.SECONDS).subscribe();
-
- assertEquals(admin.topics().getSubscriptions(topicName), Lists.newArrayList("my-sub"));
-
- publishMessagesOnPersistentTopic(topicName, 5, 0);
-
- // Allow at least 1ms for messages to have different timestamps
- Thread.sleep(1);
- long firstTimestamp = System.currentTimeMillis();
- publishMessagesOnPersistentTopic(topicName, 3, 5);
-
- Thread.sleep(1);
- long secondTimestamp = System.currentTimeMillis();
-
- publishMessagesOnPersistentTopic(topicName, 2, 8);
-
- List<Message<byte[]>> messages = admin.topics().peekMessages(topicName, "my-sub", 10);
- assertEquals(messages.size(), 10);
- messages.forEach(message -> {
- LOG.info("Peeked message: {}", new String(message.getData()));
- });
-
- for (int i = 0; i < 10; i++) {
- Message<byte[]> message = consumer.receive();
- consumer.acknowledge(message);
- }
-
- admin.topics().resetCursor(topicName, "my-sub", firstTimestamp);
-
- int receivedAfterReset = 0;
-
- // Should received messages from 4-9
- for (int i = 4; i < 10; i++) {
- Message<byte[]> message = consumer.receive();
- consumer.acknowledge(message);
- ++receivedAfterReset;
- String expected = "message-" + i;
- assertEquals(new String(message.getData()), expected);
- }
- assertEquals(receivedAfterReset, 6);
-
- // Reset at 2nd timestamp
- receivedAfterReset = 0;
- admin.topics().resetCursor(topicName, "my-sub", secondTimestamp);
-
- // Should received messages from 7-9
- for (int i = 7; i < 10; i++) {
- Message<byte[]> message = consumer.receive();
- consumer.acknowledge(message);
- ++receivedAfterReset;
- String expected = "message-" + i;
- assertEquals(new String(message.getData()), expected);
- }
- assertEquals(receivedAfterReset, 3);
-
- consumer.close();
- admin.topics().deleteSubscription(topicName, "my-sub");
-
- assertEquals(admin.topics().getSubscriptions(topicName), Lists.newArrayList());
- admin.topics().delete(topicName);
- }
-
- @Test(dataProvider = "topicName")
- public void partitionedTopicsCursorReset(String topicName) throws Exception {
- admin.namespaces().setRetention("prop-xyz/ns1", new RetentionPolicies(10, 10));
- topicName = "persistent://prop-xyz/ns1/" + topicName;
-
- admin.topics().createPartitionedTopic(topicName, 4);
-
- // create consumer and subscription
- Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("my-sub")
- .subscriptionType(SubscriptionType.Exclusive).acknowledgmentGroupTime(0, TimeUnit.SECONDS).subscribe();
-
- List<String> topics = admin.topics().getList("prop-xyz/ns1");
- assertEquals(topics.size(), 4);
-
- assertEquals(admin.topics().getSubscriptions(topicName), Lists.newArrayList("my-sub"));
-
- publishMessagesOnPersistentTopic(topicName, 5, 0);
- Thread.sleep(1);
-
- long timestamp = System.currentTimeMillis();
- publishMessagesOnPersistentTopic(topicName, 5, 5);
-
- for (int i = 0; i < 10; i++) {
- Message<byte[]> message = consumer.receive();
- consumer.acknowledge(message);
- }
- // messages should still be available due to retention
-
- admin.topics().resetCursor(topicName, "my-sub", timestamp);
-
- Set<String> expectedMessages = Sets.newHashSet();
- Set<String> receivedMessages = Sets.newHashSet();
- for (int i = 4; i < 10; i++) {
- Message<byte[]> message = consumer.receive();
- consumer.acknowledge(message);
- expectedMessages.add("message-" + i);
- receivedMessages.add(new String(message.getData()));
- }
-
- receivedMessages.removeAll(expectedMessages);
- assertEquals(receivedMessages.size(), 0);
-
- consumer.close();
- admin.topics().deleteSubscription(topicName, "my-sub");
- admin.topics().deletePartitionedTopic(topicName);
- }
-
- @Test
- public void persistentTopicsInvalidCursorReset() throws Exception {
- admin.namespaces().setRetention("prop-xyz/ns1", new RetentionPolicies(10, 10));
-
- assertEquals(admin.topics().getList("prop-xyz/ns1"), Lists.newArrayList());
-
- String topicName = "persistent://prop-xyz/ns1/invalidcursorreset";
- // Force to create a topic
- publishMessagesOnPersistentTopic(topicName, 0);
- assertEquals(admin.topics().getList("prop-xyz/ns1"), Lists.newArrayList(topicName));
-
- // create consumer and subscription
- URL pulsarUrl = new URL("http://127.0.0.1" + ":" + BROKER_WEBSERVICE_PORT);
- PulsarClient client = PulsarClient.builder().serviceUrl(pulsarUrl.toString()).statsInterval(0, TimeUnit.SECONDS)
- .build();
- Consumer<byte[]> consumer = client.newConsumer().topic(topicName).subscriptionName("my-sub")
- .subscriptionType(SubscriptionType.Exclusive).subscribe();
-
- assertEquals(admin.topics().getSubscriptions(topicName), Lists.newArrayList("my-sub"));
-
- publishMessagesOnPersistentTopic(topicName, 10);
-
- List<Message<byte[]>> messages = admin.topics().peekMessages(topicName, "my-sub", 10);
- assertEquals(messages.size(), 10);
-
- for (int i = 0; i < 10; i++) {
- Message<byte[]> message = consumer.receive();
- consumer.acknowledge(message);
- }
- // use invalid timestamp
- try {
- admin.topics().resetCursor(topicName, "my-sub", System.currentTimeMillis() - 190000);
- } catch (Exception e) {
- // fail the test
- throw e;
- }
-
- admin.topics().resetCursor(topicName, "my-sub", System.currentTimeMillis() + 90000);
- consumer = client.newConsumer().topic(topicName).subscriptionName("my-sub").subscribe();
- consumer.close();
- client.close();
-
- admin.topics().deleteSubscription(topicName, "my-sub");
-
- assertEquals(admin.topics().getSubscriptions(topicName), Lists.newArrayList());
- admin.topics().delete(topicName);
- }
-
- @Test
- public void testObjectWithUnknowProperties() {
-
- class CustomTenantAdmin extends TenantInfo {
- @SuppressWarnings("unused")
- public int newTenant;
- }
-
- TenantInfo pa = new TenantInfo(Sets.newHashSet("test_appid1", "test_appid2"), Sets.newHashSet("test"));
- CustomTenantAdmin cpa = new CustomTenantAdmin();
- cpa.setAdminRoles(pa.getAdminRoles());
- cpa.setAllowedClusters(pa.getAllowedClusters());
- cpa.newTenant = 100;
-
- try {
- admin.tenants().createTenant("test-property", cpa);
- } catch (Exception e) {
- fail("Should not happen : ", e);
- }
- }
-
- /**
- * <pre>
- * Verify: PersistentTopicsBase.expireMessages()/expireMessagesForAllSubscriptions()
- * 1. Created multiple shared subscriptions and publisher on topic
- * 2. Publish messages on the topic
- * 3. expire message on sub-1 : backlog for sub-1 must be 0
- * 4. expire message on all subscriptions: backlog for all subscription must be 0
- * </pre>
- *
- * @throws Exception
- */
- @Test
- public void testPersistentTopicsExpireMessages() throws Exception {
-
- // Force to create a topic
- publishMessagesOnPersistentTopic("persistent://prop-xyz/ns1/ds2", 0);
- assertEquals(admin.topics().getList("prop-xyz/ns1"),
- Lists.newArrayList("persistent://prop-xyz/ns1/ds2"));
-
- // create consumer and subscription
- URL pulsarUrl = new URL("http://127.0.0.1" + ":" + BROKER_WEBSERVICE_PORT);
- PulsarClient client = PulsarClient.builder().serviceUrl(pulsarUrl.toString()).statsInterval(0, TimeUnit.SECONDS)
- .build();
- ConsumerBuilder<byte[]> consumerBuilder = client.newConsumer().topic("persistent://prop-xyz/ns1/ds2")
- .subscriptionType(SubscriptionType.Shared);
- Consumer<byte[]> consumer1 = consumerBuilder.clone().subscriptionName("my-sub1").subscribe();
- Consumer<byte[]> consumer2 = consumerBuilder.clone().subscriptionName("my-sub2").subscribe();
- Consumer<byte[]> consumer3 = consumerBuilder.clone().subscriptionName("my-sub3").subscribe();
-
- assertEquals(admin.topics().getSubscriptions("persistent://prop-xyz/ns1/ds2").size(), 3);
-
- publishMessagesOnPersistentTopic("persistent://prop-xyz/ns1/ds2", 10);
-
- TopicStats topicStats = admin.topics().getStats("persistent://prop-xyz/ns1/ds2");
- assertEquals(topicStats.subscriptions.get("my-sub1").msgBacklog, 10);
- assertEquals(topicStats.subscriptions.get("my-sub2").msgBacklog, 10);
- assertEquals(topicStats.subscriptions.get("my-sub3").msgBacklog, 10);
-
- Thread.sleep(1000); // wait for 1 seconds to expire message
- admin.topics().expireMessages("persistent://prop-xyz/ns1/ds2", "my-sub1", 1);
- Thread.sleep(1000); // wait for 1 seconds to execute expire message as it is async
-
- topicStats = admin.topics().getStats("persistent://prop-xyz/ns1/ds2");
- assertEquals(topicStats.subscriptions.get("my-sub1").msgBacklog, 0);
- assertEquals(topicStats.subscriptions.get("my-sub2").msgBacklog, 10);
- assertEquals(topicStats.subscriptions.get("my-sub3").msgBacklog, 10);
-
- admin.topics().expireMessagesForAllSubscriptions("persistent://prop-xyz/ns1/ds2", 1);
- Thread.sleep(1000); // wait for 1 seconds to execute expire message as it is async
-
- topicStats = admin.topics().getStats("persistent://prop-xyz/ns1/ds2");
- assertEquals(topicStats.subscriptions.get("my-sub1").msgBacklog, 0);
- assertEquals(topicStats.subscriptions.get("my-sub2").msgBacklog, 0);
- assertEquals(topicStats.subscriptions.get("my-sub3").msgBacklog, 0);
-
- consumer1.close();
- consumer2.close();
- consumer3.close();
-
- }
-
- /**
- * Verify: PersistentTopicsBase.expireMessages()/expireMessagesForAllSubscriptions() for PartitionTopic
- *
- * @throws Exception
- */
- @Test
- public void testPersistentTopicExpireMessageOnParitionTopic() throws Exception {
-
- admin.topics().createPartitionedTopic("persistent://prop-xyz/ns1/ds1", 4);
-
- // create consumer and subscription
- URL pulsarUrl = new URL("http://127.0.0.1" + ":" + BROKER_WEBSERVICE_PORT);
- PulsarClient client = PulsarClient.builder().serviceUrl(pulsarUrl.toString()).statsInterval(0, TimeUnit.SECONDS)
- .build();
- Consumer<byte[]> consumer = client.newConsumer().topic("persistent://prop-xyz/ns1/ds1")
- .subscriptionName("my-sub").subscribe();
-
- Producer<byte[]> producer = client.newProducer(Schema.BYTES)
- .topic("persistent://prop-xyz/ns1/ds1")
- .enableBatching(false)
- .messageRoutingMode(MessageRoutingMode.RoundRobinPartition)
- .create();
- for (int i = 0; i < 10; i++) {
- String message = "message-" + i;
- producer.send(message.getBytes());
- }
-
- PartitionedTopicStats topicStats = admin.topics().getPartitionedStats("persistent://prop-xyz/ns1/ds1",
- true);
- assertEquals(topicStats.subscriptions.get("my-sub").msgBacklog, 10);
-
- TopicStats partitionStatsPartition0 = topicStats.partitions
- .get("persistent://prop-xyz/ns1/ds1-partition-0");
- TopicStats partitionStatsPartition1 = topicStats.partitions
- .get("persistent://prop-xyz/ns1/ds1-partition-1");
- assertEquals(partitionStatsPartition0.subscriptions.get("my-sub").msgBacklog, 3, 1);
- assertEquals(partitionStatsPartition1.subscriptions.get("my-sub").msgBacklog, 3, 1);
-
- Thread.sleep(1000);
- admin.topics().expireMessagesForAllSubscriptions("persistent://prop-xyz/ns1/ds1", 1);
- Thread.sleep(1000);
-
- topicStats = admin.topics().getPartitionedStats("persistent://prop-xyz/ns1/ds1", true);
- partitionStatsPartition0 = topicStats.partitions.get("persistent://prop-xyz/ns1/ds1-partition-0");
- partitionStatsPartition1 = topicStats.partitions.get("persistent://prop-xyz/ns1/ds1-partition-1");
- assertEquals(partitionStatsPartition0.subscriptions.get("my-sub").msgBacklog, 0);
- assertEquals(partitionStatsPartition1.subscriptions.get("my-sub").msgBacklog, 0);
-
- producer.close();
- consumer.close();
- client.close();
-
- }
-
- /**
- * This test-case verifies that broker should support both url/uri encoding for topic-name. It calls below api with
- * url-encoded and also uri-encoded topic-name in http request: a. PartitionedMetadataLookup b. TopicLookupBase c.
- * Topic Stats
- *
- * @param topicName
- * @throws Exception
- */
- @Test(dataProvider = "topicName")
- public void testPulsarAdminForUriAndUrlEncoding(String topicName) throws Exception {
- final String ns1 = "prop-xyz/ns1";
- final String topic1 = "persistent://" + ns1 + "/" + topicName;
- final String urlEncodedTopic = Codec.encode(topicName);
- final String uriEncodedTopic = urlEncodedTopic.replaceAll("\\+", "%20");
- final int numOfPartitions = 4;
- admin.topics().createPartitionedTopic(topic1, numOfPartitions);
- // Create a consumer to get stats on this topic
- pulsarClient.newConsumer().topic(topic1).subscriptionName("my-subscriber-name").subscribe();
-
- TopicsImpl persistent = (TopicsImpl) admin.topics();
- Field field = TopicsImpl.class.getDeclaredField("adminV2Topics");
- field.setAccessible(true);
- WebTarget persistentTopics = (WebTarget) field.get(persistent);
-
- // (1) Get PartitionedMetadata : with Url and Uri encoding
- final CompletableFuture<PartitionedTopicMetadata> urlEncodedPartitionedMetadata = new CompletableFuture<>();
- // (a) Url encoding
- persistent.asyncGetRequest(
- persistentTopics.path("persistent").path(ns1).path(urlEncodedTopic).path("partitions"),
- new InvocationCallback<PartitionedTopicMetadata>() {
- @Override
- public void completed(PartitionedTopicMetadata response) {
- urlEncodedPartitionedMetadata.complete(response);
- }
-
- @Override
- public void failed(Throwable e) {
- urlEncodedPartitionedMetadata.completeExceptionally(e);
- }
- });
- final CompletableFuture<PartitionedTopicMetadata> uriEncodedPartitionedMetadata = new CompletableFuture<>();
- // (b) Uri encoding
- persistent.asyncGetRequest(
- persistentTopics.path("persistent").path(ns1).path(uriEncodedTopic).path("partitions"),
- new InvocationCallback<PartitionedTopicMetadata>() {
- @Override
- public void completed(PartitionedTopicMetadata response) {
- uriEncodedPartitionedMetadata.complete(response);
- }
-
- @Override
- public void failed(Throwable e) {
- uriEncodedPartitionedMetadata.completeExceptionally(e);
- }
- });
- assertEquals(urlEncodedPartitionedMetadata.get().partitions, numOfPartitions);
- assertEquals(urlEncodedPartitionedMetadata.get().partitions, (uriEncodedPartitionedMetadata.get().partitions));
-
- // (2) Get Topic Lookup
- LookupImpl lookup = (LookupImpl) admin.lookups();
- Field field2 = LookupImpl.class.getDeclaredField("v2lookup");
- field2.setAccessible(true);
- WebTarget target2 = (WebTarget) field2.get(lookup);
- // (a) Url encoding
- LookupData urlEncodedLookupData = lookup
- .request(target2.path("/topic/persistent").path(ns1 + "/" + urlEncodedTopic))
- .get(LookupData.class);
- // (b) Uri encoding
- LookupData uriEncodedLookupData = lookup
- .request(target2.path("/topic/persistent").path(ns1 + "/" + uriEncodedTopic))
- .get(LookupData.class);
- Assert.assertNotNull(urlEncodedLookupData.getBrokerUrl());
- assertEquals(urlEncodedLookupData.getBrokerUrl(), uriEncodedLookupData.getBrokerUrl());
-
- // (3) Get Topic Stats
- final CompletableFuture<TopicStats> urlStats = new CompletableFuture<>();
- // (a) Url encoding
- persistent.asyncGetRequest(persistentTopics.path("persistent").path(ns1).path(urlEncodedTopic + "-partition-1").path("stats"),
- new InvocationCallback<TopicStats>() {
- @Override
- public void completed(TopicStats response) {
- urlStats.complete(response);
- }
-
- @Override
- public void failed(Throwable e) {
- urlStats.completeExceptionally(e);
- }
- });
- // (b) Uri encoding
- final CompletableFuture<TopicStats> uriStats = new CompletableFuture<>();
- persistent.asyncGetRequest(
- persistentTopics.path("persistent").path(ns1).path(uriEncodedTopic + "-partition-1").path("stats"),
- new InvocationCallback<TopicStats>() {
- @Override
- public void completed(TopicStats response) {
- uriStats.complete(response);
- }
-
- @Override
- public void failed(Throwable e) {
- uriStats.completeExceptionally(e);
- }
- });
- assertEquals(urlStats.get().subscriptions.size(), 1);
- assertEquals(uriStats.get().subscriptions.size(), 1);
- }
-
- static class MockedPulsarService extends MockedPulsarServiceBaseTest {
-
- private ServiceConfiguration conf;
-
- public MockedPulsarService(ServiceConfiguration conf) {
- super();
- this.conf = conf;
- }
-
- @Override
- protected void setup() throws Exception {
- super.conf.setLoadManagerClassName(conf.getLoadManagerClassName());
- super.internalSetup();
- }
-
- @Override
- protected void cleanup() throws Exception {
- super.internalCleanup();
- }
-
- public PulsarService getPulsar() {
- return pulsar;
- }
-
- public PulsarAdmin getAdmin() {
- return admin;
- }
- }
-
- @Test
- public void testTopicBundleRangeLookup() throws PulsarAdminException, PulsarServerException, Exception {
- admin.clusters().createCluster("usw", new ClusterData());
- TenantInfo tenantInfo = new TenantInfo(Sets.newHashSet("role1", "role2"),
- Sets.newHashSet("test", "usw"));
- admin.tenants().updateTenant("prop-xyz", tenantInfo);
- admin.namespaces().createNamespace("prop-xyz/getBundleNs", 100);
- assertEquals(admin.namespaces().getPolicies("prop-xyz/getBundleNs").bundles.numBundles, 100);
-
- // (1) create a topic
- final String topicName = "persistent://prop-xyz/getBundleNs/topic1";
- String bundleRange = admin.lookups().getBundleRange(topicName);
- assertEquals(bundleRange, pulsar.getNamespaceService().getBundle(TopicName.get(topicName)).getBundleRange());
- }
-
- @Test
- public void testTriggerCompaction() throws Exception {
- String topicName = "persistent://prop-xyz/ns1/topic1";
-
- // create a topic by creating a producer
- pulsarClient.newProducer(Schema.BYTES).topic(topicName).create().close();
- assertNotNull(pulsar.getBrokerService().getTopicReference(topicName));
-
- // mock actual compaction, we don't need to really run it
- CompletableFuture<Long> promise = new CompletableFuture<Long>();
- Compactor compactor = pulsar.getCompactor();
- doReturn(promise).when(compactor).compact(topicName);
- admin.topics().triggerCompaction(topicName);
-
- // verify compact called once
- verify(compactor).compact(topicName);
- try {
- admin.topics().triggerCompaction(topicName);
-
- fail("Shouldn't be able to run while already running");
- } catch (ConflictException e) {
- // expected
- }
- // compact shouldn't have been called again
- verify(compactor).compact(topicName);
-
- // complete first compaction, and trigger again
- promise.complete(1L);
- admin.topics().triggerCompaction(topicName);
-
- // verify compact was called again
- verify(compactor, times(2)).compact(topicName);
- }
-
- @Test
- public void testCompactionStatus() throws Exception {
- String topicName = "persistent://prop-xyz/ns1/topic1";
-
- // create a topic by creating a producer
- pulsarClient.newProducer(Schema.BYTES).topic(topicName).create().close();
- assertNotNull(pulsar.getBrokerService().getTopicReference(topicName));
-
- assertEquals(admin.topics().compactionStatus(topicName).status,
- LongRunningProcessStatus.Status.NOT_RUN);
-
- // mock actual compaction, we don't need to really run it
- CompletableFuture<Long> promise = new CompletableFuture<Long>();
- Compactor compactor = pulsar.getCompactor();
- doReturn(promise).when(compactor).compact(topicName);
- admin.topics().triggerCompaction(topicName);
-
- assertEquals(admin.topics().compactionStatus(topicName).status,
- LongRunningProcessStatus.Status.RUNNING);
-
- promise.complete(1L);
-
- assertEquals(admin.topics().compactionStatus(topicName).status,
- LongRunningProcessStatus.Status.SUCCESS);
-
- CompletableFuture<Long> errorPromise = new CompletableFuture<Long>();
- doReturn(errorPromise).when(compactor).compact(topicName);
- admin.topics().triggerCompaction(topicName);
- errorPromise.completeExceptionally(new Exception("Failed at something"));
-
- assertEquals(admin.topics().compactionStatus(topicName).status,
- LongRunningProcessStatus.Status.ERROR);
- assertTrue(admin.topics().compactionStatus(topicName).lastError.contains("Failed at something"));
- }
-}
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest2.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest2.java
deleted file mode 100644
index 8df934853d..0000000000
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTest2.java
+++ /dev/null
@@ -1,881 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pulsar.broker.admin;
-
-import static org.apache.commons.lang3.StringUtils.isBlank;
-import static org.testng.Assert.assertEquals;
-import static org.testng.Assert.assertFalse;
-import static org.testng.Assert.assertNotNull;
-import static org.testng.Assert.assertTrue;
-import static org.testng.Assert.fail;
-
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-import com.google.common.collect.Sets;
-
-import java.net.URL;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.TimeUnit;
-
-import org.apache.bookkeeper.mledger.impl.ManagedCursorImpl;
-import org.apache.bookkeeper.mledger.impl.ManagedLedgerImpl;
-import org.apache.pulsar.broker.PulsarService;
-import org.apache.pulsar.broker.admin.AdminApiTest.MockedPulsarService;
-import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest;
-import org.apache.pulsar.broker.loadbalance.impl.ModularLoadManagerImpl;
-import org.apache.pulsar.broker.loadbalance.impl.SimpleLoadManagerImpl;
-import org.apache.pulsar.broker.service.Topic;
-import org.apache.pulsar.broker.service.persistent.PersistentTopic;
-import org.apache.pulsar.client.admin.PulsarAdmin;
-import org.apache.pulsar.client.admin.PulsarAdminException;
-import org.apache.pulsar.client.admin.PulsarAdminException.PreconditionFailedException;
-import org.apache.pulsar.client.api.Consumer;
-import org.apache.pulsar.client.api.Message;
-import org.apache.pulsar.client.api.MessageRoutingMode;
-import org.apache.pulsar.client.api.Producer;
-import org.apache.pulsar.client.api.PulsarClient;
-import org.apache.pulsar.client.api.SubscriptionType;
-import org.apache.pulsar.client.impl.MessageIdImpl;
-import org.apache.pulsar.common.naming.TopicDomain;
-import org.apache.pulsar.common.naming.TopicName;
-import org.apache.pulsar.common.policies.data.AutoFailoverPolicyData;
-import org.apache.pulsar.common.policies.data.AutoFailoverPolicyType;
-import org.apache.pulsar.common.policies.data.BrokerNamespaceIsolationData;
-import org.apache.pulsar.common.policies.data.ClusterData;
-import org.apache.pulsar.common.policies.data.ConsumerStats;
-import org.apache.pulsar.common.policies.data.FailureDomain;
-import org.apache.pulsar.common.policies.data.NamespaceIsolationData;
-import org.apache.pulsar.common.policies.data.NonPersistentTopicStats;
-import org.apache.pulsar.common.policies.data.PartitionedTopicStats;
-import org.apache.pulsar.common.policies.data.PersistencePolicies;
-import org.apache.pulsar.common.policies.data.PersistentTopicInternalStats;
-import org.apache.pulsar.common.policies.data.TopicStats;
-import org.apache.pulsar.common.policies.data.TenantInfo;
-import org.apache.pulsar.common.policies.data.RetentionPolicies;
-import org.apache.pulsar.common.policies.data.SubscriptionStats;
-import org.testng.Assert;
-import org.testng.annotations.AfterMethod;
-import org.testng.annotations.BeforeMethod;
-import org.testng.annotations.DataProvider;
-import org.testng.annotations.Test;
-
-public class AdminApiTest2 extends MockedPulsarServiceBaseTest {
-
- private MockedPulsarService mockPulsarSetup;
-
- @BeforeMethod
- @Override
- public void setup() throws Exception {
- conf.setLoadBalancerEnabled(true);
- super.internalSetup();
-
- // create otherbroker to test redirect on calls that need
- // namespace ownership
- mockPulsarSetup = new MockedPulsarService(this.conf);
- mockPulsarSetup.setup();
-
- // Setup namespaces
- admin.clusters().createCluster("test", new ClusterData("http://127.0.0.1" + ":" + BROKER_WEBSERVICE_PORT));
- TenantInfo tenantInfo = new TenantInfo(Sets.newHashSet("role1", "role2"), Sets.newHashSet("test"));
- admin.tenants().createTenant("prop-xyz", tenantInfo);
- admin.namespaces().createNamespace("prop-xyz/ns1", Sets.newHashSet("test"));
- }
-
- @AfterMethod
- @Override
- public void cleanup() throws Exception {
- super.internalCleanup();
- mockPulsarSetup.cleanup();
- }
-
- @DataProvider(name = "topicType")
- public Object[][] topicTypeProvider() {
- return new Object[][] { { TopicDomain.persistent.value() }, { TopicDomain.non_persistent.value() } };
- }
-
- @DataProvider(name = "namespaceNames")
- public Object[][] namespaceNameProvider() {
- return new Object[][] { { "ns1" }, { "global" } };
- }
-
- /**
- * <pre>
- * It verifies increasing partitions for partitioned-topic.
- * 1. create a partitioned-topic
- * 2. update partitions with larger number of partitions
- * 3. verify: getPartitionedMetadata and check number of partitions
- * 4. verify: this api creates existing subscription to new partitioned-topics
- * so, message will not be lost in new partitions
- * a. start producer and produce messages
- * b. check existing subscription for new topics and it should have backlog msgs
- *
- * </pre>
- *
- * @param topicName
- * @throws Exception
- */
- @Test
- public void testIncrementPartitionsOfTopic() throws Exception {
- final String topicName = "increment-partitionedTopic";
- final String subName1 = topicName + "-my-sub-1";
- final String subName2 = topicName + "-my-sub-2";
- final int startPartitions = 4;
- final int newPartitions = 8;
- final String partitionedTopicName = "persistent://prop-xyz/ns1/" + topicName;
-
- URL pulsarUrl = new URL("http://127.0.0.1" + ":" + BROKER_WEBSERVICE_PORT);
-
- admin.topics().createPartitionedTopic(partitionedTopicName, startPartitions);
- // validate partition topic is created
- assertEquals(admin.topics().getPartitionedTopicMetadata(partitionedTopicName).partitions,
- startPartitions);
-
- // create consumer and subscriptions : check subscriptions
- PulsarClient client = PulsarClient.builder().serviceUrl(pulsarUrl.toString()).build();
- Consumer<byte[]> consumer1 = client.newConsumer().topic(partitionedTopicName).subscriptionName(subName1)
- .subscriptionType(SubscriptionType.Shared).subscribe();
- assertEquals(admin.topics().getSubscriptions(partitionedTopicName), Lists.newArrayList(subName1));
- Consumer<byte[]> consumer2 = client.newConsumer().topic(partitionedTopicName).subscriptionName(subName2)
- .subscriptionType(SubscriptionType.Shared).subscribe();
- assertEquals(Sets.newHashSet(admin.topics().getSubscriptions(partitionedTopicName)),
- Sets.newHashSet(subName1, subName2));
-
- // (1) update partitions
- admin.topics().updatePartitionedTopic(partitionedTopicName, newPartitions);
- // invalidate global-cache to make sure that mock-zk-cache reds fresh data
- pulsar.getGlobalZkCache().invalidateAll();
- // verify new partitions have been created
- assertEquals(admin.topics().getPartitionedTopicMetadata(partitionedTopicName).partitions,
- newPartitions);
- // (2) No Msg loss: verify new partitions have the same existing subscription names
- final String newPartitionTopicName = TopicName.get(partitionedTopicName).getPartition(startPartitions + 1)
- .toString();
-
- // (3) produce messages to all partitions including newly created partitions (RoundRobin)
- Producer<byte[]> producer = client.newProducer()
- .topic(partitionedTopicName)
- .enableBatching(false)
- .messageRoutingMode(MessageRoutingMode.RoundRobinPartition)
- .create();
- final int totalMessages = newPartitions * 2;
- for (int i = 0; i < totalMessages; i++) {
- String message = "message-" + i;
- producer.send(message.getBytes());
- }
-
- // (4) verify existing subscription has not lost any message: create new consumer with sub-2: it will load all
- // newly created partition topics
- consumer2.close();
- consumer2 = client.newConsumer().topic(partitionedTopicName).subscriptionName(subName2)
- .subscriptionType(SubscriptionType.Shared).subscribe();
- // sometime: mockZk fails to refresh ml-cache: so, invalidate the cache to get fresh data
- pulsar.getLocalZkCacheService().managedLedgerListCache().clearTree();
- assertEquals(Sets.newHashSet(admin.topics().getSubscriptions(newPartitionTopicName)),
- Sets.newHashSet(subName1, subName2));
-
- assertEquals(Sets.newHashSet(admin.topics().getList("prop-xyz/ns1")).size(), newPartitions);
-
- // test cumulative stats for partitioned topic
- PartitionedTopicStats topicStats = admin.topics().getPartitionedStats(partitionedTopicName, false);
- assertEquals(topicStats.subscriptions.keySet(), Sets.newTreeSet(Lists.newArrayList(subName1, subName2)));
- assertEquals(topicStats.subscriptions.get(subName2).consumers.size(), 1);
- assertEquals(topicStats.subscriptions.get(subName2).msgBacklog, totalMessages);
- assertEquals(topicStats.publishers.size(), 1);
- assertEquals(topicStats.partitions, Maps.newHashMap());
-
- // (5) verify: each partition should have backlog
- topicStats = admin.topics().getPartitionedStats(partitionedTopicName, true);
- assertEquals(topicStats.metadata.partitions, newPartitions);
- Set<String> partitionSet = Sets.newHashSet();
- for (int i = 0; i < newPartitions; i++) {
- partitionSet.add(partitionedTopicName + "-partition-" + i);
- }
- assertEquals(topicStats.partitions.keySet(), partitionSet);
- for (int i = 0; i < newPartitions; i++) {
- TopicStats partitionStats = topicStats.partitions
- .get(TopicName.get(partitionedTopicName).getPartition(i).toString());
- assertEquals(partitionStats.publishers.size(), 1);
- assertEquals(partitionStats.subscriptions.get(subName2).consumers.size(), 1);
- assertEquals(partitionStats.subscriptions.get(subName2).msgBacklog, 2, 1);
- }
-
- producer.close();
- consumer1.close();
- consumer2.close();
- consumer2.close();
- }
-
- /**
- * verifies admin api command for non-persistent topic. It verifies: partitioned-topic, stats
- *
- * @throws Exception
- */
- @Test
- public void nonPersistentTopics() throws Exception {
- final String topicName = "nonPersistentTopic";
-
- final String persistentTopicName = "non-persistent://prop-xyz/ns1/" + topicName;
- // Force to create a topic
- publishMessagesOnTopic("non-persistent://prop-xyz/ns1/" + topicName, 0, 0);
-
- // create consumer and subscription
- URL pulsarUrl = new URL("http://127.0.0.1" + ":" + BROKER_WEBSERVICE_PORT);
- PulsarClient client = PulsarClient.builder().serviceUrl(pulsarUrl.toString()).statsInterval(0, TimeUnit.SECONDS)
- .build();
- Consumer<byte[]> consumer = client.newConsumer().topic(persistentTopicName).subscriptionName("my-sub")
- .subscribe();
-
- publishMessagesOnTopic("non-persistent://prop-xyz/ns1/" + topicName, 10, 0);
-
- TopicStats topicStats = admin.topics().getStats(persistentTopicName);
- assertEquals(topicStats.subscriptions.keySet(), Sets.newTreeSet(Lists.newArrayList("my-sub")));
- assertEquals(topicStats.subscriptions.get("my-sub").consumers.size(), 1);
- assertEquals(topicStats.publishers.size(), 0);
-
- PersistentTopicInternalStats internalStats = admin.topics().getInternalStats(persistentTopicName);
- assertEquals(internalStats.cursors.keySet(), Sets.newTreeSet(Lists.newArrayList("my-sub")));
-
- consumer.close();
- client.close();
-
- topicStats = admin.topics().getStats(persistentTopicName);
- assertTrue(topicStats.subscriptions.keySet().contains("my-sub"));
- assertEquals(topicStats.publishers.size(), 0);
-
- // test partitioned-topic
- final String partitionedTopicName = "non-persistent://prop-xyz/ns1/paritioned";
- assertEquals(admin.topics().getPartitionedTopicMetadata(partitionedTopicName).partitions, 0);
- admin.topics().createPartitionedTopic(partitionedTopicName, 5);
- assertEquals(admin.topics().getPartitionedTopicMetadata(partitionedTopicName).partitions, 5);
- }
-
- private void publishMessagesOnTopic(String topicName, int messages, int startIdx) throws Exception {
- Producer<byte[]> producer = pulsarClient.newProducer()
- .topic(topicName)
- .enableBatching(false)
- .messageRoutingMode(MessageRoutingMode.SinglePartition)
- .create();
-
- for (int i = startIdx; i < (messages + startIdx); i++) {
- String message = "message-" + i;
- producer.send(message.getBytes());
- }
-
- producer.close();
- }
-
- /**
- * verifies validation on persistent-policies
- *
- * @throws Exception
- */
- @Test
- public void testSetPersistencepolicies() throws Exception {
-
- final String namespace = "prop-xyz/ns2";
- admin.namespaces().createNamespace(namespace, Sets.newHashSet("test"));
-
- assertEquals(admin.namespaces().getPersistence(namespace), new PersistencePolicies(1, 1, 1, 0.0));
- admin.namespaces().setPersistence(namespace, new PersistencePolicies(3, 3, 3, 10.0));
- assertEquals(admin.namespaces().getPersistence(namespace), new PersistencePolicies(3, 3, 3, 10.0));
-
- try {
- admin.namespaces().setPersistence(namespace, new PersistencePolicies(3, 4, 3, 10.0));
- fail("should have failed");
- } catch (PulsarAdminException e) {
- assertEquals(e.getStatusCode(), 412);
- }
- try {
- admin.namespaces().setPersistence(namespace, new PersistencePolicies(3, 3, 4, 10.0));
- fail("should have failed");
- } catch (PulsarAdminException e) {
- assertEquals(e.getStatusCode(), 412);
- }
- try {
- admin.namespaces().setPersistence(namespace, new PersistencePolicies(6, 3, 1, 10.0));
- fail("should have failed");
- } catch (PulsarAdminException e) {
- assertEquals(e.getStatusCode(), 412);
- }
-
- // make sure policies has not been changed
- assertEquals(admin.namespaces().getPersistence(namespace), new PersistencePolicies(3, 3, 3, 10.0));
- }
-
- /**
- * validates update of persistent-policies reflects on managed-ledger and managed-cursor
- *
- * @throws Exception
- */
- @Test
- public void testUpdatePersistencePolicyUpdateManagedCursor() throws Exception {
-
- final String namespace = "prop-xyz/ns2";
- final String topicName = "persistent://" + namespace + "/topic1";
- admin.namespaces().createNamespace(namespace, Sets.newHashSet("test"));
-
- admin.namespaces().setPersistence(namespace, new PersistencePolicies(3, 3, 3, 50.0));
- assertEquals(admin.namespaces().getPersistence(namespace), new PersistencePolicies(3, 3, 3, 50.0));
-
- Producer<byte[]> producer = pulsarClient.newProducer()
- .topic(topicName)
- .enableBatching(false)
- .messageRoutingMode(MessageRoutingMode.SinglePartition)
- .create();
- Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("my-sub").subscribe();
-
- PersistentTopic topic = (PersistentTopic) pulsar.getBrokerService().getOrCreateTopic(topicName).get();
- ManagedLedgerImpl managedLedger = (ManagedLedgerImpl) topic.getManagedLedger();
- ManagedCursorImpl cursor = (ManagedCursorImpl) managedLedger.getCursors().iterator().next();
-
- final double newThrottleRate = 100;
- final int newEnsembleSize = 5;
- admin.namespaces().setPersistence(namespace, new PersistencePolicies(newEnsembleSize, 3, 3, newThrottleRate));
-
- retryStrategically((test) -> managedLedger.getConfig().getEnsembleSize() == newEnsembleSize
- && cursor.getThrottleMarkDelete() != newThrottleRate, 5, 200);
-
- // (1) verify cursor.markDelete has been updated
- assertEquals(cursor.getThrottleMarkDelete(), newThrottleRate);
-
- // (2) verify new ledger creation takes new config
-
- producer.close();
- consumer.close();
-
- }
-
- /**
- * Verify unloading topic
- *
- * @throws Exception
- */
- @Test(dataProvider = "topicType")
- public void testUnloadTopic(final String topicType) throws Exception {
-
- final String namespace = "prop-xyz/ns2";
- final String topicName = topicType + "://" + namespace + "/topic1";
- admin.namespaces().createNamespace(namespace, Sets.newHashSet("test"));
-
- // create a topic by creating a producer
- Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName).create();
- producer.close();
-
- Topic topic = pulsar.getBrokerService().getTopicIfExists(topicName).join().get();
- final boolean isPersistentTopic = topic instanceof PersistentTopic;
-
- // (1) unload the topic
- unloadTopic(topicName, isPersistentTopic);
-
- // topic must be removed from map
- assertFalse(pulsar.getBrokerService().getTopicReference(topicName).isPresent());
-
- // recreation of producer will load the topic again
- pulsarClient.newProducer().topic(topicName).create();
- topic = pulsar.getBrokerService().getTopicReference(topicName).get();
- assertNotNull(topic);
- // unload the topic
- unloadTopic(topicName, isPersistentTopic);
- // producer will retry and recreate the topic
- for (int i = 0; i < 5; i++) {
- if (!pulsar.getBrokerService().getTopicReference(topicName).isPresent() || i != 4) {
- Thread.sleep(200);
- }
- }
- // topic should be loaded by this time
- topic = pulsar.getBrokerService().getTopicReference(topicName).get();
- assertNotNull(topic);
- }
-
- private void unloadTopic(String topicName, boolean isPersistentTopic) throws Exception {
- admin.topics().unload(topicName);
- }
-
- /**
- * Verifies reset-cursor at specific position using admin-api.
- *
- * <pre>
- * 1. Publish 50 messages
- * 2. Consume 20 messages
- * 3. reset cursor position on 10th message
- * 4. consume 40 messages from reset position
- * </pre>
- *
- * @param namespaceName
- * @throws Exception
- */
- @Test(dataProvider = "namespaceNames", timeOut = 10000)
- public void testResetCursorOnPosition(String namespaceName) throws Exception {
- final String topicName = "persistent://prop-xyz/use/" + namespaceName + "/resetPosition";
- final int totalProducedMessages = 50;
-
- // set retention
- admin.namespaces().setRetention("prop-xyz/ns1", new RetentionPolicies(10, 10));
-
- // create consumer and subscription
- Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("my-sub")
- .subscriptionType(SubscriptionType.Shared).subscribe();
-
- assertEquals(admin.topics().getSubscriptions(topicName), Lists.newArrayList("my-sub"));
-
- publishMessagesOnPersistentTopic(topicName, totalProducedMessages, 0);
-
- List<Message<byte[]>> messages = admin.topics().peekMessages(topicName, "my-sub", 10);
- assertEquals(messages.size(), 10);
-
- Message<byte[]> message = null;
- MessageIdImpl resetMessageId = null;
- int resetPositionId = 10;
- for (int i = 0; i < 20; i++) {
- message = consumer.receive(1, TimeUnit.SECONDS);
- consumer.acknowledge(message);
- if (i == resetPositionId) {
- resetMessageId = (MessageIdImpl) message.getMessageId();
- }
- }
-
- // close consumer which will clean up intenral-receive-queue
- consumer.close();
-
- // messages should still be available due to retention
- MessageIdImpl messageId = new MessageIdImpl(resetMessageId.getLedgerId(), resetMessageId.getEntryId(), -1);
- // reset position at resetMessageId
- admin.topics().resetCursor(topicName, "my-sub", messageId);
-
- consumer = pulsarClient.newConsumer().topic(topicName).subscriptionName("my-sub")
- .subscriptionType(SubscriptionType.Shared).subscribe();
- MessageIdImpl msgId2 = (MessageIdImpl) consumer.receive(1, TimeUnit.SECONDS).getMessageId();
- assertEquals(resetMessageId, msgId2);
-
- int receivedAfterReset = 1; // start with 1 because we have already received 1 msg
-
- for (int i = 0; i < totalProducedMessages; i++) {
- message = consumer.receive(500, TimeUnit.MILLISECONDS);
- if (message == null) {
- break;
- }
- consumer.acknowledge(message);
- ++receivedAfterReset;
- }
- assertEquals(receivedAfterReset, totalProducedMessages - resetPositionId);
-
- // invalid topic name
- try {
- admin.topics().resetCursor(topicName + "invalid", "my-sub", messageId);
- fail("It should have failed due to invalid topic name");
- } catch (PulsarAdminException.NotFoundException e) {
- // Ok
- }
-
- // invalid cursor name
- try {
- admin.topics().resetCursor(topicName, "invalid-sub", messageId);
- fail("It should have failed due to invalid subscription name");
- } catch (PulsarAdminException.NotFoundException e) {
- // Ok
- }
-
- // invalid position
- try {
- messageId = new MessageIdImpl(0, 0, -1);
- admin.topics().resetCursor(topicName, "my-sub", messageId);
- fail("It should have failed due to invalid subscription name");
- } catch (PulsarAdminException.PreconditionFailedException e) {
- // Ok
- }
-
- consumer.close();
- }
-
- private void publishMessagesOnPersistentTopic(String topicName, int messages, int startIdx) throws Exception {
- Producer<byte[]> producer = pulsarClient.newProducer()
- .topic(topicName)
- .enableBatching(false)
- .messageRoutingMode(MessageRoutingMode.SinglePartition)
- .create();
-
- for (int i = startIdx; i < (messages + startIdx); i++) {
- String message = "message-" + i;
- producer.send(message.getBytes());
- }
-
- producer.close();
- }
-
- /**
- * It verifies that pulsar with different load-manager generates different load-report and returned by admin-api
- *
- * @throws Exception
- */
- @Test
- public void testLoadReportApi() throws Exception {
-
- this.conf.setLoadManagerClassName(SimpleLoadManagerImpl.class.getName());
- MockedPulsarService mockPulsarSetup1 = new MockedPulsarService(this.conf);
- mockPulsarSetup1.setup();
- PulsarService simpleLoadManager = mockPulsarSetup1.getPulsar();
- PulsarAdmin simpleLoadManagerAdmin = mockPulsarSetup1.getAdmin();
- assertNotNull(simpleLoadManagerAdmin.brokerStats().getLoadReport());
-
- this.conf.setLoadManagerClassName(ModularLoadManagerImpl.class.getName());
- MockedPulsarService mockPulsarSetup2 = new MockedPulsarService(this.conf);
- mockPulsarSetup2.setup();
- PulsarService modularLoadManager = mockPulsarSetup2.getPulsar();
- PulsarAdmin modularLoadManagerAdmin = mockPulsarSetup2.getAdmin();
- assertNotNull(modularLoadManagerAdmin.brokerStats().getLoadReport());
-
- simpleLoadManagerAdmin.close();
- simpleLoadManager.close();
- modularLoadManagerAdmin.close();
- modularLoadManager.close();
- mockPulsarSetup1.cleanup();
- mockPulsarSetup2.cleanup();
- }
-
- @Test
- public void testPeerCluster() throws Exception {
- admin.clusters().createCluster("us-west1",
- new ClusterData("http://broker.messaging.west1.example.com" + ":" + BROKER_WEBSERVICE_PORT));
- admin.clusters().createCluster("us-west2",
- new ClusterData("http://broker.messaging.west2.example.com" + ":" + BROKER_WEBSERVICE_PORT));
- admin.clusters().createCluster("us-east1",
- new ClusterData("http://broker.messaging.east1.example.com" + ":" + BROKER_WEBSERVICE_PORT));
- admin.clusters().createCluster("us-east2",
- new ClusterData("http://broker.messaging.east2.example.com" + ":" + BROKER_WEBSERVICE_PORT));
-
- admin.clusters().updatePeerClusterNames("us-west1", Sets.newLinkedHashSet(Lists.newArrayList("us-west2")));
- assertEquals(admin.clusters().getCluster("us-west1").getPeerClusterNames(), Lists.newArrayList("us-west2"));
- assertEquals(admin.clusters().getCluster("us-west2").getPeerClusterNames(), null);
- // update cluster with duplicate peer-clusters in the list
- admin.clusters().updatePeerClusterNames("us-west1", Sets.newLinkedHashSet(
- Lists.newArrayList("us-west2", "us-east1", "us-west2", "us-east1", "us-west2", "us-east1")));
- assertEquals(admin.clusters().getCluster("us-west1").getPeerClusterNames(),
- Lists.newArrayList("us-west2", "us-east1"));
- admin.clusters().updatePeerClusterNames("us-west1", null);
- assertEquals(admin.clusters().getCluster("us-west1").getPeerClusterNames(), null);
-
- // Check name validation
- try {
- admin.clusters().updatePeerClusterNames("us-west1",
- Sets.newLinkedHashSet(Lists.newArrayList("invalid-cluster")));
- fail("should have failed");
- } catch (PulsarAdminException e) {
- assertTrue(e instanceof PreconditionFailedException);
- }
-
- // Cluster itselft can't be part of peer-list
- try {
- admin.clusters().updatePeerClusterNames("us-west1", Sets.newLinkedHashSet(Lists.newArrayList("us-west1")));
- fail("should have failed");
- } catch (PulsarAdminException e) {
- assertTrue(e instanceof PreconditionFailedException);
- }
- }
-
- /**
- * It validates that peer-cluster can't coexist in replication-cluster list
- *
- * @throws Exception
- */
- @Test
- public void testReplicationPeerCluster() throws Exception {
- admin.clusters().createCluster("us-west1",
- new ClusterData("http://broker.messaging.west1.example.com" + ":" + BROKER_WEBSERVICE_PORT));
- admin.clusters().createCluster("us-west2",
- new ClusterData("http://broker.messaging.west2.example.com" + ":" + BROKER_WEBSERVICE_PORT));
- admin.clusters().createCluster("us-west3",
- new ClusterData("http://broker.messaging.west2.example.com" + ":" + BROKER_WEBSERVICE_PORT));
- admin.clusters().createCluster("us-west4",
- new ClusterData("http://broker.messaging.west2.example.com" + ":" + BROKER_WEBSERVICE_PORT));
- admin.clusters().createCluster("us-east1",
- new ClusterData("http://broker.messaging.east1.example.com" + ":" + BROKER_WEBSERVICE_PORT));
- admin.clusters().createCluster("us-east2",
- new ClusterData("http://broker.messaging.east2.example.com" + ":" + BROKER_WEBSERVICE_PORT));
- admin.clusters().createCluster("global", new ClusterData());
-
- final String property = "peer-prop";
- Set<String> allowedClusters = Sets.newHashSet("us-west1", "us-west2", "us-west3", "us-west4", "us-east1",
- "us-east2");
- TenantInfo propConfig = new TenantInfo(Sets.newHashSet("test"), allowedClusters);
- admin.tenants().createTenant(property, propConfig);
-
- final String namespace = property + "/global/conflictPeer";
- admin.namespaces().createNamespace(namespace);
-
- admin.clusters().updatePeerClusterNames("us-west1",
- Sets.newLinkedHashSet(Lists.newArrayList("us-west2", "us-west3")));
- assertEquals(admin.clusters().getCluster("us-west1").getPeerClusterNames(),
- Lists.newArrayList("us-west2", "us-west3"));
-
- // (1) no conflicting peer
- Set<String> clusterIds = Sets.newHashSet("us-east1", "us-east2");
- admin.namespaces().setNamespaceReplicationClusters(namespace, clusterIds);
-
- // (2) conflicting peer
- clusterIds = Sets.newHashSet("us-west2", "us-west3", "us-west1");
- try {
- admin.namespaces().setNamespaceReplicationClusters(namespace, clusterIds);
- fail("Peer-cluster can't coexist in replication cluster list");
- } catch (PulsarAdminException.ConflictException e) {
- // Ok
- }
-
- clusterIds = Sets.newHashSet("us-west2", "us-west3");
- // no peer coexist in replication clusters
- admin.namespaces().setNamespaceReplicationClusters(namespace, clusterIds);
-
- clusterIds = Sets.newHashSet("us-west1", "us-west4");
- // no peer coexist in replication clusters
- admin.namespaces().setNamespaceReplicationClusters(namespace, clusterIds);
- }
-
- @Test
- public void clusterFailureDomain() throws PulsarAdminException {
-
- final String cluster = pulsar.getConfiguration().getClusterName();
- // create
- FailureDomain domain = new FailureDomain();
- domain.setBrokers(Sets.newHashSet("b1", "b2", "b3"));
- admin.clusters().createFailureDomain(cluster, "domain-1", domain);
- admin.clusters().updateFailureDomain(cluster, "domain-1", domain);
-
- assertEquals(admin.clusters().getFailureDomain(cluster, "domain-1"), domain);
-
- Map<String, FailureDomain> domains = admin.clusters().getFailureDomains(cluster);
- assertEquals(domains.size(), 1);
- assertTrue(domains.containsKey("domain-1"));
-
- try {
- // try to create domain with already registered brokers
- admin.clusters().createFailureDomain(cluster, "domain-2", domain);
- fail("should have failed because of brokers are already registered");
- } catch (PulsarAdminException.ConflictException e) {
- // Ok
- }
-
- admin.clusters().deleteFailureDomain(cluster, "domain-1");
- assertTrue(admin.clusters().getFailureDomains(cluster).isEmpty());
-
- admin.clusters().createFailureDomain(cluster, "domain-2", domain);
- domains = admin.clusters().getFailureDomains(cluster);
- assertEquals(domains.size(), 1);
- assertTrue(domains.containsKey("domain-2"));
- }
-
- @Test
- public void namespaceAntiAffinity() throws PulsarAdminException {
- final String namespace = "prop-xyz/ns1";
- final String antiAffinityGroup = "group";
- assertTrue(isBlank(admin.namespaces().getNamespaceAntiAffinityGroup(namespace)));
- admin.namespaces().setNamespaceAntiAffinityGroup(namespace, antiAffinityGroup);
- assertEquals(admin.namespaces().getNamespaceAntiAffinityGroup(namespace), antiAffinityGroup);
- admin.namespaces().deleteNamespaceAntiAffinityGroup(namespace);
- assertTrue(isBlank(admin.namespaces().getNamespaceAntiAffinityGroup(namespace)));
-
- final String ns1 = "prop-xyz/antiAG1";
- final String ns2 = "prop-xyz/antiAG2";
- final String ns3 = "prop-xyz/antiAG3";
- admin.namespaces().createNamespace(ns1, Sets.newHashSet("test"));
- admin.namespaces().createNamespace(ns2, Sets.newHashSet("test"));
- admin.namespaces().createNamespace(ns3, Sets.newHashSet("test"));
- admin.namespaces().setNamespaceAntiAffinityGroup(ns1, antiAffinityGroup);
- admin.namespaces().setNamespaceAntiAffinityGroup(ns2, antiAffinityGroup);
- admin.namespaces().setNamespaceAntiAffinityGroup(ns3, antiAffinityGroup);
-
- Set<String> namespaces = new HashSet<>(
- admin.namespaces().getAntiAffinityNamespaces("prop-xyz", "test", antiAffinityGroup));
- assertEquals(namespaces.size(), 3);
- assertTrue(namespaces.contains(ns1));
- assertTrue(namespaces.contains(ns2));
- assertTrue(namespaces.contains(ns3));
-
- List<String> namespaces2 = admin.namespaces().getAntiAffinityNamespaces("prop-xyz", "test", "invalid-group");
- assertEquals(namespaces2.size(), 0);
- }
-
- @Test
- public void testNonPersistentTopics() throws Exception {
- final String namespace = "prop-xyz/ns2";
- final String topicName = "non-persistent://" + namespace + "/topic";
- admin.namespaces().createNamespace(namespace, 20);
- admin.namespaces().setNamespaceReplicationClusters(namespace, Sets.newHashSet("test"));
- int totalTopics = 100;
-
- Set<String> topicNames = Sets.newHashSet();
- for (int i = 0; i < totalTopics; i++) {
- topicNames.add(topicName + i);
- Producer<byte[]> producer = pulsarClient.newProducer().topic(topicName + i).create();
- producer.close();
- }
-
- for (int i = 0; i < totalTopics; i++) {
- Topic topic = pulsar.getBrokerService().getTopicReference(topicName + i).get();
- assertNotNull(topic);
- }
-
- Set<String> topicsInNs = Sets.newHashSet(admin.topics().getList(namespace));
- assertEquals(topicsInNs.size(), totalTopics);
- topicsInNs.removeAll(topicNames);
- assertEquals(topicsInNs.size(), 0);
- }
-
- @Test
- public void testPublishConsumerStats() throws Exception {
- final String topicName = "statTopic";
- final String subscriberName = topicName + "-my-sub-1";
- final String topic = "persistent://prop-xyz/ns1/" + topicName;
- final String producerName = "myProducer";
-
- URL pulsarUrl = new URL("http://127.0.0.1" + ":" + BROKER_WEBSERVICE_PORT);
-
- PulsarClient client = PulsarClient.builder().serviceUrl(pulsarUrl.toString()).build();
- Consumer<byte[]> consumer = client.newConsumer().topic(topic).subscriptionName(subscriberName)
- .subscriptionType(SubscriptionType.Shared).subscribe();
- Producer<byte[]> producer = client.newProducer()
- .topic(topic)
- .enableBatching(false)
- .messageRoutingMode(MessageRoutingMode.SinglePartition)
- .producerName(producerName)
- .create();
-
- retryStrategically((test) -> {
- TopicStats stats;
- try {
- stats = admin.topics().getStats(topic);
- return stats.publishers.size() > 0 && stats.subscriptions.get(subscriberName) != null
- && stats.subscriptions.get(subscriberName).consumers.size() > 0;
- } catch (PulsarAdminException e) {
- return false;
- }
- }, 5, 200);
-
- TopicStats topicStats = admin.topics().getStats(topic);
- assertEquals(topicStats.publishers.size(), 1);
- assertNotNull(topicStats.publishers.get(0).getAddress());
- assertNotNull(topicStats.publishers.get(0).getClientVersion());
- assertNotNull(topicStats.publishers.get(0).getConnectedSince());
- assertNotNull(topicStats.publishers.get(0).getProducerName());
- assertEquals(topicStats.publishers.get(0).getProducerName(), producerName);
-
- SubscriptionStats subscriber = topicStats.subscriptions.get(subscriberName);
- assertNotNull(subscriber);
- assertEquals(subscriber.consumers.size(), 1);
- ConsumerStats consumerStats = subscriber.consumers.get(0);
- assertNotNull(consumerStats.getAddress());
- assertNotNull(consumerStats.getClientVersion());
- assertNotNull(consumerStats.getConnectedSince());
-
- producer.close();
- consumer.close();
- }
-
- @Test
- public void testTenantNameWithUnderscore() throws Exception {
- TenantInfo tenantInfo = new TenantInfo(Sets.newHashSet("role1", "role2"), Sets.newHashSet("test"));
- admin.tenants().createTenant("prop_xyz", tenantInfo);
-
- admin.namespaces().createNamespace("prop_xyz/my-namespace", Sets.newHashSet("test"));
-
- String topic = "persistent://prop_xyz/use/my-namespace/my-topic";
-
- Producer<byte[]> producer = pulsarClient.newProducer().topic(topic)
- .create();
-
- TopicStats stats = admin.topics().getStats(topic);
- assertEquals(stats.publishers.size(), 1);
- producer.close();
- }
-
- @Test
- public void testTenantNameWithInvalidCharacters() throws Exception {
- TenantInfo tenantInfo = new TenantInfo(Sets.newHashSet("role1", "role2"), Sets.newHashSet("test"));
-
- // If we try to create property with invalid characters, it should fail immediately
- try {
- admin.tenants().createTenant("prop xyz", tenantInfo);
- fail("Should have failed");
- } catch (PulsarAdminException e) {
- // Expected
- }
-
- try {
- admin.tenants().createTenant("prop&xyz", tenantInfo);
- fail("Should have failed");
- } catch (PulsarAdminException e) {
- // Expected
- }
- }
-
- @Test
- public void brokerNamespaceIsolationPolicies() throws Exception {
-
- // create
- String policyName1 = "policy-1";
- String namespaceRegex = "other/use/other.*";
- String cluster = "use";
- String brokerName = pulsar.getAdvertisedAddress();
- String brokerAddress = brokerName + ":" + pulsar.getConfiguration().getWebServicePort();
- NamespaceIsolationData nsPolicyData1 = new NamespaceIsolationData();
- nsPolicyData1.namespaces = new ArrayList<String>();
- nsPolicyData1.namespaces.add(namespaceRegex);
- nsPolicyData1.primary = new ArrayList<String>();
- nsPolicyData1.primary.add(brokerName + ":[0-9]*");
- nsPolicyData1.secondary = new ArrayList<String>();
- nsPolicyData1.secondary.add(brokerName + ".*");
- nsPolicyData1.auto_failover_policy = new AutoFailoverPolicyData();
- nsPolicyData1.auto_failover_policy.policy_type = AutoFailoverPolicyType.min_available;
- nsPolicyData1.auto_failover_policy.parameters = new HashMap<String, String>();
- nsPolicyData1.auto_failover_policy.parameters.put("min_limit", "1");
- nsPolicyData1.auto_failover_policy.parameters.put("usage_threshold", "100");
- admin.clusters().createNamespaceIsolationPolicy(cluster, policyName1, nsPolicyData1);
-
- List<BrokerNamespaceIsolationData> brokerIsolationDataList = admin.clusters()
- .getBrokersWithNamespaceIsolationPolicy(cluster);
- assertEquals(brokerIsolationDataList.size(), 1);
- assertEquals(brokerIsolationDataList.get(0).brokerName, brokerAddress);
- assertEquals(brokerIsolationDataList.get(0).namespaceRegex.size(), 1);
- assertEquals(brokerIsolationDataList.get(0).namespaceRegex.get(0), namespaceRegex);
-
- BrokerNamespaceIsolationData brokerIsolationData = admin.clusters()
- .getBrokerWithNamespaceIsolationPolicy(cluster, brokerAddress);
- assertEquals(brokerIsolationData.brokerName, brokerAddress);
- assertEquals(brokerIsolationData.namespaceRegex.size(), 1);
- assertEquals(brokerIsolationData.namespaceRegex.get(0), namespaceRegex);
-
- try {
- admin.clusters().getBrokerWithNamespaceIsolationPolicy(cluster, "invalid-broker");
- Assert.fail("should have failed due to invalid broker address");
- } catch (PulsarAdminException.NotFoundException e) {// expected
- }
- }
-
- @Test
- public void clustersList() throws PulsarAdminException {
- final String cluster = pulsar.getConfiguration().getClusterName();
- admin.clusters().createCluster("global", new ClusterData("http://localhost:6650"));
-
- // Global cluster, if there, should be omitted from the results
- assertEquals(admin.clusters().getClusters(), Lists.newArrayList(cluster));
- }
-}
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTlsAuthTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTlsAuthTest.java
deleted file mode 100644
index 24a07a391f..0000000000
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminApiTlsAuthTest.java
+++ /dev/null
@@ -1,322 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pulsar.broker.admin;
-
-import com.google.common.collect.ImmutableSet;
-
-import java.util.List;
-import java.security.cert.X509Certificate;
-import javax.net.ssl.SSLContext;
-import javax.ws.rs.NotAuthorizedException;
-import javax.ws.rs.client.Client;
-import javax.ws.rs.client.ClientBuilder;
-import javax.ws.rs.client.WebTarget;
-import javax.ws.rs.core.GenericType;
-import javax.ws.rs.core.MediaType;
-
-import lombok.extern.slf4j.Slf4j;
-
-import org.apache.http.conn.ssl.NoopHostnameVerifier;
-
-import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest;
-import org.apache.pulsar.client.admin.PulsarAdmin;
-import org.apache.pulsar.client.admin.PulsarAdminException;
-import org.apache.pulsar.client.admin.internal.JacksonConfigurator;
-import org.apache.pulsar.common.policies.data.TenantInfo;
-import org.apache.pulsar.common.util.SecurityUtility;
-
-import org.glassfish.jersey.client.ClientConfig;
-import org.glassfish.jersey.client.ClientProperties;
-import org.glassfish.jersey.jackson.JacksonFeature;
-import org.glassfish.jersey.media.multipart.MultiPartFeature;
-
-import org.testng.Assert;
-import org.testng.annotations.AfterMethod;
-import org.testng.annotations.BeforeMethod;
-import org.testng.annotations.Test;
-
-@Slf4j
-public class AdminApiTlsAuthTest extends MockedPulsarServiceBaseTest {
- private static String getTLSFile(String name) {
- return String.format("./src/test/resources/authentication/tls-http/%s.pem", name);
- }
-
- @BeforeMethod
- @Override
- public void setup() throws Exception {
- conf.setLoadBalancerEnabled(true);
- conf.setTlsEnabled(true);
- conf.setTlsCertificateFilePath(getTLSFile("broker.cert"));
- conf.setTlsKeyFilePath(getTLSFile("broker.key-pk8"));
- conf.setTlsTrustCertsFilePath(getTLSFile("ca.cert"));
- conf.setAuthenticationEnabled(true);
- conf.setAuthenticationProviders(
- ImmutableSet.of("org.apache.pulsar.broker.authentication.AuthenticationProviderTls"));
- conf.setSuperUserRoles(ImmutableSet.of("admin", "superproxy"));
- conf.setProxyRoles(ImmutableSet.of("proxy", "superproxy"));
- conf.setAuthorizationEnabled(true);
-
- super.internalSetup();
- }
-
- @AfterMethod
- @Override
- public void cleanup() throws Exception {
- super.internalCleanup();
- }
-
- WebTarget buildWebClient(String user) throws Exception {
- ClientConfig httpConfig = new ClientConfig();
- httpConfig.property(ClientProperties.FOLLOW_REDIRECTS, true);
- httpConfig.property(ClientProperties.ASYNC_THREADPOOL_SIZE, 8);
- httpConfig.register(MultiPartFeature.class);
-
- ClientBuilder clientBuilder = ClientBuilder.newBuilder().withConfig(httpConfig)
- .register(JacksonConfigurator.class).register(JacksonFeature.class);
-
- X509Certificate trustCertificates[] = SecurityUtility.loadCertificatesFromPemFile(
- getTLSFile("ca.cert"));
- SSLContext sslCtx = SecurityUtility.createSslContext(
- false, trustCertificates,
- SecurityUtility.loadCertificatesFromPemFile(getTLSFile(user + ".cert")),
- SecurityUtility.loadPrivateKeyFromPemFile(getTLSFile(user + ".key-pk8")));
- clientBuilder.sslContext(sslCtx).hostnameVerifier(NoopHostnameVerifier.INSTANCE);
- Client client = clientBuilder.build();
-
- return client.target(brokerUrlTls.toString());
- }
-
- PulsarAdmin buildAdminClient(String user) throws Exception {
- return PulsarAdmin.builder()
- .allowTlsInsecureConnection(false)
- .enableTlsHostnameVerification(false)
- .serviceHttpUrl(brokerUrlTls.toString())
- .authentication("org.apache.pulsar.client.impl.auth.AuthenticationTls",
- String.format("tlsCertFile:%s,tlsKeyFile:%s",
- getTLSFile(user + ".cert"), getTLSFile(user + ".key-pk8")))
- .tlsTrustCertsFilePath(getTLSFile("ca.cert")).build();
- }
-
- @Test
- public void testSuperUserCanListTenants() throws Exception {
- try (PulsarAdmin admin = buildAdminClient("admin")) {
- admin.tenants().createTenant("tenant1",
- new TenantInfo(ImmutableSet.of("foobar"),
- ImmutableSet.of("test")));
- Assert.assertEquals(ImmutableSet.of("tenant1"), admin.tenants().getTenants());
- }
- }
-
- @Test
- public void testProxyRoleCantListTenants() throws Exception {
- try (PulsarAdmin admin = buildAdminClient("admin")) {
- admin.tenants().createTenant("tenant1",
- new TenantInfo(ImmutableSet.of("foobar"),
- ImmutableSet.of("test")));
- }
- try (PulsarAdmin admin = buildAdminClient("proxy")) {
- admin.tenants().getTenants();
- Assert.fail("Shouldn't be able to list tenants");
- } catch (PulsarAdminException.NotAuthorizedException e) {
- // expected
- }
- }
-
- @Test
- public void testProxyRoleCantListNamespacesEvenWithAccess() throws Exception {
- try (PulsarAdmin admin = buildAdminClient("admin")) {
- admin.tenants().createTenant("tenant1",
- new TenantInfo(ImmutableSet.of("proxy"),
- ImmutableSet.of("test")));
- admin.namespaces().createNamespace("tenant1/ns1");
- }
- try (PulsarAdmin admin = buildAdminClient("proxy")) {
- admin.namespaces().getNamespaces("tenant1");
- Assert.fail("Shouldn't be able to list namespaces");
- } catch (PulsarAdminException.NotAuthorizedException e) {
- // expected
- }
- }
-
- @Test
- public void testAuthorizedUserAsOriginalPrincipal() throws Exception {
- try (PulsarAdmin admin = buildAdminClient("admin")) {
- admin.tenants().createTenant("tenant1",
- new TenantInfo(ImmutableSet.of("proxy", "user1"),
- ImmutableSet.of("test")));
- admin.namespaces().createNamespace("tenant1/ns1");
- }
- WebTarget root = buildWebClient("proxy");
- Assert.assertEquals(ImmutableSet.of("tenant1/ns1"),
- root.path("/admin/v2/namespaces").path("tenant1")
- .request(MediaType.APPLICATION_JSON)
- .header("X-Original-Principal", "user1")
- .get(new GenericType<List<String>>() {}));
- }
-
- @Test
- public void testUnauthorizedUserAsOriginalPrincipal() throws Exception {
- try (PulsarAdmin admin = buildAdminClient("admin")) {
- admin.tenants().createTenant("tenant1",
- new TenantInfo(ImmutableSet.of("proxy", "user1"),
- ImmutableSet.of("test")));
- admin.namespaces().createNamespace("tenant1/ns1");
- }
- WebTarget root = buildWebClient("proxy");
- try {
- root.path("/admin/v2/namespaces").path("tenant1")
- .request(MediaType.APPLICATION_JSON)
- .header("X-Original-Principal", "user2")
- .get(new GenericType<List<String>>() {});
- Assert.fail("user2 should not be authorized");
- } catch (NotAuthorizedException e) {
- // expected
- }
- }
-
- @Test
- public void testAuthorizedUserAsOriginalPrincipalButProxyNotAuthorized() throws Exception {
- try (PulsarAdmin admin = buildAdminClient("admin")) {
- admin.tenants().createTenant("tenant1",
- new TenantInfo(ImmutableSet.of("user1"),
- ImmutableSet.of("test")));
- admin.namespaces().createNamespace("tenant1/ns1");
- }
- WebTarget root = buildWebClient("proxy");
- try {
- root.path("/admin/v2/namespaces").path("tenant1")
- .request(MediaType.APPLICATION_JSON)
- .header("X-Original-Principal", "user1")
- .get(new GenericType<List<String>>() {});
- Assert.fail("Shouldn't be able to list namespaces");
- } catch (NotAuthorizedException e) {
- // expected
- }
- }
-
- @Test
- public void testAuthorizedUserAsOriginalPrincipalProxyIsSuperUser() throws Exception {
- try (PulsarAdmin admin = buildAdminClient("admin")) {
- admin.tenants().createTenant("tenant1",
- new TenantInfo(ImmutableSet.of("user1"),
- ImmutableSet.of("test")));
- admin.namespaces().createNamespace("tenant1/ns1");
- }
- WebTarget root = buildWebClient("superproxy");
- Assert.assertEquals(ImmutableSet.of("tenant1/ns1"),
- root.path("/admin/v2/namespaces").path("tenant1")
- .request(MediaType.APPLICATION_JSON)
- .header("X-Original-Principal", "user1")
- .get(new GenericType<List<String>>() {}));
- }
-
- @Test
- public void testUnauthorizedUserAsOriginalPrincipalProxyIsSuperUser() throws Exception {
- try (PulsarAdmin admin = buildAdminClient("admin")) {
- admin.tenants().createTenant("tenant1",
- new TenantInfo(ImmutableSet.of("user1"),
- ImmutableSet.of("test")));
- admin.namespaces().createNamespace("tenant1/ns1");
- }
- WebTarget root = buildWebClient("superproxy");
- try {
- root.path("/admin/v2/namespaces").path("tenant1")
- .request(MediaType.APPLICATION_JSON)
- .header("X-Original-Principal", "user2")
- .get(new GenericType<List<String>>() {});
- Assert.fail("user2 should not be authorized");
- } catch (NotAuthorizedException e) {
- // expected
- }
- }
-
- @Test
- public void testProxyUserViaProxy() throws Exception {
- try (PulsarAdmin admin = buildAdminClient("admin")) {
- admin.tenants().createTenant("tenant1",
- new TenantInfo(ImmutableSet.of("proxy"),
- ImmutableSet.of("test")));
- admin.namespaces().createNamespace("tenant1/ns1");
- }
- WebTarget root = buildWebClient("superproxy");
- try {
- root.path("/admin/v2/namespaces").path("tenant1")
- .request(MediaType.APPLICATION_JSON)
- .header("X-Original-Principal", "proxy")
- .get(new GenericType<List<String>>() {});
- Assert.fail("proxy should not be authorized");
- } catch (NotAuthorizedException e) {
- // expected
- }
- }
-
- @Test
- public void testSuperProxyUserAndAdminCanListTenants() throws Exception {
- try (PulsarAdmin admin = buildAdminClient("admin")) {
- admin.tenants().createTenant("tenant1",
- new TenantInfo(ImmutableSet.of("user1"),
- ImmutableSet.of("test")));
- }
- WebTarget root = buildWebClient("superproxy");
- Assert.assertEquals(ImmutableSet.of("tenant1"),
- root.path("/admin/v2/tenants")
- .request(MediaType.APPLICATION_JSON)
- .header("X-Original-Principal", "admin")
- .get(new GenericType<List<String>>() {}));
- }
-
- @Test
- public void testSuperProxyUserAndNonAdminCannotListTenants() throws Exception {
- try (PulsarAdmin admin = buildAdminClient("admin")) {
- admin.tenants().createTenant("tenant1",
- new TenantInfo(ImmutableSet.of("proxy"),
- ImmutableSet.of("test")));
- }
- WebTarget root = buildWebClient("superproxy");
- try {
- root.path("/admin/v2/tenants")
- .request(MediaType.APPLICATION_JSON)
- .header("X-Original-Principal", "user1")
- .get(new GenericType<List<String>>() {});
- Assert.fail("user1 should not be authorized");
- } catch (NotAuthorizedException e) {
- // expected
- }
- }
-
- @Test
- public void testNonProxyCannotSetOriginalPrincipal() throws Exception {
- try (PulsarAdmin admin = buildAdminClient("admin")) {
- admin.tenants().createTenant("tenant1",
- new TenantInfo(ImmutableSet.of("user1"),
- ImmutableSet.of("test")));
- admin.namespaces().createNamespace("tenant1/ns1");
- }
- WebTarget root = buildWebClient("admin");
- try {
- root.path("/admin/v2/namespaces").path("tenant1")
- .request(MediaType.APPLICATION_JSON)
- .header("X-Original-Principal", "user1")
- .get(new GenericType<List<String>>() {});
- Assert.fail("admin shouldn't be able to act as proxy even if it is superuser");
- } catch (NotAuthorizedException e) {
- // expected
- }
- }
-}
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminResourceTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminResourceTest.java
deleted file mode 100644
index 7ad60c2b45..0000000000
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminResourceTest.java
+++ /dev/null
@@ -1,69 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pulsar.broker.admin;
-
-import static org.testng.Assert.assertEquals;
-import static org.testng.Assert.fail;
-
-import javax.ws.rs.core.Response.Status;
-import org.apache.pulsar.broker.web.RestException;
-import org.apache.pulsar.common.util.Codec;
-import org.testng.annotations.Test;
-
-/**
- * Unit test {@link AdminResource}.
- */
-public class AdminResourceTest {
-
- private static AdminResource mockResource() {
- return new AdminResource() {
-
- @Override
- protected String domain() {
- return "persistent";
- }
- };
- }
-
- @Test
- public void testValidatePartitionedTopicNameSuccess() {
- String tenant = "test-tenant";
- String namespace = "test-namespace";
- String topic = Codec.encode("test-topic");
-
- AdminResource resource = mockResource();
- resource.validatePartitionedTopicName(tenant, namespace, topic);
- }
-
- @Test
- public void testValidatePartitionedTopicNameInvalid() {
- String tenant = "test-tenant";
- String namespace = "test-namespace";
- String topic = Codec.encode("test-topic-partition-0");
-
- AdminResource resource = mockResource();
- try {
- resource.validatePartitionedTopicName(tenant, namespace, topic);
- fail("Should fail validation on invalid partitioned topic");
- } catch (RestException re) {
- assertEquals(Status.PRECONDITION_FAILED.getStatusCode(), re.getResponse().getStatus());
- }
- }
-
-}
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminTest.java
deleted file mode 100644
index 1f0058c74c..0000000000
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/AdminTest.java
+++ /dev/null
@@ -1,666 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pulsar.broker.admin;
-
-import static org.apache.pulsar.broker.cache.ConfigurationCacheService.POLICIES;
-import static org.mockito.Mockito.doNothing;
-import static org.mockito.Mockito.doReturn;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.never;
-import static org.mockito.Mockito.spy;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
-import static org.testng.Assert.assertEquals;
-import static org.testng.Assert.assertNotNull;
-import static org.testng.Assert.assertNotSame;
-import static org.testng.Assert.assertTrue;
-import static org.testng.Assert.fail;
-
-import com.google.common.collect.Lists;
-import com.google.common.collect.Sets;
-import java.lang.reflect.Field;
-import java.net.URI;
-import java.time.Clock;
-import java.time.Instant;
-import java.time.ZoneId;
-import java.util.ArrayList;
-import java.util.Collection;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-import java.util.concurrent.CountDownLatch;
-import javax.ws.rs.core.Response.Status;
-import javax.ws.rs.core.StreamingOutput;
-import javax.ws.rs.core.UriInfo;
-import org.apache.bookkeeper.conf.ClientConfiguration;
-import org.apache.bookkeeper.mledger.proto.PendingBookieOpsStats;
-import org.apache.bookkeeper.util.ZkUtils;
-import org.apache.pulsar.broker.admin.v1.BrokerStats;
-import org.apache.pulsar.broker.admin.v1.Brokers;
-import org.apache.pulsar.broker.admin.v1.Clusters;
-import org.apache.pulsar.broker.admin.v1.Namespaces;
-import org.apache.pulsar.broker.admin.v1.PersistentTopics;
-import org.apache.pulsar.broker.admin.v1.Properties;
-import org.apache.pulsar.broker.admin.v1.ResourceQuotas;
-import org.apache.pulsar.broker.admin.v2.SchemasResource;
-import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest;
-import org.apache.pulsar.broker.cache.ConfigurationCacheService;
-import org.apache.pulsar.broker.web.PulsarWebResource;
-import org.apache.pulsar.broker.web.RestException;
-import org.apache.pulsar.common.conf.InternalConfigurationData;
-import org.apache.pulsar.common.naming.NamespaceName;
-import org.apache.pulsar.common.policies.data.AuthAction;
-import org.apache.pulsar.common.policies.data.AutoFailoverPolicyData;
-import org.apache.pulsar.common.policies.data.AutoFailoverPolicyType;
-import org.apache.pulsar.common.policies.data.BundlesData;
-import org.apache.pulsar.common.policies.data.ClusterData;
-import org.apache.pulsar.common.policies.data.NamespaceIsolationData;
-import org.apache.pulsar.common.policies.data.Policies;
-import org.apache.pulsar.common.policies.data.TenantInfo;
-import org.apache.pulsar.common.policies.data.ResourceQuota;
-import org.apache.pulsar.common.stats.AllocatorStats;
-import org.apache.pulsar.common.stats.Metrics;
-import org.apache.pulsar.common.util.ObjectMapperFactory;
-import org.apache.pulsar.policies.data.loadbalancer.LocalBrokerData;
-import org.apache.zookeeper.CreateMode;
-import org.apache.zookeeper.KeeperException.Code;
-import org.apache.zookeeper.ZooDefs;
-import org.apache.zookeeper.ZooDefs.Ids;
-import org.testng.annotations.AfterMethod;
-import org.testng.annotations.BeforeMethod;
-import org.testng.annotations.Test;
-
-@Test
-public class AdminTest extends MockedPulsarServiceBaseTest {
- private final String configClusterName = "use";
- private ConfigurationCacheService configurationCache;
- private Clusters clusters;
- private Properties properties;
- private Namespaces namespaces;
- private PersistentTopics persistentTopics;
- private Brokers brokers;
- private ResourceQuotas resourceQuotas;
- private BrokerStats brokerStats;
- private SchemasResource schemasResource;
- private Field uriField;
- private Clock mockClock = Clock.fixed(
- Instant.ofEpochSecond(365248800),
- ZoneId.of("-05:00")
- );
-
- public AdminTest() {
- super();
- conf.setClusterName(configClusterName);
- }
-
- @Override
- @BeforeMethod
- public void setup() throws Exception {
- super.internalSetup();
-
- configurationCache = pulsar.getConfigurationCache();
-
- clusters = spy(new Clusters());
- clusters.setPulsar(pulsar);
- doReturn(mockZookKeeper).when(clusters).globalZk();
- doReturn(configurationCache.clustersCache()).when(clusters).clustersCache();
- doReturn(configurationCache.clustersListCache()).when(clusters).clustersListCache();
- doReturn(configurationCache.namespaceIsolationPoliciesCache()).when(clusters).namespaceIsolationPoliciesCache();
- doReturn("test").when(clusters).clientAppId();
- doNothing().when(clusters).validateSuperUserAccess();
-
- properties = spy(new Properties());
- properties.setServletContext(new MockServletContext());
- properties.setPulsar(pulsar);
- doReturn(mockZookKeeper).when(properties).globalZk();
- doReturn(configurationCache.propertiesCache()).when(properties).tenantsCache();
- doReturn("test").when(properties).clientAppId();
- doNothing().when(properties).validateSuperUserAccess();
-
- namespaces = spy(new Namespaces());
- namespaces.setServletContext(new MockServletContext());
- namespaces.setPulsar(pulsar);
- doReturn(mockZookKeeper).when(namespaces).globalZk();
- doReturn(mockZookKeeper).when(namespaces).localZk();
- doReturn(configurationCache.propertiesCache()).when(namespaces).tenantsCache();
- doReturn(configurationCache.policiesCache()).when(namespaces).policiesCache();
- doReturn("test").when(namespaces).clientAppId();
- doReturn(Sets.newTreeSet(Lists.newArrayList("use", "usw", "usc", "global"))).when(namespaces).clusters();
- doNothing().when(namespaces).validateAdminAccessForTenant("my-tenant");
- doNothing().when(namespaces).validateAdminAccessForTenant("other-tenant");
- doNothing().when(namespaces).validateAdminAccessForTenant("new-property");
-
- brokers = spy(new Brokers());
- brokers.setServletContext(new MockServletContext());
- brokers.setPulsar(pulsar);
- doReturn(mockZookKeeper).when(brokers).globalZk();
- doReturn(mockZookKeeper).when(brokers).localZk();
- doReturn(configurationCache.clustersListCache()).when(brokers).clustersListCache();
- doReturn("test").when(brokers).clientAppId();
- doNothing().when(brokers).validateSuperUserAccess();
-
- uriField = PulsarWebResource.class.getDeclaredField("uri");
- uriField.setAccessible(true);
-
- persistentTopics = spy(new PersistentTopics());
- persistentTopics.setServletContext(new MockServletContext());
- persistentTopics.setPulsar(pulsar);
- doReturn(mockZookKeeper).when(persistentTopics).globalZk();
- doReturn(mockZookKeeper).when(persistentTopics).localZk();
- doReturn(configurationCache.propertiesCache()).when(persistentTopics).tenantsCache();
- doReturn(configurationCache.policiesCache()).when(persistentTopics).policiesCache();
- doReturn("test").when(persistentTopics).clientAppId();
- doReturn("persistent").when(persistentTopics).domain();
- doReturn(Sets.newTreeSet(Lists.newArrayList("use", "usw", "usc"))).when(persistentTopics).clusters();
- doNothing().when(persistentTopics).validateAdminAccessForTenant("my-tenant");
- doNothing().when(persistentTopics).validateAdminAccessForTenant("other-tenant");
- doNothing().when(persistentTopics).validateAdminAccessForTenant("prop-xyz");
-
- resourceQuotas = spy(new ResourceQuotas());
- resourceQuotas.setServletContext(new MockServletContext());
- resourceQuotas.setPulsar(pulsar);
- doReturn(mockZookKeeper).when(resourceQuotas).globalZk();
- doReturn(mockZookKeeper).when(resourceQuotas).localZk();
- doReturn(configurationCache.propertiesCache()).when(resourceQuotas).tenantsCache();
- doReturn(configurationCache.policiesCache()).when(resourceQuotas).policiesCache();
-
- brokerStats = spy(new BrokerStats());
- brokerStats.setServletContext(new MockServletContext());
- brokerStats.setPulsar(pulsar);
- doReturn(mockZookKeeper).when(brokerStats).globalZk();
- doReturn(mockZookKeeper).when(brokerStats).localZk();
- doReturn(configurationCache.propertiesCache()).when(brokerStats).tenantsCache();
- doReturn(configurationCache.policiesCache()).when(brokerStats).policiesCache();
-
- schemasResource = spy(new SchemasResource(mockClock));
- schemasResource.setServletContext(new MockServletContext());
- schemasResource.setPulsar(pulsar);
- doReturn(mockZookKeeper).when(schemasResource).globalZk();
- doReturn(mockZookKeeper).when(schemasResource).localZk();
- doReturn(configurationCache.propertiesCache()).when(schemasResource).tenantsCache();
- doReturn(configurationCache.policiesCache()).when(schemasResource).policiesCache();
- }
-
- @Override
- @AfterMethod
- public void cleanup() throws Exception {
- super.internalCleanup();
- }
-
- @Test
- void internalConfiguration() throws Exception {
- InternalConfigurationData expectedData = new InternalConfigurationData(
- pulsar.getConfiguration().getZookeeperServers(),
- pulsar.getConfiguration().getConfigurationStoreServers(),
- new ClientConfiguration().getZkLedgersRootPath());
-
- assertEquals(brokers.getInternalConfigurationData(), expectedData);
- }
-
- @Test
- void clusters() throws Exception {
- assertEquals(clusters.getClusters(), Lists.newArrayList());
- verify(clusters, never()).validateSuperUserAccess();
-
- clusters.createCluster("use", new ClusterData("http://broker.messaging.use.example.com"));
- verify(clusters, times(1)).validateSuperUserAccess();
- // ensure to read from ZooKeeper directly
- clusters.clustersListCache().clear();
- assertEquals(clusters.getClusters(), Lists.newArrayList("use"));
-
- // Check creating existing cluster
- try {
- clusters.createCluster("use", new ClusterData("http://broker.messaging.use.example.com"));
- fail("should have failed");
- } catch (RestException e) {
- assertEquals(e.getResponse().getStatus(), Status.CONFLICT.getStatusCode());
- }
-
- // Check deleting non-existing cluster
- try {
- clusters.deleteCluster("usc");
- fail("should have failed");
- } catch (RestException e) {
- assertEquals(e.getResponse().getStatus(), Status.NOT_FOUND.getStatusCode());
- }
-
- assertEquals(clusters.getCluster("use"), new ClusterData("http://broker.messaging.use.example.com"));
- verify(clusters, times(4)).validateSuperUserAccess();
-
- clusters.updateCluster("use", new ClusterData("http://new-broker.messaging.use.example.com"));
- verify(clusters, times(5)).validateSuperUserAccess();
-
- assertEquals(clusters.getCluster("use"), new ClusterData("http://new-broker.messaging.use.example.com"));
- verify(clusters, times(6)).validateSuperUserAccess();
-
- try {
- clusters.getNamespaceIsolationPolicies("use");
- fail("should have failed");
- } catch (RestException e) {
- assertEquals(e.getResponse().getStatus(), 404);
- }
- NamespaceIsolationData policyData = new NamespaceIsolationData();
- policyData.namespaces = new ArrayList<String>();
- policyData.namespaces.add("dummy/colo/ns");
- policyData.primary = new ArrayList<String>();
- policyData.primary.add("localhost" + ":" + BROKER_WEBSERVICE_PORT);
- policyData.secondary = new ArrayList<String>();
- policyData.auto_failover_policy = new AutoFailoverPolicyData();
- policyData.auto_failover_policy.policy_type = AutoFailoverPolicyType.min_available;
- policyData.auto_failover_policy.parameters = new HashMap<String, String>();
- policyData.auto_failover_policy.parameters.put("min_limit", "1");
- policyData.auto_failover_policy.parameters.put("usage_threshold", "90");
- clusters.setNamespaceIsolationPolicy("use", "policy1", policyData);
- clusters.getNamespaceIsolationPolicies("use");
-
- try {
- clusters.deleteCluster("use");
- fail("should have failed");
- } catch (RestException e) {
- assertEquals(e.getResponse().getStatus(), 412);
- }
-
- clusters.deleteNamespaceIsolationPolicy("use", "policy1");
- assertTrue(clusters.getNamespaceIsolationPolicies("use").isEmpty());
-
- clusters.deleteCluster("use");
- verify(clusters, times(13)).validateSuperUserAccess();
- assertEquals(clusters.getClusters(), Lists.newArrayList());
-
- try {
- clusters.getCluster("use");
- fail("should have failed");
- } catch (RestException e) {
- assertEquals(e.getResponse().getStatus(), 404);
- }
-
- try {
- clusters.updateCluster("use", new ClusterData());
- fail("should have failed");
- } catch (RestException e) {
- assertEquals(e.getResponse().getStatus(), 404);
- }
-
- try {
- clusters.getNamespaceIsolationPolicies("use");
- fail("should have failed");
- } catch (RestException e) {
- assertEquals(e.getResponse().getStatus(), 404);
- }
-
- // Test zk failures
- mockZookKeeper.failNow(Code.SESSIONEXPIRED);
- configurationCache.clustersListCache().clear();
- try {
- clusters.getClusters();
- fail("should have failed");
- } catch (RestException e) {
- assertEquals(e.getResponse().getStatus(), Status.INTERNAL_SERVER_ERROR.getStatusCode());
- }
-
- mockZookKeeper.failNow(Code.SESSIONEXPIRED);
- try {
- clusters.createCluster("test", new ClusterData("http://broker.messaging.test.example.com"));
- fail("should have failed");
- } catch (RestException e) {
- assertEquals(e.getResponse().getStatus(), Status.INTERNAL_SERVER_ERROR.getStatusCode());
- }
-
- mockZookKeeper.failNow(Code.SESSIONEXPIRED);
- try {
- clusters.updateCluster("test", new ClusterData("http://broker.messaging.test.example.com"));
- fail("should have failed");
- } catch (RestException e) {
- assertEquals(e.getResponse().getStatus(), Status.INTERNAL_SERVER_ERROR.getStatusCode());
- }
-
- mockZookKeeper.failNow(Code.SESSIONEXPIRED);
- try {
- clusters.getCluster("test");
- fail("should have failed");
- } catch (RestException e) {
- assertEquals(e.getResponse().getStatus(), Status.INTERNAL_SERVER_ERROR.getStatusCode());
- }
-
- mockZookKeeper.failAfter(0, Code.SESSIONEXPIRED);
- try {
- clusters.deleteCluster("use");
- fail("should have failed");
- } catch (RestException e) {
- assertEquals(e.getResponse().getStatus(), Status.INTERNAL_SERVER_ERROR.getStatusCode());
- }
-
- mockZookKeeper.failAfter(1, Code.SESSIONEXPIRED);
- try {
- clusters.deleteCluster("use");
- fail("should have failed");
- } catch (RestException e) {
- assertEquals(e.getResponse().getStatus(), Status.INTERNAL_SERVER_ERROR.getStatusCode());
- }
-
- // Check name validations
- try {
- clusters.createCluster("bf@", new ClusterData("http://dummy.messaging.example.com"));
- fail("should have filed");
- } catch (RestException e) {
- assertEquals(e.getResponse().getStatus(), Status.PRECONDITION_FAILED.getStatusCode());
- }
- }
-
- @Test
- void properties() throws Exception {
- assertEquals(properties.getTenants(), Lists.newArrayList());
- verify(properties, times(1)).validateSuperUserAccess();
-
- Set<String> allowedClusters = Sets.newHashSet();
- TenantInfo tenantInfo = new TenantInfo(Sets.newHashSet("role1", "role2"), allowedClusters);
- properties.createTenant("test-property", tenantInfo);
- verify(properties, times(2)).validateSuperUserAccess();
-
- assertEquals(properties.getTenants(), Lists.newArrayList("test-property"));
- verify(properties, times(3)).validateSuperUserAccess();
-
- assertEquals(properties.getTenantAdmin("test-property"), tenantInfo);
- verify(properties, times(4)).validateSuperUserAccess();
-
- TenantInfo newPropertyAdmin = new TenantInfo(Sets.newHashSet("role1", "other-role"), allowedClusters);
- properties.updateTenant("test-property", newPropertyAdmin);
- verify(properties, times(5)).validateSuperUserAccess();
-
- // Wait for updateTenant to take effect
- Thread.sleep(100);
-
- assertEquals(properties.getTenantAdmin("test-property"), newPropertyAdmin);
- assertNotSame(properties.getTenantAdmin("test-property"), tenantInfo);
- verify(properties, times(7)).validateSuperUserAccess();
-
- // Check creating existing property
- try {
- properties.createTenant("test-property", tenantInfo);
- fail("should have failed");
- } catch (RestException e) {
- assertEquals(e.getResponse().getStatus(), Status.CONFLICT.getStatusCode());
- }
-
- // Check non-existing property
- try {
- properties.getTenantAdmin("non-existing");
- fail("should have failed");
- } catch (RestException e) {
- assertEquals(e.getResponse().getStatus(), Status.NOT_FOUND.getStatusCode());
- }
-
- try {
- properties.updateTenant("xxx-non-existing", newPropertyAdmin);
- fail("should have failed");
- } catch (RestException e) {
- assertEquals(e.getResponse().getStatus(), Status.NOT_FOUND.getStatusCode());
- }
-
- // Check deleting non-existing property
- try {
- properties.deleteTenant("non-existing");
- fail("should have failed");
- } catch (RestException e) {
- assertEquals(e.getResponse().getStatus(), Status.NOT_FOUND.getStatusCode());
- }
-
- // Test zk failures
- mockZookKeeper.failNow(Code.SESSIONEXPIRED);
- try {
- properties.getTenants();
- fail("should have failed");
- } catch (RestException e) {
- assertEquals(e.getResponse().getStatus(), Status.INTERNAL_SERVER_ERROR.getStatusCode());
- }
-
- mockZookKeeper.failNow(Code.SESSIONEXPIRED);
- try {
- properties.getTenantAdmin("my-tenant");
- fail("should have failed");
- } catch (RestException e) {
- assertEquals(e.getResponse().getStatus(), Status.INTERNAL_SERVER_ERROR.getStatusCode());
- }
-
- mockZookKeeper.failNow(Code.SESSIONEXPIRED);
- try {
- properties.updateTenant("my-tenant", newPropertyAdmin);
- fail("should have failed");
- } catch (RestException e) {
- assertEquals(e.getResponse().getStatus(), Status.INTERNAL_SERVER_ERROR.getStatusCode());
- }
-
- mockZookKeeper.failNow(Code.SESSIONEXPIRED);
- try {
- properties.createTenant("test", tenantInfo);
- fail("should have failed");
- } catch (RestException e) {
- assertEquals(e.getResponse().getStatus(), Status.INTERNAL_SERVER_ERROR.getStatusCode());
- }
-
- mockZookKeeper.failNow(Code.SESSIONEXPIRED);
- try {
- properties.deleteTenant("my-tenant");
- fail("should have failed");
- } catch (RestException e) {
- assertEquals(e.getResponse().getStatus(), Status.INTERNAL_SERVER_ERROR.getStatusCode());
- }
-
- properties.createTenant("error-property", tenantInfo);
- mockZookKeeper.failAfter(2, Code.SESSIONEXPIRED);
- try {
- properties.deleteTenant("error-property");
- fail("should have failed");
- } catch (RestException e) {
- assertEquals(e.getResponse().getStatus(), Status.INTERNAL_SERVER_ERROR.getStatusCode());
- }
-
- properties.deleteTenant("test-property");
- properties.deleteTenant("error-property");
- assertEquals(properties.getTenants(), Lists.newArrayList());
-
- // Create a namespace to test deleting a non-empty property
- clusters.createCluster("use", new ClusterData());
- newPropertyAdmin = new TenantInfo(Sets.newHashSet("role1", "other-role"), Sets.newHashSet("use"));
- properties.createTenant("my-tenant", newPropertyAdmin);
-
- namespaces.createNamespace("my-tenant", "use", "my-namespace", new BundlesData());
-
- try {
- properties.deleteTenant("my-tenant");
- fail("should have failed");
- } catch (RestException e) {
- // Ok
- }
-
- // Check name validation
- try {
- properties.createTenant("test&", tenantInfo);
- fail("should have failed");
- } catch (RestException e) {
- assertEquals(e.getResponse().getStatus(), Status.PRECONDITION_FAILED.getStatusCode());
- }
-
- namespaces.deleteNamespace("my-tenant", "use", "my-namespace", false);
- properties.deleteTenant("my-tenant");
- }
-
- @Test
- void brokers() throws Exception {
- clusters.createCluster("use", new ClusterData("http://broker.messaging.use.example.com",
- "https://broker.messaging.use.example.com:4443"));
-
- URI requestUri = new URI(
- "http://broker.messaging.use.example.com" + ":" + BROKER_WEBSERVICE_PORT + "/admin/brokers/use");
- UriInfo mockUri = mock(UriInfo.class);
- doReturn(requestUri).when(mockUri).getRequestUri();
- Field uriField = PulsarWebResource.class.getDeclaredField("uri");
- uriField.setAccessible(true);
- uriField.set(brokers, mockUri);
-
- Set<String> activeBrokers = brokers.getActiveBrokers("use");
- assertEquals(activeBrokers.size(), 1);
- assertEquals(activeBrokers, Sets.newHashSet(pulsar.getAdvertisedAddress() + ":" + BROKER_WEBSERVICE_PORT));
- }
-
- @Test
- void resourceQuotas() throws Exception {
- // get Default Resource Quota
- ResourceQuota quota = resourceQuotas.getDefaultResourceQuota();
- assertNotNull(quota);
- assertTrue(quota.getBandwidthIn() > 0);
-
- // set Default Resource Quota
- double defaultBandwidth = 1000;
- quota.setBandwidthIn(defaultBandwidth);
- quota.setBandwidthOut(defaultBandwidth);
- resourceQuotas.setDefaultResourceQuota(quota);
- assertTrue(resourceQuotas.getDefaultResourceQuota().getBandwidthIn() == defaultBandwidth);
- assertTrue(resourceQuotas.getDefaultResourceQuota().getBandwidthOut() == defaultBandwidth);
-
- String property = "prop-xyz";
- String cluster = "use";
- String namespace = "ns";
- String bundleRange = "0x00000000_0xffffffff";
- Policies policies = new Policies();
- doReturn(policies).when(resourceQuotas).getNamespacePolicies(NamespaceName.get(property, cluster, namespace));
- doReturn("client-id").when(resourceQuotas).clientAppId();
-
- try {
- resourceQuotas.setNamespaceBundleResourceQuota(property, cluster, namespace, bundleRange, quota);
- fail();
- } catch (Exception e) {
- // OK : should fail without creating policies
- }
-
- try {
- resourceQuotas.removeNamespaceBundleResourceQuota(property, cluster, namespace, bundleRange);
- fail();
- } catch (Exception e) {
- // OK : should fail without creating policies
- }
-
- // create policies
- TenantInfo admin = new TenantInfo();
- admin.getAllowedClusters().add(cluster);
- mockZookKeeper.create(PulsarWebResource.path(POLICIES, property),
- ObjectMapperFactory.getThreadLocal().writeValueAsBytes(admin), Ids.OPEN_ACL_UNSAFE,
- CreateMode.PERSISTENT);
-
- // customized bandwidth for this namespace
- double customizeBandwidth = 3000;
- quota.setBandwidthIn(customizeBandwidth);
- quota.setBandwidthOut(customizeBandwidth);
-
- // set and get Resource Quota
- resourceQuotas.setNamespaceBundleResourceQuota(property, cluster, namespace, bundleRange, quota);
- ResourceQuota bundleQuota = resourceQuotas.getNamespaceBundleResourceQuota(property, cluster, namespace,
- bundleRange);
- assertEquals(quota, bundleQuota);
-
- // remove quota which sets to default quota
- resourceQuotas.removeNamespaceBundleResourceQuota(property, cluster, namespace, bundleRange);
- bundleQuota = resourceQuotas.getNamespaceBundleResourceQuota(property, cluster, namespace, bundleRange);
- assertTrue(bundleQuota.getBandwidthIn() == defaultBandwidth);
- assertTrue(bundleQuota.getBandwidthOut() == defaultBandwidth);
- }
-
- @Test
- void brokerStats() throws Exception {
- doReturn("client-id").when(brokerStats).clientAppId();
- Collection<Metrics> metrics = brokerStats.getMetrics();
- assertNotNull(metrics);
- LocalBrokerData loadReport = (LocalBrokerData) brokerStats.getLoadReport();
- assertNotNull(loadReport);
- assertNotNull(loadReport.getCpu());
- Collection<Metrics> mBeans = brokerStats.getMBeans();
- assertTrue(!mBeans.isEmpty());
- AllocatorStats allocatorStats = brokerStats.getAllocatorStats("default");
- assertNotNull(allocatorStats);
- Map<String, Map<String, PendingBookieOpsStats>> bookieOpsStats = brokerStats.getPendingBookieOpsStats();
- assertTrue(bookieOpsStats.isEmpty());
- StreamingOutput topic = brokerStats.getTopics2();
- assertNotNull(topic);
- try {
- brokerStats.getBrokerResourceAvailability("prop", "use", "ns2");
- fail("should have failed as ModularLoadManager doesn't support it");
- } catch (RestException re) {
- // Ok
- }
- }
-
- @Test
- void persistentTopics() throws Exception {
-
- final String property = "prop-xyz";
- final String cluster = "use";
- final String namespace = "ns";
- final String topic = "ds1";
- Policies policies = new Policies();
- doReturn(policies).when(resourceQuotas).getNamespacePolicies(NamespaceName.get(property, cluster, namespace));
- doReturn("client-id").when(resourceQuotas).clientAppId();
- // create policies
- TenantInfo admin = new TenantInfo();
- admin.getAllowedClusters().add(cluster);
- ZkUtils.createFullPathOptimistic(mockZookKeeper, PulsarWebResource.path(POLICIES, property, cluster, namespace),
- ObjectMapperFactory.getThreadLocal().writeValueAsBytes(new Policies()), ZooDefs.Ids.OPEN_ACL_UNSAFE,
- CreateMode.PERSISTENT);
-
- List<String> list = persistentTopics.getList(property, cluster, namespace);
- assertTrue(list.isEmpty());
- // create topic
- assertEquals(persistentTopics.getPartitionedTopicList(property, cluster, namespace), Lists.newArrayList());
- persistentTopics.createPartitionedTopic(property, cluster, namespace, topic, 5, false);
- assertEquals(persistentTopics.getPartitionedTopicList(property, cluster, namespace), Lists
- .newArrayList(String.format("persistent://%s/%s/%s/%s", property, cluster, namespace, topic)));
-
- CountDownLatch notificationLatch = new CountDownLatch(2);
- configurationCache.policiesCache().registerListener((path, data, stat) -> {
- notificationLatch.countDown();
- });
-
- // grant permission
- final Set<AuthAction> actions = Sets.newHashSet(AuthAction.produce);
- final String role = "test-role";
- persistentTopics.grantPermissionsOnTopic(property, cluster, namespace, topic, role, actions);
- // verify permission
- Map<String, Set<AuthAction>> permission = persistentTopics.getPermissionsOnTopic(property, cluster,
- namespace, topic);
- assertEquals(permission.get(role), actions);
- // remove permission
- persistentTopics.revokePermissionsOnTopic(property, cluster, namespace, topic, role);
-
- // Wait for cache to be updated
- notificationLatch.await();
-
- // verify removed permission
- permission = persistentTopics.getPermissionsOnTopic(property, cluster, namespace, topic);
- assertTrue(permission.isEmpty());
- }
-
- @Test
- public void testRestExceptionMessage() {
- String message = "my-message";
- RestException exception = new RestException(Status.PRECONDITION_FAILED, message);
- assertEquals(exception.getMessage(), message);
-
- }
-
-}
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/CreateSubscriptionTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/CreateSubscriptionTest.java
deleted file mode 100644
index f5d8bd0db9..0000000000
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/CreateSubscriptionTest.java
+++ /dev/null
@@ -1,132 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pulsar.broker.admin;
-
-import static org.testng.Assert.assertEquals;
-import static org.testng.Assert.fail;
-
-import com.google.common.collect.Lists;
-
-import javax.ws.rs.ClientErrorException;
-import javax.ws.rs.core.Response.Status;
-
-import org.apache.pulsar.client.admin.PulsarAdminException.ConflictException;
-import org.apache.pulsar.client.api.MessageId;
-import org.apache.pulsar.client.api.Producer;
-import org.apache.pulsar.client.api.ProducerConsumerBase;
-import org.apache.pulsar.common.naming.TopicName;
-import org.testng.annotations.AfterMethod;
-import org.testng.annotations.BeforeMethod;
-import org.testng.annotations.Test;
-
-public class CreateSubscriptionTest extends ProducerConsumerBase {
-
- @BeforeMethod
- @Override
- public void setup() throws Exception {
- super.internalSetup();
- producerBaseSetup();
- }
-
- @AfterMethod
- @Override
- public void cleanup() throws Exception {
- super.internalCleanup();
- }
-
- @Test
- public void createSubscriptionSingleTopic() throws Exception {
- String topic = "persistent://my-property/my-ns/my-topic";
- admin.topics().createSubscription(topic, "sub-1", MessageId.latest);
-
- // Create should fail if the subscription already exists
- try {
- admin.topics().createSubscription(topic, "sub-1", MessageId.latest);
- fail("Should have failed");
- } catch (ConflictException e) {
- assertEquals(((ClientErrorException) e.getCause()).getResponse().getStatus(),
- Status.CONFLICT.getStatusCode());
- }
-
- assertEquals(admin.topics().getSubscriptions(topic), Lists.newArrayList("sub-1"));
-
- Producer<byte[]> p1 = pulsarClient.newProducer().topic(topic).create();
- p1.send("test-1".getBytes());
- p1.send("test-2".getBytes());
- MessageId m3 = p1.send("test-3".getBytes());
-
- assertEquals(admin.topics().getStats(topic).subscriptions.get("sub-1").msgBacklog, 3);
-
- admin.topics().createSubscription(topic, "sub-2", MessageId.latest);
- assertEquals(admin.topics().getStats(topic).subscriptions.get("sub-2").msgBacklog, 0);
-
- admin.topics().createSubscription(topic, "sub-3", MessageId.earliest);
- assertEquals(admin.topics().getStats(topic).subscriptions.get("sub-3").msgBacklog, 3);
-
- admin.topics().createSubscription(topic, "sub-5", m3);
- assertEquals(admin.topics().getStats(topic).subscriptions.get("sub-5").msgBacklog, 1);
- }
-
- @Test
- public void createSubscriptionOnPartitionedTopic() throws Exception {
- String topic = "persistent://my-property/my-ns/my-partitioned-topic";
- admin.topics().createPartitionedTopic(topic, 10);
-
- admin.topics().createSubscription(topic, "sub-1", MessageId.latest);
-
- // Create should fail if the subscription already exists
- try {
- admin.topics().createSubscription(topic, "sub-1", MessageId.latest);
- fail("Should have failed");
- } catch (Exception e) {
- // Expected
- }
-
- for (int i = 0; i < 10; i++) {
- assertEquals(admin.topics().getSubscriptions(TopicName.get(topic).getPartition(i).toString()),
- Lists.newArrayList("sub-1"));
- }
- }
-
- @Test
- public void createSubscriptionOnPartitionedTopicWithPartialFailure() throws Exception {
- String topic = "persistent://my-property/my-ns/my-partitioned-topic";
- admin.topics().createPartitionedTopic(topic, 10);
-
- // create subscription for one partition
- final String partitionedTopic0 = topic+"-partition-0";
- admin.topics().createSubscription(partitionedTopic0, "sub-1", MessageId.latest);
-
- admin.topics().createSubscription(topic, "sub-1", MessageId.latest);
-
- // Create should fail if the subscription already exists
- try {
- admin.topics().createSubscription(topic, "sub-1", MessageId.latest);
- fail("Should have failed");
- } catch (Exception e) {
- // Expected
- }
-
- for (int i = 0; i < 10; i++) {
- assertEquals(
- admin.topics().getSubscriptions(TopicName.get(topic).getPartition(i).toString()),
- Lists.newArrayList("sub-1"));
- }
- }
-}
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/IncrementPartitionsTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/IncrementPartitionsTest.java
deleted file mode 100644
index 02430c8c68..0000000000
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/IncrementPartitionsTest.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pulsar.broker.admin;
-
-import static org.testng.Assert.assertEquals;
-
-import org.apache.pulsar.broker.admin.AdminApiTest.MockedPulsarService;
-import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest;
-import org.apache.pulsar.client.api.Consumer;
-import org.apache.pulsar.common.naming.TopicName;
-import org.apache.pulsar.common.policies.data.ClusterData;
-import org.apache.pulsar.common.policies.data.TenantInfo;
-import org.testng.annotations.AfterMethod;
-import org.testng.annotations.BeforeMethod;
-import org.testng.annotations.Test;
-
-import com.google.common.collect.Lists;
-import com.google.common.collect.Sets;
-
-public class IncrementPartitionsTest extends MockedPulsarServiceBaseTest {
-
- private MockedPulsarService mockPulsarSetup;
-
- @BeforeMethod
- @Override
- public void setup() throws Exception {
- conf.setLoadBalancerEnabled(true);
- super.internalSetup();
-
- // create otherbroker to test redirect on calls that need
- // namespace ownership
- mockPulsarSetup = new MockedPulsarService(this.conf);
- mockPulsarSetup.setup();
-
- // Setup namespaces
- admin.clusters().createCluster("use", new ClusterData("http://127.0.0.1" + ":" + BROKER_WEBSERVICE_PORT));
- TenantInfo tenantInfo = new TenantInfo(Sets.newHashSet("role1", "role2"), Sets.newHashSet("use"));
- admin.tenants().createTenant("prop-xyz", tenantInfo);
- admin.namespaces().createNamespace("prop-xyz/use/ns1");
- }
-
- @AfterMethod
- @Override
- public void cleanup() throws Exception {
- super.internalCleanup();
- mockPulsarSetup.cleanup();
- }
-
- @Test
- public void testIncrementPartitionsOfTopicOnUnusedTopic() throws Exception {
- final String partitionedTopicName = "persistent://prop-xyz/use/ns1/test-topic";
-
- admin.topics().createPartitionedTopic(partitionedTopicName, 10);
- assertEquals(admin.topics().getPartitionedTopicMetadata(partitionedTopicName).partitions, 10);
-
- admin.topics().updatePartitionedTopic(partitionedTopicName, 20);
- assertEquals(admin.topics().getPartitionedTopicMetadata(partitionedTopicName).partitions, 20);
- }
-
- @Test
- public void testIncrementPartitionsOfTopic() throws Exception {
- final String partitionedTopicName = "persistent://prop-xyz/use/ns1/test-topic-2";
-
- admin.topics().createPartitionedTopic(partitionedTopicName, 10);
- assertEquals(admin.topics().getPartitionedTopicMetadata(partitionedTopicName).partitions, 10);
-
- Consumer<byte[]> consumer = pulsarClient.newConsumer().topic(partitionedTopicName).subscriptionName("sub-1")
- .subscribe();
-
- admin.topics().updatePartitionedTopic(partitionedTopicName, 20);
- assertEquals(admin.topics().getPartitionedTopicMetadata(partitionedTopicName).partitions, 20);
-
- assertEquals(admin.topics().getSubscriptions(
- TopicName.get(partitionedTopicName).getPartition(15).toString()), Lists.newArrayList("sub-1"));
-
- consumer.close();
- }
-}
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/NamespacesTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/NamespacesTest.java
deleted file mode 100644
index 9efdc6daae..0000000000
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/NamespacesTest.java
+++ /dev/null
@@ -1,1191 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pulsar.broker.admin;
-
-import static org.apache.pulsar.broker.cache.ConfigurationCacheService.POLICIES;
-import static org.mockito.Matchers.any;
-import static org.mockito.Mockito.doNothing;
-import static org.mockito.Mockito.doReturn;
-import static org.mockito.Mockito.doThrow;
-import static org.mockito.Mockito.mock;
-import static org.mockito.Mockito.spy;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
-import static org.testng.Assert.assertEquals;
-import static org.testng.Assert.assertNotEquals;
-import static org.testng.Assert.assertNotNull;
-import static org.testng.Assert.assertTrue;
-import static org.testng.Assert.fail;
-
-import java.lang.reflect.Field;
-import java.net.URI;
-import java.net.URL;
-import java.util.EnumSet;
-import java.util.List;
-import java.util.Optional;
-
-import javax.ws.rs.ClientErrorException;
-import javax.ws.rs.WebApplicationException;
-import javax.ws.rs.core.Response.Status;
-import javax.ws.rs.core.UriBuilder;
-import javax.ws.rs.core.UriInfo;
-
-import org.apache.bookkeeper.util.ZkUtils;
-import org.apache.pulsar.broker.admin.v1.Namespaces;
-import org.apache.pulsar.broker.admin.v1.PersistentTopics;
-import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest;
-import org.apache.pulsar.broker.namespace.NamespaceEphemeralData;
-import org.apache.pulsar.broker.namespace.NamespaceService;
-import org.apache.pulsar.broker.namespace.OwnershipCache;
-import org.apache.pulsar.broker.web.PulsarWebResource;
-import org.apache.pulsar.broker.web.RestException;
-import org.apache.pulsar.client.admin.PulsarAdminException;
-import org.apache.pulsar.client.api.Producer;
-import org.apache.pulsar.common.naming.NamespaceBundle;
-import org.apache.pulsar.common.naming.NamespaceBundles;
-import org.apache.pulsar.common.naming.NamespaceName;
-import org.apache.pulsar.common.naming.TopicName;
-import org.apache.pulsar.common.policies.data.AuthAction;
-import org.apache.pulsar.common.policies.data.BundlesData;
-import org.apache.pulsar.common.policies.data.ClusterData;
-import org.apache.pulsar.common.policies.data.PersistencePolicies;
-import org.apache.pulsar.common.policies.data.Policies;
-import org.apache.pulsar.common.policies.data.TenantInfo;
-import org.apache.pulsar.common.policies.data.RetentionPolicies;
-import org.apache.pulsar.common.util.ObjectMapperFactory;
-import org.apache.zookeeper.CreateMode;
-import org.apache.zookeeper.KeeperException.Code;
-import org.apache.zookeeper.ZooDefs;
-import org.hamcrest.Description;
-import org.hamcrest.Matcher;
-import org.mockito.Mockito;
-import org.testng.annotations.AfterMethod;
-import org.testng.annotations.BeforeClass;
-import org.testng.annotations.BeforeMethod;
-import org.testng.annotations.Test;
-
-import com.google.common.collect.Lists;
-import com.google.common.collect.Sets;
-
-@Test
-public class NamespacesTest extends MockedPulsarServiceBaseTest {
-
- private Namespaces namespaces;
-
- private List<NamespaceName> testLocalNamespaces;
- private List<NamespaceName> testGlobalNamespaces;
- private final String testTenant = "my-tenant";
- private final String testOtherTenant = "other-tenant";
- private final String testLocalCluster = "use";
- private final String testOtherCluster = "usc";
-
- protected NamespaceService nsSvc;
- protected Field uriField;
- protected UriInfo uriInfo;
-
- public NamespacesTest() {
- super();
- conf.setClusterName(testLocalCluster);
- }
-
- @BeforeClass
- public void initNamespace() throws Exception {
- testLocalNamespaces = Lists.newArrayList();
- testGlobalNamespaces = Lists.newArrayList();
-
- testLocalNamespaces.add(NamespaceName.get(this.testTenant, this.testLocalCluster, "test-namespace-1"));
- testLocalNamespaces.add(NamespaceName.get(this.testTenant, this.testLocalCluster, "test-namespace-2"));
- testLocalNamespaces.add(NamespaceName.get(this.testTenant, this.testOtherCluster, "test-other-namespace-1"));
- testLocalNamespaces.add(NamespaceName.get(this.testOtherTenant, this.testLocalCluster, "test-namespace-1"));
-
- testGlobalNamespaces.add(NamespaceName.get(this.testTenant, "global", "test-global-ns1"));
-
- uriField = PulsarWebResource.class.getDeclaredField("uri");
- uriField.setAccessible(true);
- uriInfo = mock(UriInfo.class);
- }
-
- @Override
- @BeforeMethod
- public void setup() throws Exception {
- super.internalSetup();
-
- namespaces = spy(new Namespaces());
- namespaces.setServletContext(new MockServletContext());
- namespaces.setPulsar(pulsar);
- doReturn(mockZookKeeper).when(namespaces).globalZk();
- doReturn(mockZookKeeper).when(namespaces).localZk();
- doReturn(pulsar.getConfigurationCache().propertiesCache()).when(namespaces).tenantsCache();
- doReturn(pulsar.getConfigurationCache().policiesCache()).when(namespaces).policiesCache();
- doReturn(false).when(namespaces).isRequestHttps();
- doReturn("test").when(namespaces).clientAppId();
- doReturn(null).when(namespaces).originalPrincipal();
- doReturn(Sets.newTreeSet(Lists.newArrayList("use", "usw", "usc", "global"))).when(namespaces).clusters();
- doNothing().when(namespaces).validateAdminAccessForTenant(this.testTenant);
- doNothing().when(namespaces).validateAdminAccessForTenant("non-existing-tenant");
- doNothing().when(namespaces).validateAdminAccessForTenant("new-property");
-
- admin.clusters().createCluster("use", new ClusterData("http://broker-use.com:" + BROKER_WEBSERVICE_PORT));
- admin.clusters().createCluster("usw", new ClusterData("http://broker-usw.com:" + BROKER_WEBSERVICE_PORT));
- admin.clusters().createCluster("usc", new ClusterData("http://broker-usc.com:" + BROKER_WEBSERVICE_PORT));
- admin.tenants().createTenant(this.testTenant,
- new TenantInfo(Sets.newHashSet("role1", "role2"), Sets.newHashSet("use", "usc", "usw")));
- admin.tenants().createTenant(this.testOtherTenant,
- new TenantInfo(Sets.newHashSet("role3", "role4"), Sets.newHashSet("use", "usc", "usw")));
-
- createTestNamespaces(this.testLocalNamespaces, new BundlesData());
- createGlobalTestNamespaces(this.testTenant, this.testGlobalNamespaces.get(0).getLocalName(),
- new BundlesData());
-
- doThrow(new RestException(Status.UNAUTHORIZED, "unauthorized")).when(namespaces)
- .validateAdminAccessForTenant(this.testOtherTenant);
-
- nsSvc = pulsar.getNamespaceService();
- }
-
- @Override
- @AfterMethod
- public void cleanup() throws Exception {
- super.internalCleanup();
- }
-
- @Test
- public void testCreateNamespaces() throws Exception {
- try {
- namespaces.createNamespace(this.testTenant, "other-colo", "my-namespace", new BundlesData());
- fail("should have failed");
- } catch (RestException e) {
- // Ok, cluster doesn't exist
- }
-
- List<NamespaceName> nsnames = Lists.newArrayList();
- nsnames.add(NamespaceName.get(this.testTenant, "use", "create-namespace-1"));
- nsnames.add(NamespaceName.get(this.testTenant, "use", "create-namespace-2"));
- nsnames.add(NamespaceName.get(this.testTenant, "usc", "create-other-namespace-1"));
- createTestNamespaces(nsnames, new BundlesData());
-
- try {
- namespaces.createNamespace(this.testTenant, "use", "create-namespace-1", new BundlesData());
- fail("should have failed");
- } catch (RestException e) {
- // Ok, namespace already exists
- }
-
- try {
- namespaces.createNamespace("non-existing-tenant", "use", "create-namespace-1", new BundlesData());
- fail("should have failed");
- } catch (RestException e) {
- // Ok, tenant doesn't exist
- }
-
- try {
- namespaces.createNamespace(this.testTenant, "use", "create-namespace-#", new BundlesData());
- fail("should have failed");
- } catch (RestException e) {
- // Ok, invalid namespace name
- assertEquals(e.getResponse().getStatus(), Status.PRECONDITION_FAILED.getStatusCode());
- }
-
- mockZookKeeper.failNow(Code.SESSIONEXPIRED);
- try {
- namespaces.createNamespace(this.testTenant, "use", "my-namespace-3", new BundlesData());
- fail("should have failed");
- } catch (RestException e) {
- // Ok
- }
- }
-
- @Test
- public void testGetNamespaces() throws Exception {
- List<String> expectedList = Lists.newArrayList(this.testLocalNamespaces.get(0).toString(),
- this.testLocalNamespaces.get(1).toString());
- expectedList.sort(null);
- assertEquals(namespaces.getNamespacesForCluster(this.testTenant, this.testLocalCluster), expectedList);
- expectedList = Lists.newArrayList(this.testLocalNamespaces.get(0).toString(),
- this.testLocalNamespaces.get(1).toString(), this.testLocalNamespaces.get(2).toString(),
- this.testGlobalNamespaces.get(0).toString());
- expectedList.sort(null);
- assertEquals(namespaces.getTenantNamespaces(this.testTenant), expectedList);
-
- try {
- namespaces.getTenantNamespaces("non-existing-tenant");
- fail("should have failed");
- } catch (RestException e) {
- // Ok, does not exist
- }
-
- try {
- namespaces.getNamespacesForCluster(this.testTenant, "other-cluster");
- fail("should have failed");
- } catch (RestException e) {
- // Ok, does not exist
- }
-
- // ZK Errors
- mockZookKeeper.failNow(Code.SESSIONEXPIRED);
- try {
- namespaces.getTenantNamespaces(this.testTenant);
- fail("should have failed");
- } catch (RestException e) {
- // Ok
- }
-
- mockZookKeeper.failNow(Code.SESSIONEXPIRED);
- try {
- namespaces.getNamespacesForCluster(this.testTenant, this.testLocalCluster);
- fail("should have failed");
- } catch (RestException e) {
- // Ok
- }
-
- }
-
- @Test(enabled = false)
- public void testGrantAndRevokePermissions() throws Exception {
- Policies expectedPolicies = new Policies();
- assertEquals(namespaces.getPolicies(this.testTenant, this.testLocalCluster,
- this.testLocalNamespaces.get(0).getLocalName()), expectedPolicies);
- assertEquals(namespaces.getPermissions(this.testTenant, this.testLocalCluster,
- this.testLocalNamespaces.get(0).getLocalName()), expectedPolicies.auth_policies.namespace_auth);
-
- namespaces.grantPermissionOnNamespace(this.testTenant, this.testLocalCluster,
- this.testLocalNamespaces.get(0).getLocalName(), "my-role", EnumSet.of(AuthAction.produce));
-
- expectedPolicies.auth_policies.namespace_auth.put("my-role", EnumSet.of(AuthAction.produce));
- assertEquals(namespaces.getPolicies(this.testTenant, this.testLocalCluster,
- this.testLocalNamespaces.get(0).getLocalName()), expectedPolicies);
- assertEquals(namespaces.getPermissions(this.testTenant, this.testLocalCluster,
- this.testLocalNamespaces.get(0).getLocalName()), expectedPolicies.auth_policies.namespace_auth);
-
- namespaces.grantPermissionOnNamespace(this.testTenant, this.testLocalCluster,
- this.testLocalNamespaces.get(0).getLocalName(), "other-role", EnumSet.of(AuthAction.consume));
- expectedPolicies.auth_policies.namespace_auth.put("other-role", EnumSet.of(AuthAction.consume));
- assertEquals(namespaces.getPolicies(this.testTenant, this.testLocalCluster,
- this.testLocalNamespaces.get(0).getLocalName()), expectedPolicies);
- assertEquals(namespaces.getPermissions(this.testTenant, this.testLocalCluster,
- this.testLocalNamespaces.get(0).getLocalName()), expectedPolicies.auth_policies.namespace_auth);
-
- namespaces.revokePermissionsOnNamespace(this.testTenant, this.testLocalCluster,
- this.testLocalNamespaces.get(0).getLocalName(), "my-role");
- expectedPolicies.auth_policies.namespace_auth.remove("my-role");
- assertEquals(namespaces.getPolicies(this.testTenant, this.testLocalCluster,
- this.testLocalNamespaces.get(0).getLocalName()), expectedPolicies);
- assertEquals(namespaces.getPermissions(this.testTenant, this.testLocalCluster,
- this.testLocalNamespaces.get(0).getLocalName()), expectedPolicies.auth_policies.namespace_auth);
-
- // Non-existing namespaces
- try {
- namespaces.getPolicies(this.testTenant, this.testLocalCluster, "non-existing-namespace-1");
- fail("should have failed");
- } catch (RestException e) {
- assertEquals(e.getResponse().getStatus(), Status.NOT_FOUND.getStatusCode());
- }
-
- try {
- namespaces.getPermissions(this.testTenant, this.testLocalCluster, "non-existing-namespace-1");
- fail("should have failed");
- } catch (RestException e) {
- assertEquals(e.getResponse().getStatus(), Status.NOT_FOUND.getStatusCode());
- }
-
- try {
- namespaces.grantPermissionOnNamespace(this.testTenant, this.testLocalCluster, "non-existing-namespace-1",
- "my-role", EnumSet.of(AuthAction.produce));
- fail("should have failed");
- } catch (RestException e) {
- assertEquals(e.getResponse().getStatus(), Status.NOT_FOUND.getStatusCode());
- }
-
- try {
- namespaces.revokePermissionsOnNamespace(this.testTenant, this.testLocalCluster,
- "non-existing-namespace-1", "my-role");
- fail("should have failed");
- } catch (RestException e) {
- assertEquals(e.getResponse().getStatus(), Status.NOT_FOUND.getStatusCode());
- }
-
- NamespaceName testNs = this.testLocalNamespaces.get(1);
-
- mockZookKeeper.failNow(Code.SESSIONEXPIRED);
- try {
- namespaces.getPolicies(testNs.getTenant(), testNs.getCluster(), testNs.getLocalName());
- fail("should have failed");
- } catch (RestException e) {
- // Ok
- }
-
- mockZookKeeper.failNow(Code.SESSIONEXPIRED);
- try {
- namespaces.getPermissions(testNs.getTenant(), testNs.getCluster(), testNs.getLocalName());
- fail("should have failed");
- } catch (RestException e) {
- // Ok
- }
-
- mockZookKeeper.failNow(Code.SESSIONEXPIRED);
- try {
- namespaces.grantPermissionOnNamespace(testNs.getTenant(), testNs.getCluster(), testNs.getLocalName(),
- "other-role", EnumSet.of(AuthAction.consume));
- fail("should have failed");
- } catch (RestException e) {
- // Ok
- }
-
- mockZookKeeper.failNow(Code.BADVERSION);
- try {
- namespaces.grantPermissionOnNamespace(testNs.getTenant(), testNs.getCluster(), testNs.getLocalName(),
- "other-role", EnumSet.of(AuthAction.consume));
- fail("should have failed");
- } catch (RestException e) {
- assertEquals(e.getResponse().getStatus(), Status.CONFLICT.getStatusCode());
- }
-
- mockZookKeeper.failNow(Code.BADVERSION);
- try {
- namespaces.revokePermissionsOnNamespace(testNs.getTenant(), testNs.getCluster(), testNs.getLocalName(),
- "other-role");
- fail("should have failed");
- } catch (RestException e) {
- assertEquals(e.getResponse().getStatus(), Status.CONFLICT.getStatusCode());
- }
-
- mockZookKeeper.failNow(Code.SESSIONEXPIRED);
- try {
- namespaces.revokePermissionsOnNamespace(testNs.getTenant(), testNs.getCluster(), testNs.getLocalName(),
- "other-role");
- fail("should have failed");
- } catch (RestException e) {
- // Ok
- }
- }
-
- @Test
- public void testGlobalNamespaceReplicationConfiguration() throws Exception {
- assertEquals(
- namespaces.getNamespaceReplicationClusters(this.testGlobalNamespaces.get(0).getTenant(),
- this.testGlobalNamespaces.get(0).getCluster(), this.testGlobalNamespaces.get(0).getLocalName()),
- Sets.newHashSet());
-
- namespaces.setNamespaceReplicationClusters(this.testGlobalNamespaces.get(0).getTenant(),
- this.testGlobalNamespaces.get(0).getCluster(), this.testGlobalNamespaces.get(0).getLocalName(),
- Lists.newArrayList("use", "usw"));
- assertEquals(
- namespaces.getNamespaceReplicationClusters(this.testGlobalNamespaces.get(0).getTenant(),
- this.testGlobalNamespaces.get(0).getCluster(), this.testGlobalNamespaces.get(0).getLocalName()),
- Lists.newArrayList("use", "usw"));
-
- try {
- namespaces.setNamespaceReplicationClusters(this.testGlobalNamespaces.get(0).getTenant(),
- this.testGlobalNamespaces.get(0).getCluster(), this.testGlobalNamespaces.get(0).getLocalName(),
- Lists.newArrayList("use", "invalid-cluster"));
- fail("should have failed");
- } catch (RestException e) {
- assertEquals(e.getResponse().getStatus(), Status.FORBIDDEN.getStatusCode());
- }
-
- try {
- namespaces.setNamespaceReplicationClusters(this.testGlobalNamespaces.get(0).getTenant(),
- this.testGlobalNamespaces.get(0).getCluster(), this.testGlobalNamespaces.get(0).getLocalName(),
- Lists.newArrayList("use", "global"));
- fail("should have failed");
- } catch (RestException e) {
- // Ok, global should not be allowed in the list of replication clusters
- assertEquals(e.getResponse().getStatus(), Status.PRECONDITION_FAILED.getStatusCode());
- }
-
- try {
- namespaces.setNamespaceReplicationClusters(this.testTenant, "global",
- this.testGlobalNamespaces.get(0).getLocalName(), Lists.newArrayList("use", "invalid-cluster"));
- fail("should have failed");
- } catch (RestException e) {
- // Ok, invalid-cluster is an invalid cluster id
- assertEquals(e.getResponse().getStatus(), Status.FORBIDDEN.getStatusCode());
- }
-
- admin.tenants().updateTenant(testTenant,
- new TenantInfo(Sets.newHashSet("role1", "role2"), Sets.newHashSet("use", "usc")));
-
- try {
- namespaces.setNamespaceReplicationClusters(this.testTenant, "global",
- this.testGlobalNamespaces.get(0).getLocalName(), Lists.newArrayList("use", "usw"));
- fail("should have failed");
- } catch (RestException e) {
- // Ok, usw was not configured in the list of allowed clusters
- assertEquals(e.getResponse().getStatus(), Status.FORBIDDEN.getStatusCode());
- }
-
- // Sometimes watcher event consumes scheduled exception, so set to always fail to ensure exception is
- // thrown for api call.
- mockZookKeeper.setAlwaysFail(Code.SESSIONEXPIRED);
- pulsar.getConfigurationCache().policiesCache().invalidate(AdminResource.path(POLICIES, this.testTenant,
- "global", this.testGlobalNamespaces.get(0).getLocalName()));
- try {
- namespaces.setNamespaceReplicationClusters(this.testTenant, "global",
- this.testGlobalNamespaces.get(0).getLocalName(), Lists.newArrayList("use"));
- fail("should have failed");
- } catch (RestException e) {
- assertEquals(e.getResponse().getStatus(), Status.INTERNAL_SERVER_ERROR.getStatusCode());
- } finally {
- mockZookKeeper.unsetAlwaysFail();
- }
-
- mockZookKeeper.failNow(Code.BADVERSION);
- try {
- namespaces.setNamespaceReplicationClusters(this.testTenant, "global",
- this.testGlobalNamespaces.get(0).getLocalName(), Lists.newArrayList("use"));
- fail("should have failed");
- } catch (RestException e) {
- assertEquals(e.getResponse().getStatus(), Status.CONFLICT.getStatusCode());
- }
-
- try {
- namespaces.getNamespaceReplicationClusters(this.testTenant, "global", "non-existing-ns");
- fail("should have failed");
- } catch (RestException e) {
- assertEquals(e.getResponse().getStatus(), Status.NOT_FOUND.getStatusCode());
- }
-
- try {
- namespaces.setNamespaceReplicationClusters(this.testTenant, "global", "non-existing-ns",
- Lists.newArrayList("use"));
- fail("should have failed");
- } catch (RestException e) {
- assertEquals(e.getResponse().getStatus(), Status.NOT_FOUND.getStatusCode());
- }
-
- mockZookKeeper.failNow(Code.SESSIONEXPIRED);
- pulsar.getConfigurationCache().policiesCache().clear();
-
- // ensure the ZooKeeper read happens, bypassing the cache
- try {
- namespaces.getNamespaceReplicationClusters(this.testTenant, "global",
- this.testGlobalNamespaces.get(0).getLocalName());
- fail("should have failed");
- } catch (RestException e) {
- assertEquals(e.getResponse().getStatus(), 500);
- }
-
- try {
- namespaces.getNamespaceReplicationClusters(this.testTenant, this.testLocalCluster,
- this.testLocalNamespaces.get(0).getLocalName());
- fail("should have failed");
- } catch (RestException e) {
- assertEquals(e.getResponse().getStatus(), Status.PRECONDITION_FAILED.getStatusCode());
- }
-
- try {
- namespaces.setNamespaceReplicationClusters(this.testTenant, this.testLocalCluster,
- this.testLocalNamespaces.get(0).getLocalName(), Lists.newArrayList("use"));
- fail("should have failed");
- } catch (RestException e) {
- assertEquals(e.getResponse().getStatus(), Status.PRECONDITION_FAILED.getStatusCode());
- }
-
- }
-
- @Test
- public void testGetBundles() throws Exception {
- BundlesData bundle = new BundlesData(Lists.newArrayList("0x00000000", "0x80000000", "0xffffffff"));
- createBundledTestNamespaces(this.testTenant, this.testLocalCluster, "test-bundled-namespace-1", bundle);
- BundlesData responseData = namespaces.getBundlesData(testTenant, this.testLocalCluster,
- "test-bundled-namespace-1");
-
- assertEquals(responseData, bundle);
- }
-
- @Test
- public void testNamespacesApiRedirects() throws Exception {
- // Redirect cases
- uriField.set(namespaces, uriInfo);
- doReturn(false).when(namespaces).isLeaderBroker();
-
- URI uri = URI.create("http://localhost" + ":" + BROKER_WEBSERVICE_PORT + "/admin/namespace/"
- + this.testLocalNamespaces.get(2).toString());
- doReturn(uri).when(uriInfo).getRequestUri();
-
- // Trick to force redirection
- conf.setAuthorizationEnabled(true);
-
- try {
- namespaces.deleteNamespace(this.testTenant, this.testOtherCluster,
- this.testLocalNamespaces.get(2).getLocalName(), false);
- fail("Should have raised exception to redirect request");
- } catch (WebApplicationException wae) {
- // OK
- assertEquals(wae.getResponse().getStatus(), Status.TEMPORARY_REDIRECT.getStatusCode());
- assertEquals(wae.getResponse().getLocation().toString(),
- UriBuilder.fromUri(uri).host("broker-usc.com").port(BROKER_WEBSERVICE_PORT).toString());
- }
-
- uri = URI.create("http://localhost" + ":" + BROKER_WEBSERVICE_PORT + "/admin/namespace/"
- + this.testLocalNamespaces.get(2).toString() + "/unload");
- doReturn(uri).when(uriInfo).getRequestUri();
-
- try {
- namespaces.unloadNamespaceBundle(this.testTenant, this.testOtherCluster,
- this.testLocalNamespaces.get(2).getLocalName(), "0x00000000_0xffffffff", false);
- fail("Should have raised exception to redirect request");
- } catch (WebApplicationException wae) {
- // OK
- assertEquals(wae.getResponse().getStatus(), Status.TEMPORARY_REDIRECT.getStatusCode());
- assertEquals(wae.getResponse().getLocation().toString(),
- UriBuilder.fromUri(uri).host("broker-usc.com").port(BROKER_WEBSERVICE_PORT).toString());
- }
-
- uri = URI.create("http://localhost" + ":" + BROKER_WEBSERVICE_PORT + "/admin/namespace/"
- + this.testGlobalNamespaces.get(0).toString() + "/configversion");
- doReturn(uri).when(uriInfo).getRequestUri();
-
- // setup to redirect to another broker in the same cluster
- doReturn(Optional.of(new URL("http://otherhost" + ":" + BROKER_WEBSERVICE_PORT))).when(nsSvc)
- .getWebServiceUrl(Mockito.argThat(new Matcher<NamespaceName>() {
-
- @Override
- public void describeTo(Description description) {
- // TODO Auto-generated method stub
-
- }
-
- @Override
- public boolean matches(Object item) {
- NamespaceName nsname = (NamespaceName) item;
- return nsname.equals(NamespacesTest.this.testGlobalNamespaces.get(0));
- }
-
- @Override
- public void _dont_implement_Matcher___instead_extend_BaseMatcher_() {
- // TODO Auto-generated method stub
-
- }
-
- }), Mockito.anyBoolean(), Mockito.anyBoolean(), Mockito.anyBoolean());
-
- admin.namespaces().setNamespaceReplicationClusters(testGlobalNamespaces.get(0).toString(),
- Sets.newHashSet("usw"));
-
- uri = URI.create("http://localhost" + ":" + BROKER_WEBSERVICE_PORT + "/admin/namespace/"
- + this.testLocalNamespaces.get(2).toString() + "?authoritative=false");
- doReturn(uri).when(uriInfo).getRequestUri();
- doReturn(true).when(namespaces).isLeaderBroker();
-
- try {
- namespaces.deleteNamespace(this.testLocalNamespaces.get(2).getTenant(),
- this.testLocalNamespaces.get(2).getCluster(), this.testLocalNamespaces.get(2).getLocalName(),
- false);
- fail("Should have raised exception to redirect request");
- } catch (WebApplicationException wae) {
- // OK
- assertEquals(wae.getResponse().getStatus(), Status.TEMPORARY_REDIRECT.getStatusCode());
- assertEquals(wae.getResponse().getLocation().toString(),
- UriBuilder.fromUri(uri).host("broker-usc.com").port(BROKER_WEBSERVICE_PORT).toString());
- }
- }
-
- @Test
- public void testDeleteNamespaces() throws Exception {
- try {
- namespaces.deleteNamespace(this.testTenant, this.testLocalCluster, "non-existing-namespace-1", false);
- fail("should have failed");
- } catch (RestException e) {
- assertEquals(e.getResponse().getStatus(), Status.NOT_FOUND.getStatusCode());
- }
-
- NamespaceName testNs = this.testLocalNamespaces.get(1);
- TopicName topicName = TopicName.get(testNs.getPersistentTopicName("my-topic"));
- ZkUtils.createFullPathOptimistic(mockZookKeeper, "/managed-ledgers/" + topicName.getPersistenceNamingEncoding(),
- new byte[0], null, null);
-
- // setup ownership to localhost
- URL localWebServiceUrl = new URL(pulsar.getWebServiceAddress());
- doReturn(Optional.of(localWebServiceUrl)).when(nsSvc).getWebServiceUrl(testNs, false, false, false);
- doReturn(true).when(nsSvc).isServiceUnitOwned(testNs);
- try {
- namespaces.deleteNamespace(testNs.getTenant(), testNs.getCluster(), testNs.getLocalName(), false);
- fail("should have failed");
- } catch (RestException e) {
- // Ok, namespace not empty
- assertEquals(e.getResponse().getStatus(), Status.CONFLICT.getStatusCode());
- }
-
- testNs = this.testGlobalNamespaces.get(0);
- // setup ownership to localhost
- doReturn(Optional.of(localWebServiceUrl)).when(nsSvc).getWebServiceUrl(testNs, false, false, false);
- doReturn(true).when(nsSvc).isServiceUnitOwned(testNs);
- namespaces.deleteNamespace(testNs.getTenant(), testNs.getCluster(), testNs.getLocalName(), false);
-
- testNs = this.testLocalNamespaces.get(0);
- // setup ownership to localhost
- doReturn(Optional.of(localWebServiceUrl)).when(nsSvc).getWebServiceUrl(testNs, false, false, false);
- doReturn(true).when(nsSvc).isServiceUnitOwned(testNs);
- namespaces.deleteNamespace(testNs.getTenant(), testNs.getCluster(), testNs.getLocalName(), false);
- List<String> nsList = Lists.newArrayList(this.testLocalNamespaces.get(1).toString(),
- this.testLocalNamespaces.get(2).toString());
- nsList.sort(null);
- assertEquals(namespaces.getTenantNamespaces(this.testTenant), nsList);
-
- testNs = this.testLocalNamespaces.get(1);
- try {
- namespaces.deleteNamespace(testNs.getTenant(), testNs.getCluster(), testNs.getLocalName(), false);
- fail("should have failed");
- } catch (RestException e) {
- // Ok
- }
-
- // delete the topic from ZK
- mockZookKeeper.delete("/managed-ledgers/" + topicName.getPersistenceNamingEncoding(), -1);
- // ensure refreshed topics list in the cache
- pulsar.getLocalZkCacheService().managedLedgerListCache().clearTree();
- // setup ownership to localhost
- doReturn(Optional.of(localWebServiceUrl)).when(nsSvc).getWebServiceUrl(testNs, false, false, false);
- doReturn(true).when(nsSvc).isServiceUnitOwned(testNs);
- namespaces.deleteNamespace(testNs.getTenant(), testNs.getCluster(), testNs.getLocalName(), false);
- }
-
- @Test
- public void testDeleteNamespaceWithBundles() throws Exception {
- URL localWebServiceUrl = new URL(pulsar.getWebServiceAddress());
- String bundledNsLocal = "test-bundled-namespace-1";
- BundlesData bundleData = new BundlesData(Lists.newArrayList("0x00000000", "0x80000000", "0xffffffff"));
- createBundledTestNamespaces(this.testTenant, this.testLocalCluster, bundledNsLocal, bundleData);
- final NamespaceName testNs = NamespaceName.get(this.testTenant, this.testLocalCluster, bundledNsLocal);
-
- org.apache.pulsar.client.admin.Namespaces namespacesAdmin = mock(org.apache.pulsar.client.admin.Namespaces.class);
- doReturn(namespacesAdmin).when(admin).namespaces();
-
- doReturn(null).when(nsSvc).getWebServiceUrl(Mockito.argThat(new Matcher<NamespaceBundle>() {
-
- @Override
- public void describeTo(Description description) {
- }
-
- @Override
- public boolean matches(Object item) {
- if (item instanceof NamespaceBundle) {
- NamespaceBundle bundle = (NamespaceBundle) item;
- return bundle.getNamespaceObject().equals(testNs);
- }
- return false;
- }
-
- @Override
- public void _dont_implement_Matcher___instead_extend_BaseMatcher_() {
- }
-
- }), Mockito.anyBoolean(), Mockito.anyBoolean(), Mockito.anyBoolean());
- doReturn(false).when(nsSvc).isServiceUnitOwned(Mockito.argThat(new Matcher<NamespaceBundle>() {
-
- @Override
- public void describeTo(Description description) {
- }
-
- @Override
- public boolean matches(Object item) {
- if (item instanceof NamespaceBundle) {
- NamespaceBundle bundle = (NamespaceBundle) item;
- return bundle.getNamespaceObject().equals(testNs);
- }
- return false;
- }
-
- @Override
- public void _dont_implement_Matcher___instead_extend_BaseMatcher_() {
- }
-
- }));
- doReturn(Optional.of(new NamespaceEphemeralData())).when(nsSvc)
- .getOwner(Mockito.argThat(new Matcher<NamespaceBundle>() {
-
- @Override
- public void describeTo(Description description) {
- }
-
- @Override
- public boolean matches(Object item) {
- if (item instanceof NamespaceBundle) {
- NamespaceBundle bundle = (NamespaceBundle) item;
- return bundle.getNamespaceObject().equals(testNs);
- }
- return false;
- }
-
- @Override
- public void _dont_implement_Matcher___instead_extend_BaseMatcher_() {
- }
- }));
-
- doThrow(new PulsarAdminException.PreconditionFailedException(
- new ClientErrorException(Status.PRECONDITION_FAILED))).when(namespacesAdmin)
- .deleteNamespaceBundle(Mockito.anyString(), Mockito.anyString());
-
- try {
- namespaces.deleteNamespaceBundle(testTenant, testLocalCluster, bundledNsLocal, "0x00000000_0x80000000",
- false);
- fail("Should have failed");
- } catch (RestException re) {
- assertEquals(re.getResponse().getStatus(), Status.PRECONDITION_FAILED.getStatusCode());
- }
-
- try {
- namespaces.deleteNamespace(testTenant, testLocalCluster, bundledNsLocal, false);
- fail("Should have failed");
- } catch (RestException re) {
- assertEquals(re.getResponse().getStatus(), Status.PRECONDITION_FAILED.getStatusCode());
- }
-
- NamespaceBundles nsBundles = nsSvc.getNamespaceBundleFactory().getBundles(testNs, bundleData);
- // make one bundle owned
- doReturn(Optional.of(localWebServiceUrl)).when(nsSvc).getWebServiceUrl(nsBundles.getBundles().get(0), false, true, false);
- doReturn(true).when(nsSvc).isServiceUnitOwned(nsBundles.getBundles().get(0));
- doNothing().when(namespacesAdmin).deleteNamespaceBundle(
- testTenant + "/" + testLocalCluster + "/" + bundledNsLocal, "0x00000000_0x80000000");
-
- try {
- namespaces.deleteNamespaceBundle(testTenant, testLocalCluster, bundledNsLocal, "0x80000000_0xffffffff",
- false);
- fail("Should have failed");
- } catch (RestException re) {
- assertEquals(re.getResponse().getStatus(), Status.PRECONDITION_FAILED.getStatusCode());
- }
-
- try {
- namespaces.deleteNamespace(testTenant, testLocalCluster, bundledNsLocal, false);
- fail("should have failed");
- } catch (RestException re) {
- assertEquals(re.getResponse().getStatus(), Status.PRECONDITION_FAILED.getStatusCode());
- }
-
- // ensure all three bundles are owned by the local broker
- for (NamespaceBundle bundle : nsBundles.getBundles()) {
- doReturn(Optional.of(localWebServiceUrl)).when(nsSvc).getWebServiceUrl(bundle, false, true, false);
- doReturn(true).when(nsSvc).isServiceUnitOwned(bundle);
- }
- doNothing().when(namespacesAdmin).deleteNamespaceBundle(Mockito.anyString(), Mockito.anyString());
- }
-
- @Test
- public void testUnloadNamespaces() throws Exception {
- final NamespaceName testNs = this.testLocalNamespaces.get(1);
- URL localWebServiceUrl = new URL(pulsar.getWebServiceAddress());
- doReturn(Optional.of(localWebServiceUrl)).when(nsSvc)
- .getWebServiceUrl(Mockito.argThat(new Matcher<NamespaceBundle>() {
-
- @Override
- public void describeTo(Description description) {
- // TODO Auto-generated method stub
-
- }
-
- @Override
- public boolean matches(Object item) {
- if (item instanceof NamespaceName) {
- NamespaceName ns = (NamespaceName) item;
- return ns.equals(testNs);
- }
- return false;
- }
-
- @Override
- public void _dont_implement_Matcher___instead_extend_BaseMatcher_() {
- // TODO Auto-generated method stub
-
- }
-
- }), Mockito.anyBoolean(), Mockito.anyBoolean(), Mockito.anyBoolean());
- doReturn(true).when(nsSvc).isServiceUnitOwned(Mockito.argThat(new Matcher<NamespaceBundle>() {
-
- @Override
- public void describeTo(Description description) {
- // TODO Auto-generated method stub
-
- }
-
- @Override
- public boolean matches(Object item) {
- if (item instanceof NamespaceName) {
- NamespaceName ns = (NamespaceName) item;
- return ns.equals(testNs);
- }
- return false;
- }
-
- @Override
- public void _dont_implement_Matcher___instead_extend_BaseMatcher_() {
- // TODO Auto-generated method stub
-
- }
-
- }));
-
- NamespaceBundle bundle = nsSvc.getNamespaceBundleFactory().getFullBundle(testNs);
- doNothing().when(namespaces).validateBundleOwnership(bundle, false, true);
-
- // The namespace unload should succeed on all the bundles
- namespaces.unloadNamespace(testNs.getTenant(), testNs.getCluster(), testNs.getLocalName());
- }
-
- @Test
- public void testSplitBundles() throws Exception {
- URL localWebServiceUrl = new URL(pulsar.getWebServiceAddress());
- String bundledNsLocal = "test-bundled-namespace-1";
- BundlesData bundleData = new BundlesData(Lists.newArrayList("0x00000000", "0xffffffff"));
- createBundledTestNamespaces(this.testTenant, this.testLocalCluster, bundledNsLocal, bundleData);
- final NamespaceName testNs = NamespaceName.get(this.testTenant, this.testLocalCluster, bundledNsLocal);
-
- OwnershipCache MockOwnershipCache = spy(pulsar.getNamespaceService().getOwnershipCache());
- doNothing().when(MockOwnershipCache).disableOwnership(any(NamespaceBundle.class));
- Field ownership = NamespaceService.class.getDeclaredField("ownershipCache");
- ownership.setAccessible(true);
- ownership.set(pulsar.getNamespaceService(), MockOwnershipCache);
- mockWebUrl(localWebServiceUrl, testNs);
-
- // split bundles
- try {
- namespaces.splitNamespaceBundle(testTenant, testLocalCluster, bundledNsLocal, "0x00000000_0xffffffff",
- false, true);
- // verify split bundles
- BundlesData bundlesData = namespaces.getBundlesData(testTenant, testLocalCluster, bundledNsLocal);
- assertNotNull(bundlesData);
- assertTrue(bundlesData.boundaries.size() == 3);
- assertTrue(bundlesData.boundaries.get(0).equals("0x00000000"));
- assertTrue(bundlesData.boundaries.get(1).equals("0x7fffffff"));
- assertTrue(bundlesData.boundaries.get(2).equals("0xffffffff"));
- } catch (RestException re) {
- assertEquals(re.getResponse().getStatus(), Status.PRECONDITION_FAILED.getStatusCode());
- }
- }
-
- @Test
- public void testSplitBundleWithUnDividedRange() throws Exception {
- URL localWebServiceUrl = new URL(pulsar.getWebServiceAddress());
- String bundledNsLocal = "test-bundled-namespace-1";
- BundlesData bundleData = new BundlesData(
- Lists.newArrayList("0x00000000", "0x08375b1a", "0x08375b1b", "0xffffffff"));
- createBundledTestNamespaces(this.testTenant, this.testLocalCluster, bundledNsLocal, bundleData);
- final NamespaceName testNs = NamespaceName.get(this.testTenant, this.testLocalCluster, bundledNsLocal);
-
- OwnershipCache MockOwnershipCache = spy(pulsar.getNamespaceService().getOwnershipCache());
- doNothing().when(MockOwnershipCache).disableOwnership(any(NamespaceBundle.class));
- Field ownership = NamespaceService.class.getDeclaredField("ownershipCache");
- ownership.setAccessible(true);
- ownership.set(pulsar.getNamespaceService(), MockOwnershipCache);
- mockWebUrl(localWebServiceUrl, testNs);
-
- // split bundles
- try {
- namespaces.splitNamespaceBundle(testTenant, testLocalCluster, bundledNsLocal, "0x08375b1a_0x08375b1b",
- false, false);
- } catch (RestException re) {
- assertEquals(re.getResponse().getStatus(), Status.PRECONDITION_FAILED.getStatusCode());
- }
- }
-
- @Test
- public void testUnloadNamespaceWithBundles() throws Exception {
- URL localWebServiceUrl = new URL(pulsar.getWebServiceAddress());
- String bundledNsLocal = "test-bundled-namespace-1";
- BundlesData bundleData = new BundlesData(Lists.newArrayList("0x00000000", "0x80000000", "0xffffffff"));
- createBundledTestNamespaces(this.testTenant, this.testLocalCluster, bundledNsLocal, bundleData);
- final NamespaceName testNs = NamespaceName.get(this.testTenant, this.testLocalCluster, bundledNsLocal);
-
- doReturn(Optional.of(localWebServiceUrl)).when(nsSvc)
- .getWebServiceUrl(Mockito.argThat(new Matcher<NamespaceBundle>() {
-
- @Override
- public void describeTo(Description description) {
- // TODO Auto-generated method stub
-
- }
-
- @Override
- public boolean matches(Object item) {
- if (item instanceof NamespaceBundle) {
- NamespaceBundle bundle = (NamespaceBundle) item;
- return bundle.getNamespaceObject().equals(testNs);
- }
- return false;
- }
-
- @Override
- public void _dont_implement_Matcher___instead_extend_BaseMatcher_() {
- // TODO Auto-generated method stub
-
- }
-
- }), Mockito.anyBoolean(), Mockito.anyBoolean(), Mockito.anyBoolean());
- doReturn(true).when(nsSvc).isServiceUnitOwned(Mockito.argThat(new Matcher<NamespaceBundle>() {
-
- @Override
- public void describeTo(Description description) {
- // TODO Auto-generated method stub
-
- }
-
- @Override
- public boolean matches(Object item) {
- if (item instanceof NamespaceBundle) {
- NamespaceBundle bundle = (NamespaceBundle) item;
- return bundle.getNamespaceObject().equals(testNs);
- }
- return false;
- }
-
- @Override
- public void _dont_implement_Matcher___instead_extend_BaseMatcher_() {
- // TODO Auto-generated method stub
-
- }
-
- }));
-
- NamespaceBundles nsBundles = nsSvc.getNamespaceBundleFactory().getBundles(testNs, bundleData);
- NamespaceBundle testBundle = nsBundles.getBundles().get(0);
- // make one bundle owned
- doReturn(Optional.of(localWebServiceUrl)).when(nsSvc).getWebServiceUrl(testBundle, false, true, false);
- doReturn(true).when(nsSvc).isServiceUnitOwned(testBundle);
- doNothing().when(nsSvc).unloadNamespaceBundle(testBundle);
- namespaces.unloadNamespaceBundle(testTenant, testLocalCluster, bundledNsLocal, "0x00000000_0x80000000",
- false);
- verify(nsSvc, times(1)).unloadNamespaceBundle(testBundle);
- try {
- namespaces.unloadNamespaceBundle(testTenant, testLocalCluster, bundledNsLocal, "0x00000000_0x88000000",
- false);
- fail("should have failed");
- } catch (RestException re) {
- // ok
- }
- }
-
- private void createBundledTestNamespaces(String property, String cluster, String namespace, BundlesData bundle)
- throws Exception {
- namespaces.createNamespace(property, cluster, namespace, bundle);
- }
-
- private void createGlobalTestNamespaces(String property, String namespace, BundlesData bundle) throws Exception {
- namespaces.createNamespace(property, "global", namespace, bundle);
- }
-
- private void createTestNamespaces(List<NamespaceName> nsnames, BundlesData bundle) throws Exception {
- for (NamespaceName nsName : nsnames) {
- namespaces.createNamespace(nsName.getTenant(), nsName.getCluster(), nsName.getLocalName(), bundle);
- }
- }
-
- @Test
- public void testValidateAdminAccessOnTenant() throws Exception {
-
- try {
- final String property = "prop";
- pulsar.getConfiguration().setAuthenticationEnabled(true);
- pulsar.getConfiguration().setAuthorizationEnabled(true);
- final String path = PulsarWebResource.path(POLICIES, property);
- final String data = ObjectMapperFactory.getThreadLocal().writeValueAsString(
- new TenantInfo(Sets.newHashSet(namespaces.clientAppId()), Sets.newHashSet("use")));
- ZkUtils.createFullPathOptimistic(pulsar.getConfigurationCache().getZooKeeper(), path, data.getBytes(),
- ZooDefs.Ids.OPEN_ACL_UNSAFE, CreateMode.PERSISTENT);
- namespaces.validateAdminAccessForTenant(property);
- } finally {
- pulsar.getConfiguration().setAuthenticationEnabled(false);
- pulsar.getConfiguration().setAuthorizationEnabled(false);
- }
- }
-
- @Test
- public void testValidateNamespaceOwnershipWithBundles() throws Exception {
- try {
- URL localWebServiceUrl = new URL(pulsar.getWebServiceAddress());
- String bundledNsLocal = "test-bundled-namespace-1";
- BundlesData bundleData = new BundlesData(Lists.newArrayList("0x00000000", "0xffffffff"));
- createBundledTestNamespaces(this.testTenant, this.testLocalCluster, bundledNsLocal, bundleData);
- final NamespaceName testNs = NamespaceName.get(this.testTenant, this.testLocalCluster, bundledNsLocal);
- mockWebUrl(localWebServiceUrl, testNs);
-
- OwnershipCache MockOwnershipCache = spy(pulsar.getNamespaceService().getOwnershipCache());
- doNothing().when(MockOwnershipCache).disableOwnership(any(NamespaceBundle.class));
- Field ownership = NamespaceService.class.getDeclaredField("ownershipCache");
- ownership.setAccessible(true);
- ownership.set(pulsar.getNamespaceService(), MockOwnershipCache);
- namespaces.validateNamespaceOwnershipWithBundles(this.testTenant, this.testLocalCluster, bundledNsLocal,
- false, true, bundleData);
- } catch (RestException e) {
- fail("ValidateNamespaceOwnershipWithBundles failed");
- }
- }
-
- @Test
- public void testRetention() throws Exception {
- try {
- URL localWebServiceUrl = new URL(pulsar.getWebServiceAddress());
- String bundledNsLocal = "test-bundled-namespace-1";
- BundlesData bundleData = new BundlesData(Lists.newArrayList("0x00000000", "0xffffffff"));
- createBundledTestNamespaces(this.testTenant, this.testLocalCluster, bundledNsLocal, bundleData);
- final NamespaceName testNs = NamespaceName.get(this.testTenant, this.testLocalCluster, bundledNsLocal);
- mockWebUrl(localWebServiceUrl, testNs);
-
- OwnershipCache MockOwnershipCache = spy(pulsar.getNamespaceService().getOwnershipCache());
- doNothing().when(MockOwnershipCache).disableOwnership(any(NamespaceBundle.class));
- Field ownership = NamespaceService.class.getDeclaredField("ownershipCache");
- ownership.setAccessible(true);
- ownership.set(pulsar.getNamespaceService(), MockOwnershipCache);
- RetentionPolicies retention = new RetentionPolicies(10, 10);
- namespaces.setRetention(this.testTenant, this.testLocalCluster, bundledNsLocal, retention);
- RetentionPolicies retention2 = namespaces.getRetention(this.testTenant, this.testLocalCluster,
- bundledNsLocal);
- assertEquals(retention, retention2);
- } catch (RestException e) {
- fail("ValidateNamespaceOwnershipWithBundles failed");
- }
- }
-
- @Test
- public void testRetentionUnauthorized() throws Exception {
- try {
- NamespaceName testNs = this.testLocalNamespaces.get(3);
- RetentionPolicies retention = new RetentionPolicies(10, 10);
- namespaces.setRetention(testNs.getTenant(), testNs.getCluster(), testNs.getLocalName(), retention);
- fail("Should fail");
- } catch (RestException e) {
- assertEquals(e.getResponse().getStatus(), Status.UNAUTHORIZED.getStatusCode());
- }
- }
-
- @Test
- public void testPersistence() throws Exception {
- NamespaceName testNs = this.testLocalNamespaces.get(0);
- PersistencePolicies persistence1 = new PersistencePolicies(3, 2, 1, 0.0);
- namespaces.setPersistence(testNs.getTenant(), testNs.getCluster(), testNs.getLocalName(), persistence1);
- PersistencePolicies persistence2 = namespaces.getPersistence(testNs.getTenant(), testNs.getCluster(),
- testNs.getLocalName());
- assertEquals(persistence2, persistence1);
- }
-
- @Test
- public void testPersistenceUnauthorized() throws Exception {
- try {
- NamespaceName testNs = this.testLocalNamespaces.get(3);
- PersistencePolicies persistence = new PersistencePolicies(3, 2, 1, 0.0);
- namespaces.setPersistence(testNs.getTenant(), testNs.getCluster(), testNs.getLocalName(), persistence);
- fail("Should fail");
- } catch (RestException e) {
- assertEquals(e.getResponse().getStatus(), Status.UNAUTHORIZED.getStatusCode());
- }
- }
-
- @Test
- public void testValidateTopicOwnership() throws Exception {
- URL localWebServiceUrl = new URL(pulsar.getWebServiceAddress());
- String bundledNsLocal = "test-bundled-namespace-1";
- BundlesData bundleData = new BundlesData(Lists.newArrayList("0x00000000", "0xffffffff"));
- createBundledTestNamespaces(this.testTenant, this.testLocalCluster, bundledNsLocal, bundleData);
- final NamespaceName testNs = NamespaceName.get(this.testTenant, this.testLocalCluster, bundledNsLocal);
- OwnershipCache MockOwnershipCache = spy(pulsar.getNamespaceService().getOwnershipCache());
- doNothing().when(MockOwnershipCache).disableOwnership(any(NamespaceBundle.class));
- Field ownership = NamespaceService.class.getDeclaredField("ownershipCache");
- ownership.setAccessible(true);
- ownership.set(pulsar.getNamespaceService(), MockOwnershipCache);
- TopicName topicName = TopicName.get(testNs.getPersistentTopicName("my-topic"));
- PersistentTopics topics = spy(new PersistentTopics());
- topics.setServletContext(new MockServletContext());
- topics.setPulsar(pulsar);
- doReturn(false).when(topics).isRequestHttps();
- doReturn("test").when(topics).clientAppId();
- doReturn(null).when(topics).originalPrincipal();
- mockWebUrl(localWebServiceUrl, testNs);
- doReturn("persistent").when(topics).domain();
-
- topics.validateTopicName(topicName.getTenant(), topicName.getCluster(),
- topicName.getNamespacePortion(), topicName.getEncodedLocalName());
- topics.validateAdminOperationOnTopic(false);
- }
-
- @Test
- public void testIsLeader() throws Exception {
- assertTrue(namespaces.isLeaderBroker());
- }
-
- /**
- * Verifies that deleteNamespace cleans up policies(global,local), bundle cache and bundle ownership
- *
- * @throws Exception
- */
- @Test
- public void testDeleteNamespace() throws Exception {
-
- final String namespace = this.testTenant + "/use/deleteNs";
- admin.namespaces().createNamespace(namespace, 100);
- assertEquals(admin.namespaces().getPolicies(namespace).bundles.numBundles, 100);
-
- // (1) Force topic creation and namespace being loaded
- final String topicName = "persistent://" + namespace + "/my-topic";
- TopicName topic = TopicName.get(topicName);
-
- Producer producer = pulsarClient.createProducer(topicName);
- producer.close();
- NamespaceBundle bundle1 = pulsar.getNamespaceService().getBundle(topic);
- // (2) Delete topic
- admin.topics().delete(topicName);
- // (3) Delete ns
- admin.namespaces().deleteNamespace(namespace);
- // (4) check bundle
- NamespaceBundle bundle2 = pulsar.getNamespaceService().getBundle(topic);
- assertNotEquals(bundle1.getBundleRange(), bundle2.getBundleRange());
- // returns full bundle if policies not present
- assertEquals("0x00000000_0xffffffff", bundle2.getBundleRange());
-
- }
-
- private void mockWebUrl(URL localWebServiceUrl, NamespaceName namespace) throws Exception {
- doReturn(Optional.of(localWebServiceUrl)).when(nsSvc)
- .getWebServiceUrl(Mockito.argThat(new Matcher<NamespaceBundle>() {
- @Override
- public void describeTo(Description description) {
- }
-
- @Override
- public boolean matches(Object item) {
- if (item instanceof NamespaceBundle) {
- NamespaceBundle bundle = (NamespaceBundle) item;
- return bundle.getNamespaceObject().equals(namespace);
- }
- return false;
- }
-
- @Override
- public void _dont_implement_Matcher___instead_extend_BaseMatcher_() {
- }
- }), Mockito.anyBoolean(), Mockito.anyBoolean(), Mockito.anyBoolean());
- doReturn(true).when(nsSvc).isServiceUnitOwned(Mockito.argThat(new Matcher<NamespaceBundle>() {
- @Override
- public void describeTo(Description description) {
- }
-
- @Override
- public boolean matches(Object item) {
- if (item instanceof NamespaceBundle) {
- NamespaceBundle bundle = (NamespaceBundle) item;
- return bundle.getNamespaceObject().equals(namespace);
- }
- return false;
- }
-
- @Override
- public void _dont_implement_Matcher___instead_extend_BaseMatcher_() {
- }
- }));
- }
-}
diff --git a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v1/V1_AdminApiTest.java b/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v1/V1_AdminApiTest.java
deleted file mode 100644
index 803f8af028..0000000000
--- a/pulsar-broker/src/test/java/org/apache/pulsar/broker/admin/v1/V1_AdminApiTest.java
+++ /dev/null
@@ -1,2007 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.pulsar.broker.admin.v1;
-
-import static org.mockito.Mockito.doReturn;
-import static org.mockito.Mockito.spy;
-import static org.mockito.Mockito.times;
-import static org.mockito.Mockito.verify;
-import static org.testng.Assert.assertEquals;
-import static org.testng.Assert.assertFalse;
-import static org.testng.Assert.assertNotEquals;
-import static org.testng.Assert.assertNotNull;
-import static org.testng.Assert.assertTrue;
-import static org.testng.Assert.fail;
-
-import java.lang.reflect.Field;
-import java.net.URL;
-import java.util.ArrayList;
-import java.util.Arrays;
-import java.util.EnumSet;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Optional;
-import java.util.Set;
-import java.util.concurrent.CompletableFuture;
-import java.util.concurrent.ExecutorService;
-import java.util.concurrent.Executors;
-import java.util.concurrent.TimeUnit;
-
-import javax.ws.rs.client.InvocationCallback;
-import javax.ws.rs.client.WebTarget;
-
-import org.apache.pulsar.broker.PulsarServerException;
-import org.apache.pulsar.broker.PulsarService;
-import org.apache.pulsar.broker.ServiceConfiguration;
-import org.apache.pulsar.broker.auth.MockedPulsarServiceBaseTest;
-import org.apache.pulsar.broker.namespace.NamespaceEphemeralData;
-import org.apache.pulsar.broker.namespace.NamespaceService;
-import org.apache.pulsar.broker.service.BrokerService;
-import org.apache.pulsar.client.admin.LongRunningProcessStatus;
-import org.apache.pulsar.client.admin.PulsarAdmin;
-import org.apache.pulsar.client.admin.PulsarAdminException;
-import org.apache.pulsar.client.admin.PulsarAdminException.ConflictException;
-import org.apache.pulsar.client.admin.PulsarAdminException.NotAuthorizedException;
-import org.apache.pulsar.client.admin.PulsarAdminException.NotFoundException;
-import org.apache.pulsar.client.admin.PulsarAdminException.PreconditionFailedException;
-import org.apache.pulsar.client.admin.internal.LookupImpl;
-import org.apache.pulsar.client.admin.internal.TopicsImpl;
-import org.apache.pulsar.client.admin.internal.TenantsImpl;
-import org.apache.pulsar.client.api.Consumer;
-import org.apache.pulsar.client.api.ConsumerBuilder;
-import org.apache.pulsar.client.api.Message;
-import org.apache.pulsar.client.api.MessageRoutingMode;
-import org.apache.pulsar.client.api.Producer;
-import org.apache.pulsar.client.api.PulsarClient;
-import org.apache.pulsar.client.api.Schema;
-import org.apache.pulsar.client.api.SubscriptionType;
-import org.apache.pulsar.common.lookup.data.LookupData;
-import org.apache.pulsar.common.naming.NamespaceBundle;
-import org.apache.pulsar.common.naming.NamespaceBundleFactory;
-import org.apache.pulsar.common.naming.NamespaceBundles;
-import org.apache.pulsar.common.naming.NamespaceName;
-import org.apache.pulsar.common.naming.TopicDomain;
-import org.apache.pulsar.common.naming.TopicName;
-import org.apache.pulsar.common.partition.PartitionedTopicMetadata;
-import org.apache.pulsar.common.policies.data.AuthAction;
-import org.apache.pulsar.common.policies.data.AutoFailoverPolicyData;
-import org.apache.pulsar.common.policies.data.AutoFailoverPolicyType;
-import org.apache.pulsar.common.policies.data.BacklogQuota;
-import org.apache.pulsar.common.policies.data.BacklogQuota.BacklogQuotaType;
-import org.apache.pulsar.common.policies.data.BacklogQuota.RetentionPolicy;
-import org.apache.pulsar.common.policies.data.BrokerAssignment;
-import org.apache.pulsar.common.policies.data.ClusterData;
-import org.apache.pulsar.common.policies.data.NamespaceIsolationData;
-import org.apache.pulsar.common.policies.data.NamespaceOwnershipStatus;
-import org.apache.pulsar.common.policies.data.PartitionedTopicStats;
-import org.apache.pulsar.common.policies.data.PersistencePolicies;
-import org.apache.pulsar.common.policies.data.PersistentTopicInternalStats;
-import org.apache.pulsar.common.policies.data.TopicStats;
-import org.apache.pulsar.common.policies.data.Policies;
-import org.apache.pulsar.common.policies.data.TenantInfo;
-import org.apache.pulsar.common.policies.data.RetentionPolicies;
-import org.apache.pulsar.common.util.Codec;
-import org.apache.pulsar.common.util.ObjectMapperFactory;
-import org.apache.pulsar.compaction.Compactor;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.testng.Assert;
-import org.testng.annotations.AfterMethod;
-import org.testng.annotations.BeforeMethod;
-import org.testng.annotations.DataProvider;
-import org.testng.annotations.Test;
-
-import com.google.common.collect.BoundType;
-import com.google.common.collect.Lists;
-import com.google.common.collect.Maps;
-import com.google.common.collect.Range;
-import com.google.common.collect.Sets;
-import com.google.common.hash.Hashing;
-
-import lombok.extern.slf4j.Slf4j;
-
-@Slf4j
-public class V1_AdminApiTest extends MockedPulsarServiceBaseTest {
-
- private static final Logger LOG = LoggerFactory.getLogger(V1_AdminApiTest.class);
-
- private final String TLS_SERVER_CERT_FILE_PATH = "./src/test/resources/certificate/server.crt";
- private final String TLS_SERVER_KEY_FILE_PATH = "./src/test/resources/certificate/server.key";
-
- private MockedPulsarService mockPulsarSetup;
-
- private PulsarService otherPulsar;
-
- private PulsarAdmin adminTls;
- private PulsarAdmin otheradmin;
-
- private NamespaceBundleFactory bundleFactory;
-
- @BeforeMethod
- @Override
- public void setup() throws Exception {
- conf.setLoadBalancerEnabled(true);
- conf.setTlsEnabled(true);
- conf.setTlsCertificateFilePath(TLS_SERVER_CERT_FILE_PATH);
- conf.setTlsKeyFilePath(TLS_SERVER_KEY_FILE_PATH);
-
- super.internalSetup();
-
- bundleFactory = new NamespaceBundleFactory(pulsar, Hashing.crc32());
-
- adminTls = spy(PulsarAdmin.builder().tlsTrustCertsFilePath(TLS_SERVER_CERT_FILE_PATH)
- .serviceHttpUrl(brokerUrlTls.toString()).build());
-
- // create otherbroker to test redirect on calls that need
- // namespace ownership
- mockPulsarSetup = new MockedPulsarService(this.conf);
- mockPulsarSetup.setup();
- otherPulsar = mockPulsarSetup.getPulsar();
- otheradmin = mockPulsarSetup.getAdmin();
-
- // Setup namespaces
- admin.clusters().createCluster("use", new ClusterData("http://127.0.0.1" + ":" + BROKER_WEBSERVICE_PORT));
- TenantInfo tenantInfo = new TenantInfo(Sets.newHashSet("role1", "role2"), Sets.newHashSet("use"));
- admin.tenants().createTenant("prop-xyz", tenantInfo);
- admin.namespaces().createNamespace("prop-xyz/use/ns1");
- }
-
- @AfterMethod
- @Override
- public void cleanup() throws Exception {
- adminTls.close();
- super.internalCleanup();
- mockPulsarSetup.cleanup();
- }
-
- @DataProvider(name = "numBundles")
- public static Object[][] numBundles() {
- return new Object[][] { { 1 }, { 4 } };
- }
-
- @DataProvider(name = "bundling")
- public static Object[][] bundling() {
- return new Object[][] { { 0 }, { 4 } };
- }
-
- @DataProvider(name = "topicName")
- public Object[][] topicNamesProvider() {
- return new Object[][] { { "topic_+&*%{}() \\/$@#^%" }, { "simple-topicName" } };
- }
-
- @DataProvider(name = "topicType")
- public Object[][] topicTypeProvider() {
- return new Object[][] { { TopicDomain.persistent.value() }, { TopicDomain.non_persistent.value() } };
- }
-
- @Test
- public void clusters() throws Exception {
- admin.clusters().createCluster("usw",
- new ClusterData("http://broker.messaging.use.example.com" + ":" + BROKER_WEBSERVICE_PORT));
- // "test" cluster is part of config-default cluster and it's znode gets created when PulsarService creates
- // failure-domain znode of this default cluster
- assertEquals(admin.clusters().getClusters(), Lists.newArrayList("use", "usw"));
-
- assertEquals(admin.clusters().getCluster("use"),
- new ClusterData("http://127.0.0.1" + ":" + BROKER_WEBSERVICE_PORT));
-
- admin.clusters().updateCluster("usw",
- new ClusterData("http://new-broker.messaging.usw.example.com" + ":" + BROKER_WEBSERVICE_PORT));
- assertEquals(admin.clusters().getClusters(), Lists.newArrayList("use", "usw"));
- assertEquals(admin.clusters().getCluster("usw"),
- new ClusterData("http://new-broker.messaging.usw.example.com" + ":" + BROKER_WEBSERVICE_PORT));
-
- admin.clusters().updateCluster("usw",
- new ClusterData("http://new-broker.messaging.usw.example.com" + ":" + BROKER_WEBSERVICE_PORT,
- "https://new-broker.messaging.usw.example.com" + ":" + BROKER_WEBSERVICE_PORT_TLS));
- assertEquals(admin.clusters().getClusters(), Lists.newArrayList("use", "usw"));
- assertEquals(admin.clusters().getCluster("usw"),
- new ClusterData("http://new-broker.messaging.usw.example.com" + ":" + BROKER_WEBSERVICE_PORT,
- "https://new-broker.messaging.usw.example.com" + ":" + BROKER_WEBSERVICE_PORT_TLS));
-
- admin.clusters().deleteCluster("usw");
- Thread.sleep(300);
-
- assertEquals(admin.clusters().getClusters(), Lists.newArrayList("use"));
-
- admin.namespaces().deleteNamespace("prop-xyz/use/ns1");
- admin.clusters().deleteCluster("use");
- assertEquals(admin.clusters().getClusters(), Lists.newArrayList());
-
- // Check name validation
- try {
- admin.clusters().createCluster("bf!", new ClusterData("http://dummy.messaging.example.com"));
- fail("should have failed");
- } catch (PulsarAdminException e) {
- assertTrue(e instanceof PreconditionFailedException);
- }
- }
-
- @Test
- public void clusterNamespaceIsolationPolicies() throws PulsarAdminException {
- try {
- // create
- String policyName1 = "policy-1";
- NamespaceIsolationData nsPolicyData1 = new NamespaceIsolationData();
- nsPolicyData1.namespaces = new ArrayList<String>();
- nsPolicyData1.namespaces.add("other/use/other.*");
- nsPolicyData1.primary = new ArrayList<String>();
- nsPolicyData1.primary.add("prod1-broker[4-6].messaging.use.example.com");
- nsPolicyData1.secondary = new ArrayList<String>();
- nsPolicyData1.secondary.add("prod1-broker.*.messaging.use.example.com");
- nsPolicyData1.auto_failover_policy = new AutoFailoverPolicyData();
- nsPolicyData1.auto_failover_policy.policy_type = AutoFailoverPolicyType.min_available;
- nsPolicyData1.auto_failover_policy.parameters = new HashMap<String, String>();
- nsPolicyData1.auto_failover_policy.parameters.put("min_limit", "1");
- nsPolicyData1.auto_failover_policy.parameters.put("usage_threshold", "100");
- admin.clusters().createNamespaceIsolationPolicy("use", policyName1, nsPolicyData1);
-
- String policyName2 = "policy-2";
- NamespaceIsolationData nsPolicyData2 = new NamespaceIsolationData();
- nsPolicyData2.namespaces = new ArrayList<String>();
- nsPolicyData2.namespaces.add("other/use/other.*");
- nsPolicyData2.primary = new ArrayList<String>();
- nsPolicyData2.primary.add("prod1-broker[4-6].messaging.use.example.com");
- nsPolicyData2.secondary = new ArrayList<String>();
- nsPolicyData2.secondary.add("prod1-broker.*.messaging.use.example.com");
- nsPolicyData2.auto_failover_policy = new AutoFailoverPolicyData();
- nsPolicyData2.auto_failover_policy.policy_type = AutoFailoverPolicyType.min_available;
- nsPolicyData2.auto_failover_policy.parameters = new HashMap<String, String>();
- nsPolicyData2.auto_failover_policy.parameters.put("min_limit", "1");
- nsPolicyData2.auto_failover_policy.parameters.put("usage_threshold", "100");
- admin.clusters().createNamespaceIsolationPolicy("use", policyName2, nsPolicyData2);
-
- // verify create indirectly with get
- Map<String, NamespaceIsolationData> policiesMap = admin.clusters().getNamespaceIsolationPolicies("use");
- assertEquals(policiesMap.get(policyName1), nsPolicyData1);
- assertEquals(policiesMap.get(policyName2), nsPolicyData2);
-
- // verify update of primary
- nsPolicyData1.primary.remove(0);
- nsPolicyData1.primary.add("prod1-broker[1-2].messaging.use.example.com");
- admin.clusters().updateNamespaceIsolationPolicy("use", policyName1, nsPolicyData1);
-
- // verify primary change
- policiesMap = admin.clusters().getNamespaceIsolationPolicies("use");
- assertEquals(policiesMap.get(policyName1), nsPolicyData1);
-
- // verify update of secondary
- nsPolicyData1.secondary.remove(0);
- nsPolicyData1.secondary.add("prod1-broker[3-4].messaging.use.example.com");
- admin.clusters().updateNamespaceIsolationPolicy("use", policyName1, nsPolicyData1);
-
- // verify secondary change
- policiesMap = admin.clusters().getNamespaceIsolationPolicies("use");
- assertEquals(policiesMap.get(policyName1), nsPolicyData1);
-
- // verify update of failover policy limit
- nsPolicyData1.auto_failover_policy.parameters.put("min_limit", "10");
- admin.clusters().updateNamespaceIsolationPolicy("use", policyName1, nsPolicyData1);
-
- // verify min_limit change
- policiesMap = admin.clusters().getNamespaceIsolationPolicies("use");
- assertEquals(policiesMap.get(policyName1), nsPolicyData1);
-
- // verify update of failover usage_threshold limit
- nsPolicyData1.auto_failover_policy.parameters.put("usage_threshold", "80");
- admin.clusters().updateNamespaceIsolationPolicy("use", policyName1, nsPolicyData1);
-
- // verify usage_threshold change
- policiesMap = admin.clusters().getNamespaceIsolationPolicies("use");
- assertEquals(policiesMap.get(policyName1), nsPolicyData1);
-
- // verify single get
- NamespaceIsolationData policy1Data = admin.clusters().getNamespaceIsolationPolicy("use", policyName1);
- assertEquals(policy1Data, nsPolicyData1);
-
- // verify creation of more than one policy
- admin.clusters().createNamespaceIsolationPolicy("use", policyName2, nsPolicyData1);
-
- try {
- admin.clusters().getNamespaceIsolationPolicy("use", "no-such-policy");
- fail("should have raised exception");
- } catch (PulsarAdminException e) {
- assertTrue(e instanceof NotFoundException);
- }
-
- // verify delete cluster failed
- try {
- admin.clusters().deleteCluster("use");
- fail("should have raised exception");
- } catch (PulsarAdminException e) {
- assertTrue(e instanceof PreconditionFailedException);
- }
-
- // verify delete
- admin.clusters().deleteNamespaceIsolationPolicy("use", policyName1);
- admin.clusters().deleteNamespaceIsolationPolicy("use", policyName2);
-
- try {
- admin.clusters().getNamespaceIsolationPolicy("use", policyName1);
- fail("should have raised exception");
- } catch (PulsarAdminException e) {
- assertTrue(e instanceof NotFoundException);
- }
-
- try {
- admin.clusters().getNamespaceIsolationPolicy("use", policyName2);
- fail("should have raised exception");
- } catch (PulsarAdminException e) {
- assertTrue(e instanceof NotFoundException);
- }
-
- try {
- admin.clusters().getNamespaceIsolationPolicies("usc");
- fail("should have raised exception");
- } catch (PulsarAdminException e) {
- assertTrue(e instanceof NotFoundException);
- }
-
- try {
- admin.clusters().getNamespaceIsolationPolicy("usc", "no-such-cluster");
- fail("should have raised exception");
- } catch (PulsarAdminException e) {
- assertTrue(e instanceof PreconditionFailedException);
- }
-
- try {
- admin.clusters().createNamespaceIsolationPolicy("usc", "no-such-cluster", nsPolicyData1);
- fail("should have raised exception");
- } catch (PulsarAdminException e) {
- assertTrue(e instanceof PreconditionFailedException);
- }
-
- try {
- admin.clusters().updateNamespaceIsolationPolicy("usc", "no-such-cluster", policy1Data);
- fail("should have raised exception");
- } catch (PulsarAdminException e) {
- assertTrue(e instanceof PreconditionFailedException);
- }
-
- } catch (PulsarAdminException e) {
- LOG.warn("TEST FAILED [{}]", e.getMessage());
- throw e;
- }
- }
-
- @Test
- public void brokers() throws Exception {
- List<String> list = admin.brokers().getActiveBrokers("use");
- Assert.assertNotNull(list);
- Assert.assertEquals(list.size(), 1);
-
- List<String> list2 = otheradmin.brokers().getActiveBrokers("test");
- Assert.assertNotNull(list2);
- Assert.assertEquals(list2.size(), 1);
-
- Map<String, NamespaceOwnershipStatus> nsMap = admin.brokers().getOwnedNamespaces("use", list.get(0));
- // since sla-monitor ns is not created nsMap.size() == 1 (for HeartBeat Namespace)
- Assert.assertEquals(1, nsMap.size());
- for (String ns : nsMap.keySet()) {
- NamespaceOwnershipStatus nsStatus = nsMap.get(ns);
- if (ns.equals(
- NamespaceService.getHeartbeatNamespace(pulsar.getAdvertisedAddress(), pulsar.getConfiguration())
- + "/0x00000000_0xffffffff")) {
- assertEquals(nsStatus.broker_assignment, BrokerAssignment.shared);
- assertFalse(nsStatus.is_controlled);
- assertTrue(nsStatus.is_active);
- }
- }
-
- String[] parts = list.get(0).split(":");
- Assert.assertEquals(parts.length, 2);
- Map<String, NamespaceOwnershipStatus> nsMap2 = adminTls.brokers().getOwnedNamespaces("use",
- String.format("%s:%d", parts[0], BROKER_WEBSERVICE_PORT_TLS));
- Assert.assertEquals(nsMap2.size(), 1);
-
- admin.namespaces().deleteNamespace("prop-xyz/use/ns1");
- admin.clusters().deleteCluster("use");
- assertEquals(admin.clusters().getClusters(), Lists.newArrayList());
- }
-
- /**
- * <pre>
- * Verifies: zk-update configuration updates service-config
- * 1. create znode for dynamic-config
- * 2. start pulsar service so, pulsar can set the watch on that znode
- * 3. update the configuration with new value
- * 4. wait and verify that new value has been updated
- * </pre>
- *
- * @throws Exception
- */
- @Test
- public void testUpdateDynamicConfigurationWithZkWatch() throws Exception {
- final int initValue = 30000;
- pulsar.getConfiguration().setBrokerShutdownTimeoutMs(initValue);
- // (1) try to update dynamic field
- final long shutdownTime = 10;
- // update configuration
- admin.brokers().updateDynamicConfiguration("brokerShutdownTimeoutMs", Long.toString(shutdownTime));
- // sleep incrementally as zk-watch notification is async and may take some time
- for (int i = 0; i < 5; i++) {
- if (pulsar.getConfiguration().getBrokerShutdownTimeoutMs() != initValue) {
- Thread.sleep(50 + (i * 10));
- }
- }
- // wait config to be updated
- for (int i = 0; i < 5; i++) {
- if (pulsar.getConfiguration().getBrokerShutdownTimeoutMs() != shutdownTime) {
- Thread.sleep(100 + (i * 10));
- } else {
- break;
- }
- }
- // verify value is updated
- assertEquals(pulsar.getConfiguration().getBrokerShutdownTimeoutMs(), shutdownTime);
-
- // (2) try to update non-dynamic field
- try {
- admin.brokers().updateDynamicConfiguration("zookeeperServers", "test-zk:1234");
- } catch (Exception e) {
- assertTrue(e instanceof PreconditionFailedException);
- }
-
- // (3) try to update non-existent field
- try {
- admin.brokers().updateDynamicConfiguration("test", Long.toString(shutdownTime));
- } catch (Exception e) {
- assertTrue(e instanceof PreconditionFailedException);
- }
-
- }
-
- /**
- * Verifies broker sets watch on dynamic-configuration map even with invalid init json data
- *
- * <pre>
- * 1. Set invalid json at dynamic-config znode
- * 2. Broker fails to deserialize znode content but sets the watch on znode
- * 3. Update znode with valid json map
- * 4. Broker should get watch and update the dynamic-config map
- * </pre>
- *
- * @throws Exception
- */
- @Test
- public void testInvalidDynamicConfigContentInZK() throws Exception {
- final int newValue = 10;
- stopBroker();
- // set invalid data into dynamic-config znode so, broker startup fail to deserialize data
- mockZookKeeper.setData(BrokerService.BROKER_SERVICE_CONFIGURATION_PATH, "$".getBytes(), -1);
- // start broker: it should have set watch even if with failure of deserialization
- startBroker();
- Assert.assertNotEquals(pulsar.getConfiguration().getBrokerShutdownTimeoutMs(), newValue);
- // update zk with config-value which should fire watch and broker should update the config value
- Map<String, String> configMap = Maps.newHashMap();
- configMap.put("brokerShutdownTimeoutMs", Integer.toString(newValue));
- mockZookKeeper.setData(BrokerService.BROKER_SERVICE_CONFIGURATION_PATH,
- ObjectMapperFactory.getThreadLocal().writeValueAsBytes(configMap), -1);
- // wait config to be updated
- for (int i = 0; i < 5; i++) {
- if (pulsar.getConfiguration().getBrokerShutdownTimeoutMs() != newValue) {
- Thread.sleep(100 + (i * 10));
- } else {
- break;
- }
- }
- // verify value is updated
- assertEquals(pulsar.getConfiguration().getBrokerShutdownTimeoutMs(), newValue);
- }
-
- /**
- * <pre>
- * verifies: that registerListener updates pulsar.config value with newly updated zk-dynamic config
- * 1.start pulsar
- * 2.update zk-config with admin api
- * 3. trigger watch and listener
- * 4. verify that config is updated
- * </pre>
- *
- * @throws Exception
- */
- @Test
- public void testUpdateDynamicLocalConfiguration() throws Exception {
- // (1) try to update dynamic field
- final long initValue = 30000;
- final long shutdownTime = 10;
- pulsar.getConfiguration().setBrokerShutdownTimeoutMs(initValue);
- // update configuration
- admin.brokers().updateDynamicConfiguration("brokerShutdownTimeoutMs", Long.toString(shutdownTime));
- // sleep incrementally as zk-watch notification is async and may take some time
- for (int i = 0; i < 5; i++) {
- if (pulsar.getConfiguration().getBrokerShutdownTimeoutMs() == initValue) {
- Thread.sleep(50 + (i * 10));
- }
- }
-
- // verify value is updated
- assertEquals(pulsar.getConfiguration().getBrokerShutdownTimeoutMs(), shutdownTime);
- }
-
- @Test
- public void testUpdatableConfigurationName() throws Exception {
- // (1) try to update dynamic field
- final String configName = "brokerShutdownTimeoutMs";
- assertTrue(admin.brokers().getDynamicConfigurationNames().contains(configName));
- }
-
- @Test
- public void testGetDynamicLocalConfiguration() throws Exception {
- // (1) try to update dynamic field
- final String configName = "brokerShutdownTimeoutMs";
- final long shutdownTime = 10;
- pulsar.getConfiguration().setBrokerShutdownTimeoutMs(30000);
- Map<String, String> configs = admin.brokers().getAllDynamicConfigurations();
- assertTrue(configs.isEmpty());
- assertNotEquals(pulsar.getConfiguration().getBrokerShutdownTimeoutMs(), shutdownTime);
- // update configuration
- admin.brokers().updateDynamicConfiguration(configName, Long.toString(shutdownTime));
- // Now, znode is created: updateConfigurationAndregisterListeners and check if configuration updated
- assertEquals(Long.parseLong(admin.brokers().getAllDynamicConfigurations().get(configName)), shutdownTime);
- }
-
- @Test(enabled = true)
- public void properties() throws PulsarAdminException {
- Set<String> allowedClusters = Sets.newHashSet("use");
- TenantInfo tenantInfo = new TenantInfo(Sets.newHashSet("role1", "role2"), allowedClusters);
- admin.tenants().updateTenant("prop-xyz", tenantInfo);
-
- assertEquals(admin.tenants().getTenants(), Lists.newArrayList("prop-xyz"));
-
- assertEquals(admin.tenants().getTenantInfo("prop-xyz"), tenantInfo);
-
- TenantInfo newPropertyAdmin = new TenantInfo(Sets.newHashSet("role3", "role4"), allowedClusters);
- admin.tenants().updateTenant("prop-xyz", newPropertyAdmin);
-
- assertEquals(admin.tenants().getTenantInfo("prop-xyz"), newPropertyAdmin);
-
- admin.namespaces().deleteNamespace("prop-xyz/use/ns1");
- admin.tenants().deleteTenant("prop-xyz");
- assertEquals(admin.tenants().getTenants(), Lists.newArrayList());
-
- // Check name validation
- try {
- admin.tenants().createTenant("prop-xyz&", tenantInfo);
- fail("should have failed");
- } catch (PulsarAdminException e) {
- assertTrue(e instanceof PreconditionFailedException);
- }
- }
-
- @Test(invocationCount = 1)
- public void namespaces() throws PulsarAdminException, PulsarServerException, Exception {
- admin.clusters().createCluster("usw", new ClusterData());
- TenantInfo tenantInfo = new TenantInfo(Sets.newHashSet("role1", "role2"),
- Sets.newHashSet("use", "usw"));
- admin.tenants().updateTenant("prop-xyz", tenantInfo);
-
- assertEquals(admin.namespaces().getPolicies("prop-xyz/use/ns1").bundles, Policies.defaultBundle());
-
- admin.namespaces().createNamespace("prop-xyz/use/ns2");
-
- admin.namespaces().createNamespace("prop-xyz/use/ns3", 4);
- assertEquals(admin.namespaces().getPolicies("prop-xyz/use/ns3").bundles.numBundles, 4);
- assertEquals(admin.namespaces().getPolicies("prop-xyz/use/ns3").bundles.boundaries.size(), 5);
-
- admin.namespaces().deleteNamespace("prop-xyz/use/ns3");
-
- try {
- admin.namespaces().createNamespace("non-existing/usw/ns1");
- fail("Should not have passed");
- } catch (NotFoundException e) {
- // Ok
- }
-
- assertEquals(admin.namespaces().getNamespaces("prop-xyz"),
- Lists.newArrayList("prop-xyz/use/ns1", "prop-xyz/use/ns2"));
- assertEquals(admin.namespaces().getNamespaces("prop-xyz", "use"),
- Lists.newArrayList("prop-xyz/use/ns1", "prop-xyz/use/ns2"));
-
- try {
- admin.namespaces().createNamespace("prop-xyz/usc/ns1");
- fail("Should not have passed");
- } catch (NotAuthorizedException e) {
- // Ok, got the non authorized exception since usc cluster is not in the allowed clusters list.
- }
-
- admin.namespaces().grantPermissionOnNamespace("prop-xyz/use/ns1", "my-role", EnumSet.allOf(AuthAction.class));
-
- Policies policies = new Policies();
- policies.bundles = Policies.defaultBundle();
- policies.auth_policies.namespace_auth.put("my-role", EnumSet.allOf(AuthAction.class));
-
- assertEquals(admin.namespaces().getPolicies("prop-xyz/use/ns1"), policies);
- assertEquals(admin.namespaces().getPermissions("prop-xyz/use/ns1"), policies.auth_policies.namespace_auth);
-
- assertEquals(admin.namespaces().getTopics("prop-xyz/use/ns1"), Lists.newArrayList());
-
- admin.namespaces().revokePermissionsOnNamespace("prop-xyz/use/ns1", "my-role");
- policies.auth_policies.namespace_auth.remove("my-role");
- assertEquals(admin.namespaces().getPolicies("prop-xyz/use/ns1"), policies);
-
- assertEquals(admin.namespaces().getPersistence("prop-xyz/use/ns1"), new PersistencePolicies(1, 1, 1, 0.0));
- admin.namespaces().setPersistence("prop-xyz/use/ns1", new PersistencePolicies(3, 2, 1, 10.0));
- assertEquals(admin.namespaces().getPersistence("prop-xyz/use/ns1"), new PersistencePolicies(3, 2, 1, 10.0));
-
- // Force topic creation and namespace being loaded
- Producer<byte[]> producer = pulsarClient.newProducer(Schema.BYTES).topic("persistent://prop-xyz/use/ns1/my-topic").create();
- producer.close();
- admin.topics().delete("persistent://prop-xyz/use/ns1/my-topic");
-
- admin.namespaces().unloadNamespaceBundle("prop-xyz/use/ns1", "0x00000000_0xffffffff");
- NamespaceName ns = NamespaceName.get("prop-xyz/use/ns1");
- // Now, w/ bundle policies, we will use default bundle
- NamespaceBundle defaultBundle = bundleFactory.getFullBundle(ns);
- int i = 0;
- for (; i < 10; i++) {
- Optional<NamespaceEphemeralData> data1 = pulsar.getNamespaceService().getOwnershipCache()
- .getOwnerAsync(defaultBundle).get();
- if (!data1.isPresent()) {
- // Already unloaded
- break;
- }
- LOG.info("Waiting for unload namespace {} to complete. Current service unit isDisabled: {}", defaultBundle,
- data1.get().isDisabled());
- Thread.sleep(1000);
- }
- assertTrue(i < 10);
-
- admin.namespaces().deleteNamespace("prop-xyz/use/ns1");
- assertEquals(admin.namespaces().getNamespaces("prop-xyz", "use"), Lists.newArrayList("prop-xyz/use/ns2"));
-
- try {
(This diff was longer than 20,000 lines, and has been truncated...)
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services