You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@storm.apache.org by pt...@apache.org on 2017/11/20 19:00:47 UTC
[01/50] [abbrv] storm git commit: STORM-2764: Only login the user if
not already done
Repository: storm
Updated Branches:
refs/heads/metrics_v2 a47b5fbda -> 99bcf6844
STORM-2764: Only login the user if not already done
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/26ff239b
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/26ff239b
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/26ff239b
Branch: refs/heads/metrics_v2
Commit: 26ff239b49893ff72da64ebed9c20e453d327248
Parents: 892bf5c
Author: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Authored: Thu Sep 28 14:42:36 2017 -0500
Committer: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Committed: Fri Sep 29 09:38:48 2017 -0500
----------------------------------------------------------------------
.../storm/hdfs/blobstore/HdfsBlobStore.java | 107 ++++++++++---------
1 file changed, 59 insertions(+), 48 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/26ff239b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/blobstore/HdfsBlobStore.java
----------------------------------------------------------------------
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/blobstore/HdfsBlobStore.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/blobstore/HdfsBlobStore.java
index dadabe8..05e55bd 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/blobstore/HdfsBlobStore.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/blobstore/HdfsBlobStore.java
@@ -15,8 +15,22 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
+
package org.apache.storm.hdfs.blobstore;
+import java.io.ByteArrayOutputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.InputStream;
+import java.security.AccessController;
+import java.security.PrivilegedAction;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.Map;
+import javax.security.auth.Subject;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.storm.Config;
import org.apache.storm.blobstore.AtomicOutputStream;
import org.apache.storm.blobstore.BlobStore;
@@ -24,28 +38,15 @@ import org.apache.storm.blobstore.BlobStoreAclHandler;
import org.apache.storm.blobstore.BlobStoreFile;
import org.apache.storm.blobstore.InputStreamWithMeta;
import org.apache.storm.generated.AuthorizationException;
-import org.apache.storm.generated.KeyNotFoundException;
import org.apache.storm.generated.KeyAlreadyExistsException;
+import org.apache.storm.generated.KeyNotFoundException;
import org.apache.storm.generated.ReadableBlobMeta;
import org.apache.storm.generated.SettableBlobMeta;
import org.apache.storm.nimbus.NimbusInfo;
import org.apache.storm.utils.Utils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.security.UserGroupInformation;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-import javax.security.auth.Subject;
-import java.io.ByteArrayOutputStream;
-import java.io.FileNotFoundException;
-import java.io.IOException;
-import java.io.InputStream;
-import java.security.AccessController;
-import java.security.PrivilegedAction;
-import java.util.Iterator;
-import java.util.Map;
-
import static org.apache.storm.blobstore.BlobStoreAclHandler.ADMIN;
import static org.apache.storm.blobstore.BlobStoreAclHandler.READ;
import static org.apache.storm.blobstore.BlobStoreAclHandler.WRITE;
@@ -73,10 +74,12 @@ public class HdfsBlobStore extends BlobStore {
public static final Logger LOG = LoggerFactory.getLogger(HdfsBlobStore.class);
private static final String DATA_PREFIX = "data_";
private static final String META_PREFIX = "meta_";
- private BlobStoreAclHandler _aclHandler;
- private HdfsBlobStoreImpl _hbs;
- private Subject _localSubject;
- private Map conf;
+ private static final HashMap<String, Subject> alreadyLoggedInUsers = new HashMap<>();
+
+ private BlobStoreAclHandler aclHandler;
+ private HdfsBlobStoreImpl hbs;
+ private Subject localSubject;
+ private Map<String, Object> conf;
/**
* Get the subject from Hadoop so we can use it to validate the acls. There is no direct
@@ -106,7 +109,7 @@ public class HdfsBlobStore extends BlobStore {
*/
private Subject checkAndGetSubject(Subject who) {
if (who == null) {
- return _localSubject;
+ return localSubject;
}
return who;
}
@@ -137,7 +140,15 @@ public class HdfsBlobStore extends BlobStore {
String keyTab = (String) conf.get(Config.BLOBSTORE_HDFS_KEYTAB);
if (principal != null && keyTab != null) {
- UserGroupInformation.loginUserFromKeytab(principal, keyTab);
+ String combinedKey = principal + " from " + keyTab;
+ synchronized (alreadyLoggedInUsers) {
+ localSubject = alreadyLoggedInUsers.get(combinedKey);
+ if (localSubject == null) {
+ UserGroupInformation.loginUserFromKeytab(principal, keyTab);
+ localSubject = getHadoopUser();
+ alreadyLoggedInUsers.put(combinedKey, localSubject);
+ }
+ }
} else {
if (principal == null && keyTab != null) {
throw new RuntimeException("You must specify an HDFS principal to go with the keytab!");
@@ -147,22 +158,22 @@ public class HdfsBlobStore extends BlobStore {
throw new RuntimeException("You must specify HDFS keytab go with the principal!");
}
}
+ localSubject = getHadoopUser();
}
} catch (IOException e) {
throw new RuntimeException("Error logging in from keytab!", e);
}
+ aclHandler = new BlobStoreAclHandler(conf);
Path baseDir = new Path(overrideBase, BASE_BLOBS_DIR_NAME);
try {
if (hadoopConf != null) {
- _hbs = new HdfsBlobStoreImpl(baseDir, conf, hadoopConf);
+ hbs = new HdfsBlobStoreImpl(baseDir, conf, hadoopConf);
} else {
- _hbs = new HdfsBlobStoreImpl(baseDir, conf);
+ hbs = new HdfsBlobStoreImpl(baseDir, conf);
}
} catch (IOException e) {
throw new RuntimeException(e);
}
- _localSubject = getHadoopUser();
- _aclHandler = new BlobStoreAclHandler(conf);
}
@Override
@@ -173,21 +184,21 @@ public class HdfsBlobStore extends BlobStore {
}
who = checkAndGetSubject(who);
validateKey(key);
- _aclHandler.normalizeSettableBlobMeta(key, meta, who, READ | WRITE | ADMIN);
+ aclHandler.normalizeSettableBlobMeta(key, meta, who, READ | WRITE | ADMIN);
BlobStoreAclHandler.validateSettableACLs(key, meta.get_acl());
- _aclHandler.hasPermissions(meta.get_acl(), READ | WRITE | ADMIN, who, key);
- if (_hbs.exists(DATA_PREFIX+key)) {
+ aclHandler.hasPermissions(meta.get_acl(), READ | WRITE | ADMIN, who, key);
+ if (hbs.exists(DATA_PREFIX + key)) {
throw new KeyAlreadyExistsException(key);
}
BlobStoreFileOutputStream mOut = null;
try {
- BlobStoreFile metaFile = _hbs.write(META_PREFIX + key, true);
+ BlobStoreFile metaFile = hbs.write(META_PREFIX + key, true);
metaFile.setMetadata(meta);
mOut = new BlobStoreFileOutputStream(metaFile);
mOut.write(Utils.thriftSerialize(meta));
mOut.close();
mOut = null;
- BlobStoreFile dataFile = _hbs.write(DATA_PREFIX + key, true);
+ BlobStoreFile dataFile = hbs.write(DATA_PREFIX + key, true);
dataFile.setMetadata(meta);
return new BlobStoreFileOutputStream(dataFile);
} catch (IOException e) {
@@ -209,9 +220,9 @@ public class HdfsBlobStore extends BlobStore {
who = checkAndGetSubject(who);
SettableBlobMeta meta = getStoredBlobMeta(key);
validateKey(key);
- _aclHandler.hasPermissions(meta.get_acl(), WRITE, who, key);
+ aclHandler.hasPermissions(meta.get_acl(), WRITE, who, key);
try {
- BlobStoreFile dataFile = _hbs.write(DATA_PREFIX + key, false);
+ BlobStoreFile dataFile = hbs.write(DATA_PREFIX + key, false);
dataFile.setMetadata(meta);
return new BlobStoreFileOutputStream(dataFile);
} catch (IOException e) {
@@ -222,7 +233,7 @@ public class HdfsBlobStore extends BlobStore {
private SettableBlobMeta getStoredBlobMeta(String key) throws KeyNotFoundException {
InputStream in = null;
try {
- BlobStoreFile pf = _hbs.read(META_PREFIX + key);
+ BlobStoreFile pf = hbs.read(META_PREFIX + key);
try {
in = pf.getInputStream();
} catch (FileNotFoundException fnf) {
@@ -256,11 +267,11 @@ public class HdfsBlobStore extends BlobStore {
who = checkAndGetSubject(who);
validateKey(key);
SettableBlobMeta meta = getStoredBlobMeta(key);
- _aclHandler.validateUserCanReadMeta(meta.get_acl(), who, key);
+ aclHandler.validateUserCanReadMeta(meta.get_acl(), who, key);
ReadableBlobMeta rbm = new ReadableBlobMeta();
rbm.set_settable(meta);
try {
- BlobStoreFile pf = _hbs.read(DATA_PREFIX + key);
+ BlobStoreFile pf = hbs.read(DATA_PREFIX + key);
rbm.set_version(pf.getModTime());
} catch (IOException e) {
throw new RuntimeException(e);
@@ -276,10 +287,10 @@ public class HdfsBlobStore extends BlobStore {
}
who = checkAndGetSubject(who);
validateKey(key);
- _aclHandler.normalizeSettableBlobMeta(key, meta, who, ADMIN);
+ aclHandler.normalizeSettableBlobMeta(key, meta, who, ADMIN);
BlobStoreAclHandler.validateSettableACLs(key, meta.get_acl());
SettableBlobMeta orig = getStoredBlobMeta(key);
- _aclHandler.hasPermissions(orig.get_acl(), ADMIN, who, key);
+ aclHandler.hasPermissions(orig.get_acl(), ADMIN, who, key);
BlobStoreFileOutputStream mOut = null;
writeMetadata(key, meta);
}
@@ -290,10 +301,10 @@ public class HdfsBlobStore extends BlobStore {
who = checkAndGetSubject(who);
validateKey(key);
SettableBlobMeta meta = getStoredBlobMeta(key);
- _aclHandler.hasPermissions(meta.get_acl(), WRITE, who, key);
+ aclHandler.hasPermissions(meta.get_acl(), WRITE, who, key);
try {
- _hbs.deleteKey(DATA_PREFIX + key);
- _hbs.deleteKey(META_PREFIX + key);
+ hbs.deleteKey(DATA_PREFIX + key);
+ hbs.deleteKey(META_PREFIX + key);
} catch (IOException e) {
throw new RuntimeException(e);
}
@@ -305,9 +316,9 @@ public class HdfsBlobStore extends BlobStore {
who = checkAndGetSubject(who);
validateKey(key);
SettableBlobMeta meta = getStoredBlobMeta(key);
- _aclHandler.hasPermissions(meta.get_acl(), READ, who, key);
+ aclHandler.hasPermissions(meta.get_acl(), READ, who, key);
try {
- return new BlobStoreFileInputStream(_hbs.read(DATA_PREFIX + key));
+ return new BlobStoreFileInputStream(hbs.read(DATA_PREFIX + key));
} catch (IOException e) {
throw new RuntimeException(e);
}
@@ -316,7 +327,7 @@ public class HdfsBlobStore extends BlobStore {
@Override
public Iterator<String> listKeys() {
try {
- return new KeyTranslationIterator(_hbs.listKeys(), DATA_PREFIX);
+ return new KeyTranslationIterator(hbs.listKeys(), DATA_PREFIX);
} catch (IOException e) {
throw new RuntimeException(e);
}
@@ -332,9 +343,9 @@ public class HdfsBlobStore extends BlobStore {
who = checkAndGetSubject(who);
validateKey(key);
SettableBlobMeta meta = getStoredBlobMeta(key);
- _aclHandler.hasAnyPermissions(meta.get_acl(), READ | WRITE | ADMIN, who, key);
+ aclHandler.hasAnyPermissions(meta.get_acl(), READ | WRITE | ADMIN, who, key);
try {
- return _hbs.getBlobReplication(DATA_PREFIX + key);
+ return hbs.getBlobReplication(DATA_PREFIX + key);
} catch (IOException exp) {
throw new RuntimeException(exp);
}
@@ -346,10 +357,10 @@ public class HdfsBlobStore extends BlobStore {
validateKey(key);
SettableBlobMeta meta = getStoredBlobMeta(key);
meta.set_replication_factor(replication);
- _aclHandler.hasAnyPermissions(meta.get_acl(), WRITE | ADMIN, who, key);
+ aclHandler.hasAnyPermissions(meta.get_acl(), WRITE | ADMIN, who, key);
try {
writeMetadata(key, meta);
- return _hbs.updateBlobReplication(DATA_PREFIX + key, replication);
+ return hbs.updateBlobReplication(DATA_PREFIX + key, replication);
} catch (IOException exp) {
throw new RuntimeException(exp);
}
@@ -359,7 +370,7 @@ public class HdfsBlobStore extends BlobStore {
throws AuthorizationException, KeyNotFoundException {
BlobStoreFileOutputStream mOut = null;
try {
- BlobStoreFile hdfsFile = _hbs.write(META_PREFIX + key, false);
+ BlobStoreFile hdfsFile = hbs.write(META_PREFIX + key, false);
hdfsFile.setMetadata(meta);
mOut = new BlobStoreFileOutputStream(hdfsFile);
mOut.write(Utils.thriftSerialize(meta));
@@ -379,6 +390,6 @@ public class HdfsBlobStore extends BlobStore {
}
public void fullCleanup(long age) throws IOException {
- _hbs.fullCleanup(age);
+ hbs.fullCleanup(age);
}
}
[41/50] [abbrv] storm git commit: STORM-2810: Fix resource leaks in
storm-hdfs tests
Posted by pt...@apache.org.
http://git-wip-us.apache.org/repos/asf/storm/blob/83adfd9d/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/blobstore/BlobStoreTest.java
----------------------------------------------------------------------
diff --git a/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/blobstore/BlobStoreTest.java b/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/blobstore/BlobStoreTest.java
index 6e03bcf..f9374fe 100644
--- a/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/blobstore/BlobStoreTest.java
+++ b/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/blobstore/BlobStoreTest.java
@@ -1,3 +1,4 @@
+
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
@@ -33,10 +34,7 @@ import org.apache.storm.security.auth.NimbusPrincipal;
import org.apache.storm.security.auth.SingleUserPrincipal;
import org.apache.storm.utils.Utils;
import org.apache.commons.io.FileUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.junit.After;
-import org.junit.AfterClass;
import org.junit.Before;
import org.junit.Test;
import org.mockito.Mockito;
@@ -63,469 +61,454 @@ import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
import static org.mockito.Mockito.*;
+import org.apache.storm.hdfs.testing.MiniDFSClusterRule;
+import org.junit.ClassRule;
+
public class BlobStoreTest {
- private static final Logger LOG = LoggerFactory.getLogger(BlobStoreTest.class);
- protected static MiniDFSCluster dfscluster = null;
- protected static Configuration hadoopConf = null;
- URI base;
- File baseFile;
- private static Map conf = new HashMap();
- public static final int READ = 0x01;
- public static final int WRITE = 0x02;
- public static final int ADMIN = 0x04;
-
- @Before
- public void init() {
- System.setProperty("test.build.data", "target/test/data");
- initializeConfigs();
- baseFile = new File("/tmp/blob-store-test-"+UUID.randomUUID());
- base = baseFile.toURI();
- }
-
- @After
- public void cleanup()
- throws IOException {
- FileUtils.deleteDirectory(baseFile);
- }
-
- @AfterClass
- public static void cleanupAfterClass() throws IOException {
- if (dfscluster != null) {
- dfscluster.shutdown();
+
+ @ClassRule
+ public static final MiniDFSClusterRule DFS_CLUSTER_RULE = new MiniDFSClusterRule();
+
+ private static final Logger LOG = LoggerFactory.getLogger(BlobStoreTest.class);
+ URI base;
+ File baseFile;
+ private static final Map CONF = new HashMap();
+ public static final int READ = 0x01;
+ public static final int WRITE = 0x02;
+ public static final int ADMIN = 0x04;
+
+ @Before
+ public void init() {
+ initializeConfigs();
+ baseFile = new File("/tmp/blob-store-test-" + UUID.randomUUID());
+ base = baseFile.toURI();
+ }
+
+ @After
+ public void cleanup()
+ throws IOException {
+ FileUtils.deleteDirectory(baseFile);
+ }
+
+ // Method which initializes nimbus admin
+ public static void initializeConfigs() {
+ CONF.put(Config.NIMBUS_ADMINS, "admin");
+ CONF.put(Config.NIMBUS_SUPERVISOR_USERS, "supervisor");
+ }
+
+ //Gets Nimbus Subject with NimbusPrincipal set on it
+ public static Subject getNimbusSubject() {
+ Subject nimbus = new Subject();
+ nimbus.getPrincipals().add(new NimbusPrincipal());
+ return nimbus;
+ }
+
+ // Overloading the assertStoreHasExactly method accomodate Subject in order to check for authorization
+ public static void assertStoreHasExactly(BlobStore store, Subject who, String... keys)
+ throws IOException, KeyNotFoundException, AuthorizationException {
+ Set<String> expected = new HashSet<>(Arrays.asList(keys));
+ Set<String> found = new HashSet<>();
+ Iterator<String> c = store.listKeys();
+ while (c.hasNext()) {
+ String keyName = c.next();
+ found.add(keyName);
+ }
+ Set<String> extra = new HashSet<>(found);
+ extra.removeAll(expected);
+ assertTrue("Found extra keys in the blob store " + extra, extra.isEmpty());
+ Set<String> missing = new HashSet<>(expected);
+ missing.removeAll(found);
+ assertTrue("Found keys missing from the blob store " + missing, missing.isEmpty());
+ }
+
+ public static void assertStoreHasExactly(BlobStore store, String... keys)
+ throws IOException, KeyNotFoundException, AuthorizationException {
+ assertStoreHasExactly(store, null, keys);
+ }
+
+ // Overloading the readInt method accomodate Subject in order to check for authorization (security turned on)
+ public static int readInt(BlobStore store, Subject who, String key) throws IOException, KeyNotFoundException, AuthorizationException {
+ try (InputStream in = store.getBlob(key, who)) {
+ return in.read();
+ }
+ }
+
+ public static int readInt(BlobStore store, String key)
+ throws IOException, KeyNotFoundException, AuthorizationException {
+ return readInt(store, null, key);
+ }
+
+ public static void readAssertEquals(BlobStore store, String key, int value)
+ throws IOException, KeyNotFoundException, AuthorizationException {
+ assertEquals(value, readInt(store, key));
}
- }
-
- // Method which initializes nimbus admin
- public static void initializeConfigs() {
- conf.put(Config.NIMBUS_ADMINS,"admin");
- conf.put(Config.NIMBUS_SUPERVISOR_USERS,"supervisor");
- }
-
- //Gets Nimbus Subject with NimbusPrincipal set on it
- public static Subject getNimbusSubject() {
- Subject nimbus = new Subject();
- nimbus.getPrincipals().add(new NimbusPrincipal());
- return nimbus;
- }
-
- // Overloading the assertStoreHasExactly method accomodate Subject in order to check for authorization
- public static void assertStoreHasExactly(BlobStore store, Subject who, String ... keys)
- throws IOException, KeyNotFoundException, AuthorizationException {
- Set<String> expected = new HashSet<String>(Arrays.asList(keys));
- Set<String> found = new HashSet<String>();
- Iterator<String> c = store.listKeys();
- while (c.hasNext()) {
- String keyName = c.next();
- found.add(keyName);
+
+ // Checks for assertion when we turn on security
+ public void readAssertEqualsWithAuth(BlobStore store, Subject who, String key, int value)
+ throws IOException, KeyNotFoundException, AuthorizationException {
+ assertEquals(value, readInt(store, who, key));
}
- Set<String> extra = new HashSet<String>(found);
- extra.removeAll(expected);
- assertTrue("Found extra keys in the blob store "+extra, extra.isEmpty());
- Set<String> missing = new HashSet<String>(expected);
- missing.removeAll(found);
- assertTrue("Found keys missing from the blob store "+missing, missing.isEmpty());
- }
-
- public static void assertStoreHasExactly(BlobStore store, String ... keys)
- throws IOException, KeyNotFoundException, AuthorizationException {
- assertStoreHasExactly(store, null, keys);
- }
-
- // Overloading the readInt method accomodate Subject in order to check for authorization (security turned on)
- public static int readInt(BlobStore store, Subject who, String key) throws IOException, KeyNotFoundException, AuthorizationException {
- InputStream in = store.getBlob(key, who);
- try {
- return in.read();
- } finally {
- in.close();
+
+ private AutoCloseableBlobStoreContainer initHdfs(String dirName)
+ throws Exception {
+ Map conf = new HashMap();
+ conf.put(Config.BLOBSTORE_DIR, dirName);
+ conf.put(Config.STORM_PRINCIPAL_TO_LOCAL_PLUGIN, "org.apache.storm.security.auth.DefaultPrincipalToLocal");
+ conf.put(Config.STORM_BLOBSTORE_REPLICATION_FACTOR, 3);
+ HdfsBlobStore store = new HdfsBlobStore();
+ store.prepareInternal(conf, null, DFS_CLUSTER_RULE.getDfscluster().getConfiguration(0));
+ return new AutoCloseableBlobStoreContainer(store);
}
- }
-
- public static int readInt(BlobStore store, String key)
- throws IOException, KeyNotFoundException, AuthorizationException {
- return readInt(store, null, key);
- }
-
- public static void readAssertEquals(BlobStore store, String key, int value)
- throws IOException, KeyNotFoundException, AuthorizationException {
- assertEquals(value, readInt(store, key));
- }
-
- // Checks for assertion when we turn on security
- public void readAssertEqualsWithAuth(BlobStore store, Subject who, String key, int value)
- throws IOException, KeyNotFoundException, AuthorizationException {
- assertEquals(value, readInt(store, who, key));
- }
-
- private HdfsBlobStore initHdfs(String dirName)
- throws Exception {
- if (hadoopConf == null) {
- hadoopConf = new Configuration();
+
+ private static class AutoCloseableBlobStoreContainer implements AutoCloseable {
+
+ private final HdfsBlobStore blobStore;
+
+ public AutoCloseableBlobStoreContainer(HdfsBlobStore blobStore) {
+ this.blobStore = blobStore;
+ }
+
+ @Override
+ public void close() throws Exception {
+ this.blobStore.shutdown();
+ }
+
}
- try {
- if (dfscluster == null) {
- dfscluster = new MiniDFSCluster.Builder(hadoopConf).numDataNodes(3).build();
- dfscluster.waitActive();
- }
- } catch (IOException e) {
- LOG.error("error creating MiniDFSCluster");
+
+ @Test
+ public void testHdfsReplication()
+ throws Exception {
+ try (AutoCloseableBlobStoreContainer container = initHdfs("/storm/blobstoreReplication")) {
+ testReplication("/storm/blobstoreReplication/test", container.blobStore);
+ }
}
- Map conf = new HashMap();
- conf.put(Config.BLOBSTORE_DIR, dirName);
- conf.put(Config.STORM_PRINCIPAL_TO_LOCAL_PLUGIN,"org.apache.storm.security.auth.DefaultPrincipalToLocal");
- conf.put(Config.STORM_BLOBSTORE_REPLICATION_FACTOR, 3);
- HdfsBlobStore store = new HdfsBlobStore();
- store.prepareInternal(conf, null, dfscluster.getConfiguration(0));
- return store;
- }
-
- @Test
- public void testHdfsReplication()
- throws Exception {
- BlobStore store = initHdfs("/storm/blobstoreReplication");
- testReplication("/storm/blobstoreReplication/test", store);
- }
-
- @Test
- public void testBasicHdfs()
- throws Exception {
- testBasic(initHdfs("/storm/blobstore1"));
- }
-
- @Test
- public void testMultipleHdfs()
- throws Exception {
- // use different blobstore dir so it doesn't conflict with other test
- testMultiple(initHdfs("/storm/blobstore2"));
- }
-
- @Test
- public void testHdfsWithAuth()
- throws Exception {
- // use different blobstore dir so it doesn't conflict with other tests
- testWithAuthentication(initHdfs("/storm/blobstore3"));
- }
-
- // Test for replication.
- public void testReplication(String path, BlobStore store)
- throws Exception {
- SettableBlobMeta metadata = new SettableBlobMeta(BlobStoreAclHandler.WORLD_EVERYTHING);
- metadata.set_replication_factor(4);
- AtomicOutputStream out = store.createBlob("test", metadata, null);
- out.write(1);
- out.close();
- assertStoreHasExactly(store, "test");
- assertEquals("Blobstore replication not matching", store.getBlobReplication("test", null), 4);
- store.deleteBlob("test", null);
-
- //Test for replication with NIMBUS as user
- Subject admin = getSubject("admin");
- metadata = new SettableBlobMeta(BlobStoreAclHandler.DEFAULT);
- metadata.set_replication_factor(4);
- out = store.createBlob("test", metadata, admin);
- out.write(1);
- out.close();
- assertStoreHasExactly(store, "test");
- assertEquals("Blobstore replication not matching", store.getBlobReplication("test", admin), 4);
- store.updateBlobReplication("test", 5, admin);
- assertEquals("Blobstore replication not matching", store.getBlobReplication("test", admin), 5);
- store.deleteBlob("test", admin);
-
- //Test for replication using SUPERVISOR access
- Subject supervisor = getSubject("supervisor");
- metadata = new SettableBlobMeta(BlobStoreAclHandler.DEFAULT);
- metadata.set_replication_factor(4);
- out = store.createBlob("test", metadata, supervisor);
- out.write(1);
- out.close();
- assertStoreHasExactly(store, "test");
- assertEquals("Blobstore replication not matching", store.getBlobReplication("test", supervisor), 4);
- store.updateBlobReplication("test", 5, supervisor);
- assertEquals("Blobstore replication not matching", store.getBlobReplication("test", supervisor), 5);
- store.deleteBlob("test", supervisor);
-
- //Test for a user having read or write or admin access to read replication for a blob
- String createSubject = "createSubject";
- String writeSubject = "writeSubject";
- String adminSubject = "adminSubject";
- Subject who = getSubject(createSubject);
- AccessControl writeAccess = new AccessControl(AccessControlType.USER, READ);
- AccessControl adminAccess = new AccessControl(AccessControlType.USER, ADMIN);
- writeAccess.set_name(writeSubject);
- adminAccess.set_name(adminSubject);
- List<AccessControl> acl = Arrays.asList(writeAccess, adminAccess);
- metadata = new SettableBlobMeta(acl);
- metadata.set_replication_factor(4);
- out = store.createBlob("test", metadata, who);
- out.write(1);
- out.close();
- assertStoreHasExactly(store, "test");
- who = getSubject(writeSubject);
- assertEquals("Blobstore replication not matching", store.getBlobReplication("test", who), 4);
-
- //Test for a user having WRITE or ADMIN privileges to change replication of a blob
- who = getSubject(adminSubject);
- store.updateBlobReplication("test", 5, who);
- assertEquals("Blobstore replication not matching", store.getBlobReplication("test", who), 5);
- store.deleteBlob("test", getSubject(createSubject));
- }
-
- public Subject getSubject(String name) {
- Subject subject = new Subject();
- SingleUserPrincipal user = new SingleUserPrincipal(name);
- subject.getPrincipals().add(user);
- return subject;
- }
-
- // Check for Blobstore with authentication
- public void testWithAuthentication(BlobStore store)
- throws Exception {
- //Test for Nimbus Admin
- Subject admin = getSubject("admin");
- assertStoreHasExactly(store);
- SettableBlobMeta metadata = new SettableBlobMeta(BlobStoreAclHandler.DEFAULT);
- AtomicOutputStream out = store.createBlob("test", metadata, admin);
- assertStoreHasExactly(store, "test");
- out.write(1);
- out.close();
- store.deleteBlob("test", admin);
-
- //Test for Supervisor Admin
- Subject supervisor = getSubject("supervisor");
- assertStoreHasExactly(store);
- metadata = new SettableBlobMeta(BlobStoreAclHandler.DEFAULT);
- out = store.createBlob("test", metadata, supervisor);
- assertStoreHasExactly(store, "test");
- out.write(1);
- out.close();
- store.deleteBlob("test", supervisor);
-
- //Test for Nimbus itself as a user
- Subject nimbus = getNimbusSubject();
- assertStoreHasExactly(store);
- metadata = new SettableBlobMeta(BlobStoreAclHandler.DEFAULT);
- out = store.createBlob("test", metadata, nimbus);
- assertStoreHasExactly(store, "test");
- out.write(1);
- out.close();
- store.deleteBlob("test", nimbus);
-
- // Test with a dummy test_subject for cases where subject !=null (security turned on)
- Subject who = getSubject("test_subject");
- assertStoreHasExactly(store);
-
- // Tests for case when subject != null (security turned on) and
- // acls for the blob are set to WORLD_EVERYTHING
- metadata = new SettableBlobMeta(BlobStoreAclHandler.WORLD_EVERYTHING);
- out = store.createBlob("test", metadata, who);
- out.write(1);
- out.close();
- assertStoreHasExactly(store, "test");
- // Testing whether acls are set to WORLD_EVERYTHING
- assertTrue("ACL does not contain WORLD_EVERYTHING", metadata.toString().contains("AccessControl(type:OTHER, access:7)"));
- readAssertEqualsWithAuth(store, who, "test", 1);
-
- LOG.info("Deleting test");
- store.deleteBlob("test", who);
- assertStoreHasExactly(store);
-
- // Tests for case when subject != null (security turned on) and
- // acls are not set for the blob (DEFAULT)
- LOG.info("Creating test again");
- metadata = new SettableBlobMeta(BlobStoreAclHandler.DEFAULT);
- out = store.createBlob("test", metadata, who);
- out.write(2);
- out.close();
- assertStoreHasExactly(store, "test");
- // Testing whether acls are set to WORLD_EVERYTHING. Here the acl should not contain WORLD_EVERYTHING because
- // the subject is neither null nor empty. The ACL should however contain USER_EVERYTHING as user needs to have
- // complete access to the blob
- assertTrue("ACL does not contain WORLD_EVERYTHING", !metadata.toString().contains("AccessControl(type:OTHER, access:7)"));
- readAssertEqualsWithAuth(store, who, "test", 2);
-
- LOG.info("Updating test");
- out = store.updateBlob("test", who);
- out.write(3);
- out.close();
- assertStoreHasExactly(store, "test");
- readAssertEqualsWithAuth(store, who, "test", 3);
-
- LOG.info("Updating test again");
- out = store.updateBlob("test", who);
- out.write(4);
- out.flush();
- LOG.info("SLEEPING");
- Thread.sleep(2);
- assertStoreHasExactly(store, "test");
- readAssertEqualsWithAuth(store, who, "test", 3);
-
- //Test for subject with no principals and acls set to WORLD_EVERYTHING
- who = new Subject();
- metadata = new SettableBlobMeta(BlobStoreAclHandler.WORLD_EVERYTHING);
- LOG.info("Creating test");
- out = store.createBlob("test-empty-subject-WE", metadata, who);
- out.write(2);
- out.close();
- assertStoreHasExactly(store, "test-empty-subject-WE", "test");
- // Testing whether acls are set to WORLD_EVERYTHING
- assertTrue("ACL does not contain WORLD_EVERYTHING", metadata.toString().contains("AccessControl(type:OTHER, access:7)"));
- readAssertEqualsWithAuth(store, who, "test-empty-subject-WE", 2);
-
- //Test for subject with no principals and acls set to DEFAULT
- who = new Subject();
- metadata = new SettableBlobMeta(BlobStoreAclHandler.DEFAULT);
- LOG.info("Creating other");
- out = store.createBlob("test-empty-subject-DEF", metadata, who);
- out.write(2);
- out.close();
- assertStoreHasExactly(store, "test-empty-subject-DEF", "test", "test-empty-subject-WE");
- // Testing whether acls are set to WORLD_EVERYTHING
- assertTrue("ACL does not contain WORLD_EVERYTHING", metadata.toString().contains("AccessControl(type:OTHER, access:7)"));
- readAssertEqualsWithAuth(store, who, "test-empty-subject-DEF", 2);
-
- if (store instanceof HdfsBlobStore) {
- ((HdfsBlobStore) store).fullCleanup(1);
- } else {
- fail("Error the blobstore is of unknowntype");
+
+ @Test
+ public void testBasicHdfs()
+ throws Exception {
+ try (AutoCloseableBlobStoreContainer container = initHdfs("/storm/blobstore1")) {
+ testBasic(container.blobStore);
+ }
}
- try {
- out.close();
- } catch (IOException e) {
- //This is likely to happen when we try to commit something that
- // was cleaned up. This is expected and acceptable.
+
+ @Test
+ public void testMultipleHdfs()
+ throws Exception {
+ // use different blobstore dir so it doesn't conflict with other test
+ try (AutoCloseableBlobStoreContainer container = initHdfs("/storm/blobstore2")) {
+ testMultiple(container.blobStore);
+ }
}
- }
-
- public void testBasic(BlobStore store)
- throws Exception {
- assertStoreHasExactly(store);
- LOG.info("Creating test");
- // Tests for case when subject == null (security turned off) and
- // acls for the blob are set to WORLD_EVERYTHING
- SettableBlobMeta metadata = new SettableBlobMeta(BlobStoreAclHandler
- .WORLD_EVERYTHING);
- AtomicOutputStream out = store.createBlob("test", metadata, null);
- out.write(1);
- out.close();
- assertStoreHasExactly(store, "test");
- // Testing whether acls are set to WORLD_EVERYTHING
- assertTrue("ACL does not contain WORLD_EVERYTHING", metadata.toString().contains("AccessControl(type:OTHER, access:7)"));
- readAssertEquals(store, "test", 1);
-
- LOG.info("Deleting test");
- store.deleteBlob("test", null);
- assertStoreHasExactly(store);
-
- // The following tests are run for both hdfs and local store to test the
- // update blob interface
- metadata = new SettableBlobMeta(BlobStoreAclHandler.WORLD_EVERYTHING);
- LOG.info("Creating test again");
- out = store.createBlob("test", metadata, null);
- out.write(2);
- out.close();
- assertStoreHasExactly(store, "test");
- readAssertEquals(store, "test", 2);
- LOG.info("Updating test");
- out = store.updateBlob("test", null);
- out.write(3);
- out.close();
- assertStoreHasExactly(store, "test");
- readAssertEquals(store, "test", 3);
-
- LOG.info("Updating test again");
- out = store.updateBlob("test", null);
- out.write(4);
- out.flush();
- LOG.info("SLEEPING");
- Thread.sleep(2);
-
- if (store instanceof HdfsBlobStore) {
- ((HdfsBlobStore) store).fullCleanup(1);
- } else {
- fail("Error the blobstore is of unknowntype");
+
+ @Test
+ public void testHdfsWithAuth()
+ throws Exception {
+ // use different blobstore dir so it doesn't conflict with other tests
+ try (AutoCloseableBlobStoreContainer container = initHdfs("/storm/blobstore3")) {
+ testWithAuthentication(container.blobStore);
+ }
}
- try {
- out.close();
- } catch (IOException e) {
- //This is likely to happen when we try to commit something that
- // was cleaned up. This is expected and acceptable.
+
+ // Test for replication.
+ public void testReplication(String path, BlobStore store)
+ throws Exception {
+ SettableBlobMeta metadata = new SettableBlobMeta(BlobStoreAclHandler.WORLD_EVERYTHING);
+ metadata.set_replication_factor(4);
+ try (AtomicOutputStream out = store.createBlob("test", metadata, null)) {
+ out.write(1);
+ }
+ assertStoreHasExactly(store, "test");
+ assertEquals("Blobstore replication not matching", store.getBlobReplication("test", null), 4);
+ store.deleteBlob("test", null);
+
+ //Test for replication with NIMBUS as user
+ Subject admin = getSubject("admin");
+ metadata = new SettableBlobMeta(BlobStoreAclHandler.DEFAULT);
+ metadata.set_replication_factor(4);
+ try (AtomicOutputStream out = store.createBlob("test", metadata, admin)) {
+ out.write(1);
+ }
+ assertStoreHasExactly(store, "test");
+ assertEquals("Blobstore replication not matching", store.getBlobReplication("test", admin), 4);
+ store.updateBlobReplication("test", 5, admin);
+ assertEquals("Blobstore replication not matching", store.getBlobReplication("test", admin), 5);
+ store.deleteBlob("test", admin);
+
+ //Test for replication using SUPERVISOR access
+ Subject supervisor = getSubject("supervisor");
+ metadata = new SettableBlobMeta(BlobStoreAclHandler.DEFAULT);
+ metadata.set_replication_factor(4);
+ try (AtomicOutputStream out = store.createBlob("test", metadata, supervisor)) {
+ out.write(1);
+ }
+ assertStoreHasExactly(store, "test");
+ assertEquals("Blobstore replication not matching", store.getBlobReplication("test", supervisor), 4);
+ store.updateBlobReplication("test", 5, supervisor);
+ assertEquals("Blobstore replication not matching", store.getBlobReplication("test", supervisor), 5);
+ store.deleteBlob("test", supervisor);
+
+ //Test for a user having read or write or admin access to read replication for a blob
+ String createSubject = "createSubject";
+ String writeSubject = "writeSubject";
+ String adminSubject = "adminSubject";
+ Subject who = getSubject(createSubject);
+ AccessControl writeAccess = new AccessControl(AccessControlType.USER, READ);
+ AccessControl adminAccess = new AccessControl(AccessControlType.USER, ADMIN);
+ writeAccess.set_name(writeSubject);
+ adminAccess.set_name(adminSubject);
+ List<AccessControl> acl = Arrays.asList(writeAccess, adminAccess);
+ metadata = new SettableBlobMeta(acl);
+ metadata.set_replication_factor(4);
+ try (AtomicOutputStream out = store.createBlob("test", metadata, who)) {
+ out.write(1);
+ }
+ assertStoreHasExactly(store, "test");
+ who = getSubject(writeSubject);
+ assertEquals("Blobstore replication not matching", store.getBlobReplication("test", who), 4);
+
+ //Test for a user having WRITE or ADMIN privileges to change replication of a blob
+ who = getSubject(adminSubject);
+ store.updateBlobReplication("test", 5, who);
+ assertEquals("Blobstore replication not matching", store.getBlobReplication("test", who), 5);
+ store.deleteBlob("test", getSubject(createSubject));
}
- }
-
-
- public void testMultiple(BlobStore store)
- throws Exception {
- assertStoreHasExactly(store);
- LOG.info("Creating test");
- AtomicOutputStream out = store.createBlob("test", new SettableBlobMeta(BlobStoreAclHandler
- .WORLD_EVERYTHING), null);
- out.write(1);
- out.close();
- assertStoreHasExactly(store, "test");
- readAssertEquals(store, "test", 1);
-
- LOG.info("Creating other");
- out = store.createBlob("other", new SettableBlobMeta(BlobStoreAclHandler.WORLD_EVERYTHING),
- null);
- out.write(2);
- out.close();
- assertStoreHasExactly(store, "test", "other");
- readAssertEquals(store, "test", 1);
- readAssertEquals(store, "other", 2);
-
- LOG.info("Updating other");
- out = store.updateBlob("other", null);
- out.write(5);
- out.close();
- assertStoreHasExactly(store, "test", "other");
- readAssertEquals(store, "test", 1);
- readAssertEquals(store, "other", 5);
-
- LOG.info("Deleting test");
- store.deleteBlob("test", null);
- assertStoreHasExactly(store, "other");
- readAssertEquals(store, "other", 5);
-
- LOG.info("Creating test again");
- out = store.createBlob("test", new SettableBlobMeta(BlobStoreAclHandler.WORLD_EVERYTHING),
- null);
- out.write(2);
- out.close();
- assertStoreHasExactly(store, "test", "other");
- readAssertEquals(store, "test", 2);
- readAssertEquals(store, "other", 5);
-
- LOG.info("Updating test");
- out = store.updateBlob("test", null);
- out.write(3);
- out.close();
- assertStoreHasExactly(store, "test", "other");
- readAssertEquals(store, "test", 3);
- readAssertEquals(store, "other", 5);
-
- LOG.info("Deleting other");
- store.deleteBlob("other", null);
- assertStoreHasExactly(store, "test");
- readAssertEquals(store, "test", 3);
-
- LOG.info("Updating test again");
- out = store.updateBlob("test", null);
- out.write(4);
- out.flush();
- LOG.info("SLEEPING");
- Thread.sleep(2);
-
- if (store instanceof HdfsBlobStore) {
- ((HdfsBlobStore) store).fullCleanup(1);
- } else {
- fail("Error the blobstore is of unknowntype");
- } assertStoreHasExactly(store, "test");
- readAssertEquals(store, "test", 3);
- try {
- out.close();
- } catch (IOException e) {
- //This is likely to happen when we try to commit something that
- // was cleaned up. This is expected and acceptable.
+
+ public Subject getSubject(String name) {
+ Subject subject = new Subject();
+ SingleUserPrincipal user = new SingleUserPrincipal(name);
+ subject.getPrincipals().add(user);
+ return subject;
+ }
+
+ // Check for Blobstore with authentication
+ public void testWithAuthentication(BlobStore store)
+ throws Exception {
+ //Test for Nimbus Admin
+ Subject admin = getSubject("admin");
+ assertStoreHasExactly(store);
+ SettableBlobMeta metadata = new SettableBlobMeta(BlobStoreAclHandler.DEFAULT);
+ try (AtomicOutputStream out = store.createBlob("test", metadata, admin)) {
+ assertStoreHasExactly(store, "test");
+ out.write(1);
+ }
+ store.deleteBlob("test", admin);
+
+ //Test for Supervisor Admin
+ Subject supervisor = getSubject("supervisor");
+ assertStoreHasExactly(store);
+ metadata = new SettableBlobMeta(BlobStoreAclHandler.DEFAULT);
+ try (AtomicOutputStream out = store.createBlob("test", metadata, supervisor)) {
+ assertStoreHasExactly(store, "test");
+ out.write(1);
+ }
+ store.deleteBlob("test", supervisor);
+
+ //Test for Nimbus itself as a user
+ Subject nimbus = getNimbusSubject();
+ assertStoreHasExactly(store);
+ metadata = new SettableBlobMeta(BlobStoreAclHandler.DEFAULT);
+ try (AtomicOutputStream out = store.createBlob("test", metadata, nimbus)) {
+ assertStoreHasExactly(store, "test");
+ out.write(1);
+ }
+ store.deleteBlob("test", nimbus);
+
+ // Test with a dummy test_subject for cases where subject !=null (security turned on)
+ Subject who = getSubject("test_subject");
+ assertStoreHasExactly(store);
+
+ // Tests for case when subject != null (security turned on) and
+ // acls for the blob are set to WORLD_EVERYTHING
+ metadata = new SettableBlobMeta(BlobStoreAclHandler.WORLD_EVERYTHING);
+ try (AtomicOutputStream out = store.createBlob("test", metadata, who)) {
+ out.write(1);
+ }
+ assertStoreHasExactly(store, "test");
+ // Testing whether acls are set to WORLD_EVERYTHING
+ assertTrue("ACL does not contain WORLD_EVERYTHING", metadata.toString().contains("AccessControl(type:OTHER, access:7)"));
+ readAssertEqualsWithAuth(store, who, "test", 1);
+
+ LOG.info("Deleting test");
+ store.deleteBlob("test", who);
+ assertStoreHasExactly(store);
+
+ // Tests for case when subject != null (security turned on) and
+ // acls are not set for the blob (DEFAULT)
+ LOG.info("Creating test again");
+ metadata = new SettableBlobMeta(BlobStoreAclHandler.DEFAULT);
+ try (AtomicOutputStream out = store.createBlob("test", metadata, who)) {
+ out.write(2);
+ }
+ assertStoreHasExactly(store, "test");
+ // Testing whether acls are set to WORLD_EVERYTHING. Here the acl should not contain WORLD_EVERYTHING because
+ // the subject is neither null nor empty. The ACL should however contain USER_EVERYTHING as user needs to have
+ // complete access to the blob
+ assertTrue("ACL does not contain WORLD_EVERYTHING", !metadata.toString().contains("AccessControl(type:OTHER, access:7)"));
+ readAssertEqualsWithAuth(store, who, "test", 2);
+
+ LOG.info("Updating test");
+ try (AtomicOutputStream out = store.updateBlob("test", who)) {
+ out.write(3);
+ }
+ assertStoreHasExactly(store, "test");
+ readAssertEqualsWithAuth(store, who, "test", 3);
+
+ LOG.info("Updating test again");
+ try (AtomicOutputStream out = store.updateBlob("test", who)) {
+ out.write(4);
+ }
+ LOG.info("SLEEPING");
+ Thread.sleep(2);
+ assertStoreHasExactly(store, "test");
+ readAssertEqualsWithAuth(store, who, "test", 3);
+
+ //Test for subject with no principals and acls set to WORLD_EVERYTHING
+ who = new Subject();
+ metadata = new SettableBlobMeta(BlobStoreAclHandler.WORLD_EVERYTHING);
+ LOG.info("Creating test");
+ try (AtomicOutputStream out = store.createBlob("test-empty-subject-WE", metadata, who)) {
+ out.write(2);
+ }
+ assertStoreHasExactly(store, "test-empty-subject-WE", "test");
+ // Testing whether acls are set to WORLD_EVERYTHING
+ assertTrue("ACL does not contain WORLD_EVERYTHING", metadata.toString().contains("AccessControl(type:OTHER, access:7)"));
+ readAssertEqualsWithAuth(store, who, "test-empty-subject-WE", 2);
+
+ //Test for subject with no principals and acls set to DEFAULT
+ who = new Subject();
+ metadata = new SettableBlobMeta(BlobStoreAclHandler.DEFAULT);
+ LOG.info("Creating other");
+ try (AtomicOutputStream out = store.createBlob("test-empty-subject-DEF", metadata, who)) {
+ out.write(2);
+ }
+ assertStoreHasExactly(store, "test-empty-subject-DEF", "test", "test-empty-subject-WE");
+ // Testing whether acls are set to WORLD_EVERYTHING
+ assertTrue("ACL does not contain WORLD_EVERYTHING", metadata.toString().contains("AccessControl(type:OTHER, access:7)"));
+ readAssertEqualsWithAuth(store, who, "test-empty-subject-DEF", 2);
+
+ if (store instanceof HdfsBlobStore) {
+ ((HdfsBlobStore) store).fullCleanup(1);
+ } else {
+ fail("Error the blobstore is of unknowntype");
+ }
+ }
+
+ public void testBasic(BlobStore store)
+ throws Exception {
+ assertStoreHasExactly(store);
+ LOG.info("Creating test");
+ // Tests for case when subject == null (security turned off) and
+ // acls for the blob are set to WORLD_EVERYTHING
+ SettableBlobMeta metadata = new SettableBlobMeta(BlobStoreAclHandler.WORLD_EVERYTHING);
+ try (AtomicOutputStream out = store.createBlob("test", metadata, null)) {
+ out.write(1);
+ }
+ assertStoreHasExactly(store, "test");
+ // Testing whether acls are set to WORLD_EVERYTHING
+ assertTrue("ACL does not contain WORLD_EVERYTHING", metadata.toString().contains("AccessControl(type:OTHER, access:7)"));
+ readAssertEquals(store, "test", 1);
+
+ LOG.info("Deleting test");
+ store.deleteBlob("test", null);
+ assertStoreHasExactly(store);
+
+ // The following tests are run for both hdfs and local store to test the
+ // update blob interface
+ metadata = new SettableBlobMeta(BlobStoreAclHandler.WORLD_EVERYTHING);
+ LOG.info("Creating test again");
+ try (AtomicOutputStream out = store.createBlob("test", metadata, null)) {
+ out.write(2);
+ }
+ assertStoreHasExactly(store, "test");
+ readAssertEquals(store, "test", 2);
+ LOG.info("Updating test");
+ try (AtomicOutputStream out = store.updateBlob("test", null)) {
+ out.write(3);
+ }
+ assertStoreHasExactly(store, "test");
+ readAssertEquals(store, "test", 3);
+
+ LOG.info("Updating test again");
+ try (AtomicOutputStream out = store.updateBlob("test", null)) {
+ out.write(4);
+ }
+ LOG.info("SLEEPING");
+ Thread.sleep(2);
+
+ if (store instanceof HdfsBlobStore) {
+ ((HdfsBlobStore) store).fullCleanup(1);
+ } else {
+ fail("Error the blobstore is of unknowntype");
+ }
+ }
+
+ public void testMultiple(BlobStore store)
+ throws Exception {
+ assertStoreHasExactly(store);
+ LOG.info("Creating test");
+ try (AtomicOutputStream out = store.createBlob("test", new SettableBlobMeta(BlobStoreAclHandler.WORLD_EVERYTHING), null)) {
+ out.write(1);
+ }
+ assertStoreHasExactly(store, "test");
+ readAssertEquals(store, "test", 1);
+
+ LOG.info("Creating other");
+ try (AtomicOutputStream out = store.createBlob("other", new SettableBlobMeta(BlobStoreAclHandler.WORLD_EVERYTHING),
+ null)) {
+ out.write(2);
+ }
+ assertStoreHasExactly(store, "test", "other");
+ readAssertEquals(store, "test", 1);
+ readAssertEquals(store, "other", 2);
+
+ LOG.info("Updating other");
+ try (AtomicOutputStream out = store.updateBlob("other", null)) {
+ out.write(5);
+ }
+ assertStoreHasExactly(store, "test", "other");
+ readAssertEquals(store, "test", 1);
+ readAssertEquals(store, "other", 5);
+
+ LOG.info("Deleting test");
+ store.deleteBlob("test", null);
+ assertStoreHasExactly(store, "other");
+ readAssertEquals(store, "other", 5);
+
+ LOG.info("Creating test again");
+ try (AtomicOutputStream out = store.createBlob("test", new SettableBlobMeta(BlobStoreAclHandler.WORLD_EVERYTHING),
+ null)) {
+ out.write(2);
+ }
+ assertStoreHasExactly(store, "test", "other");
+ readAssertEquals(store, "test", 2);
+ readAssertEquals(store, "other", 5);
+
+ LOG.info("Updating test");
+ try (AtomicOutputStream out = store.updateBlob("test", null)) {
+ out.write(3);
+ }
+ assertStoreHasExactly(store, "test", "other");
+ readAssertEquals(store, "test", 3);
+ readAssertEquals(store, "other", 5);
+
+ LOG.info("Deleting other");
+ store.deleteBlob("other", null);
+ assertStoreHasExactly(store, "test");
+ readAssertEquals(store, "test", 3);
+
+ LOG.info("Updating test again");
+ try (AtomicOutputStream out = store.updateBlob("test", null)) {
+ out.write(4);
+ }
+ LOG.info("SLEEPING");
+ Thread.sleep(2);
+
+ if (store instanceof HdfsBlobStore) {
+ ((HdfsBlobStore) store).fullCleanup(1);
+ } else {
+ fail("Error the blobstore is of unknowntype");
+ }
+ assertStoreHasExactly(store, "test");
+ readAssertEquals(store, "test", 3);
}
- }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/83adfd9d/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/blobstore/HdfsBlobStoreImplTest.java
----------------------------------------------------------------------
diff --git a/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/blobstore/HdfsBlobStoreImplTest.java b/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/blobstore/HdfsBlobStoreImplTest.java
index c49c44b..fd80631 100644
--- a/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/blobstore/HdfsBlobStoreImplTest.java
+++ b/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/blobstore/HdfsBlobStoreImplTest.java
@@ -1,3 +1,4 @@
+
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
@@ -23,9 +24,6 @@ import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
-import org.junit.AfterClass;
-import org.junit.BeforeClass;
import org.junit.Test;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -39,194 +37,190 @@ import java.util.Map;
import static org.junit.Assert.*;
+import java.io.InputStream;
+import java.nio.charset.StandardCharsets;
+import org.apache.storm.hdfs.testing.MiniDFSClusterRule;
+import org.junit.ClassRule;
+
public class HdfsBlobStoreImplTest {
+
+ @ClassRule
+ public static final MiniDFSClusterRule DFS_CLUSTER_RULE = new MiniDFSClusterRule();
+
private static final Logger LOG = LoggerFactory.getLogger(HdfsBlobStoreImplTest.class);
- protected static Configuration hadoopConf;
- protected static MiniDFSCluster dfscluster;
// key dir needs to be number 0 to number of buckets, choose one so we know where to look
private static String KEYDIR = "0";
private Path blobDir = new Path("/storm/blobstore1");
private Path fullKeyDir = new Path(blobDir, KEYDIR);
private String BLOBSTORE_DATA = "data";
- public class TestHdfsBlobStoreImpl extends HdfsBlobStoreImpl {
+ public class TestHdfsBlobStoreImpl extends HdfsBlobStoreImpl implements AutoCloseable {
public TestHdfsBlobStoreImpl(Path path, Map<String, Object> conf) throws IOException {
super(path, conf);
}
public TestHdfsBlobStoreImpl(Path path, Map<String, Object> conf,
- Configuration hconf) throws IOException {
+ Configuration hconf) throws IOException {
super(path, conf, hconf);
}
protected Path getKeyDir(String key) {
return new Path(new Path(blobDir, KEYDIR), key);
}
- }
-
- @BeforeClass
- public static void init() {
- System.setProperty("test.build.data", "target/test/data");
- if (hadoopConf == null) {
- hadoopConf = new Configuration();
- }
- try {
- if (dfscluster == null) {
- dfscluster = new MiniDFSCluster.Builder(hadoopConf).build();
- dfscluster.waitActive();
- }
- } catch (IOException e) {
- LOG.error("error creating MiniDFSCluster");
- }
- }
- @AfterClass
- public static void cleanup() throws IOException {
- if (dfscluster != null) {
- dfscluster.shutdown();
+ @Override
+ public void close() throws Exception {
+ this.shutdown();
}
}
// Be careful about adding additional tests as the dfscluster will be shared
-
@Test
public void testMultiple() throws Exception {
String testString = "testingblob";
String validKey = "validkeyBasic";
- FileSystem fs = dfscluster.getFileSystem();
+ //Will be closed automatically when shutting down the DFS cluster
+ FileSystem fs = DFS_CLUSTER_RULE.getDfscluster().getFileSystem();
Map conf = new HashMap();
- TestHdfsBlobStoreImpl hbs = new TestHdfsBlobStoreImpl(blobDir, conf, hadoopConf);
- // should have created blobDir
- assertTrue("BlobStore dir wasn't created", fs.exists(blobDir));
- assertEquals("BlobStore dir was created with wrong permissions",
+ try (TestHdfsBlobStoreImpl hbs = new TestHdfsBlobStoreImpl(blobDir, conf, DFS_CLUSTER_RULE.getHadoopConf())) {
+ // should have created blobDir
+ assertTrue("BlobStore dir wasn't created", fs.exists(blobDir));
+ assertEquals("BlobStore dir was created with wrong permissions",
HdfsBlobStoreImpl.BLOBSTORE_DIR_PERMISSION, fs.getFileStatus(blobDir).getPermission());
- // test exist with non-existent key
- assertFalse("file exists but shouldn't", hbs.exists("bogus"));
-
- // test write
- BlobStoreFile pfile = hbs.write(validKey, false);
- // Adding metadata to avoid null pointer exception
- SettableBlobMeta meta = new SettableBlobMeta();
- meta.set_replication_factor(1);
- pfile.setMetadata(meta);
- OutputStream ios = pfile.getOutputStream();
- ios.write(testString.getBytes(Charset.forName("UTF-8")));
- ios.close();
-
- // test commit creates properly
- assertTrue("BlobStore key dir wasn't created", fs.exists(fullKeyDir));
- pfile.commit();
- Path dataFile = new Path(new Path(fullKeyDir, validKey), BLOBSTORE_DATA);
- assertTrue("blob data not committed", fs.exists(dataFile));
- assertEquals("BlobStore dir was created with wrong permissions",
+ // test exist with non-existent key
+ assertFalse("file exists but shouldn't", hbs.exists("bogus"));
+
+ // test write
+ BlobStoreFile pfile = hbs.write(validKey, false);
+ // Adding metadata to avoid null pointer exception
+ SettableBlobMeta meta = new SettableBlobMeta();
+ meta.set_replication_factor(1);
+ pfile.setMetadata(meta);
+ try (OutputStream ios = pfile.getOutputStream()) {
+ ios.write(testString.getBytes(StandardCharsets.UTF_8));
+ }
+
+ // test commit creates properly
+ assertTrue("BlobStore key dir wasn't created", fs.exists(fullKeyDir));
+ pfile.commit();
+ Path dataFile = new Path(new Path(fullKeyDir, validKey), BLOBSTORE_DATA);
+ assertTrue("blob data not committed", fs.exists(dataFile));
+ assertEquals("BlobStore dir was created with wrong permissions",
HdfsBlobStoreFile.BLOBSTORE_FILE_PERMISSION, fs.getFileStatus(dataFile).getPermission());
- assertTrue("key doesn't exist but should", hbs.exists(validKey));
-
- // test read
- BlobStoreFile readpFile = hbs.read(validKey);
- String readString = IOUtils.toString(readpFile.getInputStream(), "UTF-8");
- assertEquals("string read from blob doesn't match", testString, readString);
-
- // test listkeys
- Iterator<String> keys = hbs.listKeys();
- assertTrue("blob has one key", keys.hasNext());
- assertEquals("one key in blobstore", validKey, keys.next());
-
- // delete
- hbs.deleteKey(validKey);
- assertFalse("key not deleted", fs.exists(dataFile));
- assertFalse("key not deleted", hbs.exists(validKey));
-
- // Now do multiple
- String testString2 = "testingblob2";
- String validKey2= "validkey2";
-
- // test write
- pfile = hbs.write(validKey, false);
- pfile.setMetadata(meta);
- ios = pfile.getOutputStream();
- ios.write(testString.getBytes(Charset.forName("UTF-8")));
- ios.close();
-
- // test commit creates properly
- assertTrue("BlobStore key dir wasn't created", fs.exists(fullKeyDir));
- pfile.commit();
- assertTrue("blob data not committed", fs.exists(dataFile));
- assertEquals("BlobStore dir was created with wrong permissions",
+ assertTrue("key doesn't exist but should", hbs.exists(validKey));
+
+ // test read
+ BlobStoreFile readpFile = hbs.read(validKey);
+ try (InputStream inStream = readpFile.getInputStream()) {
+ String readString = IOUtils.toString(inStream, StandardCharsets.UTF_8);
+ assertEquals("string read from blob doesn't match", testString, readString);
+ }
+
+ // test listkeys
+ Iterator<String> keys = hbs.listKeys();
+ assertTrue("blob has one key", keys.hasNext());
+ assertEquals("one key in blobstore", validKey, keys.next());
+
+ // delete
+ hbs.deleteKey(validKey);
+ assertFalse("key not deleted", fs.exists(dataFile));
+ assertFalse("key not deleted", hbs.exists(validKey));
+
+ // Now do multiple
+ String testString2 = "testingblob2";
+ String validKey2 = "validkey2";
+
+ // test write
+ pfile = hbs.write(validKey, false);
+ pfile.setMetadata(meta);
+ try (OutputStream ios = pfile.getOutputStream()) {
+ ios.write(testString.getBytes(StandardCharsets.UTF_8));
+ }
+
+ // test commit creates properly
+ assertTrue("BlobStore key dir wasn't created", fs.exists(fullKeyDir));
+ pfile.commit();
+ assertTrue("blob data not committed", fs.exists(dataFile));
+ assertEquals("BlobStore dir was created with wrong permissions",
HdfsBlobStoreFile.BLOBSTORE_FILE_PERMISSION, fs.getFileStatus(dataFile).getPermission());
- assertTrue("key doesn't exist but should", hbs.exists(validKey));
-
- // test write again
- pfile = hbs.write(validKey2, false);
- pfile.setMetadata(meta);
- OutputStream ios2 = pfile.getOutputStream();
- ios2.write(testString2.getBytes(Charset.forName("UTF-8")));
- ios2.close();
-
- // test commit second creates properly
- pfile.commit();
- Path dataFile2 = new Path(new Path(fullKeyDir, validKey2), BLOBSTORE_DATA);
- assertTrue("blob data not committed", fs.exists(dataFile2));
- assertEquals("BlobStore dir was created with wrong permissions",
+ assertTrue("key doesn't exist but should", hbs.exists(validKey));
+
+ // test write again
+ pfile = hbs.write(validKey2, false);
+ pfile.setMetadata(meta);
+ try (OutputStream ios2 = pfile.getOutputStream()) {
+ ios2.write(testString2.getBytes(StandardCharsets.UTF_8));
+ }
+
+ // test commit second creates properly
+ pfile.commit();
+ Path dataFile2 = new Path(new Path(fullKeyDir, validKey2), BLOBSTORE_DATA);
+ assertTrue("blob data not committed", fs.exists(dataFile2));
+ assertEquals("BlobStore dir was created with wrong permissions",
HdfsBlobStoreFile.BLOBSTORE_FILE_PERMISSION, fs.getFileStatus(dataFile2).getPermission());
- assertTrue("key doesn't exist but should", hbs.exists(validKey2));
-
- // test listkeys
- keys = hbs.listKeys();
- int total = 0;
- boolean key1Found = false;
- boolean key2Found = false;
- while(keys.hasNext()) {
- total++;
- String key = keys.next();
- if (key.equals(validKey)) {
- key1Found = true;
- } else if (key.equals(validKey2)) {
- key2Found = true;
- } else {
- fail("Found key that wasn't expected: " + key);
+ assertTrue("key doesn't exist but should", hbs.exists(validKey2));
+
+ // test listkeys
+ keys = hbs.listKeys();
+ int total = 0;
+ boolean key1Found = false;
+ boolean key2Found = false;
+ while (keys.hasNext()) {
+ total++;
+ String key = keys.next();
+ if (key.equals(validKey)) {
+ key1Found = true;
+ } else if (key.equals(validKey2)) {
+ key2Found = true;
+ } else {
+ fail("Found key that wasn't expected: " + key);
+ }
}
+ assertEquals("number of keys is wrong", 2, total);
+ assertTrue("blobstore missing key1", key1Found);
+ assertTrue("blobstore missing key2", key2Found);
+
+ // test read
+ readpFile = hbs.read(validKey);
+ try (InputStream inStream = readpFile.getInputStream()) {
+ String readString = IOUtils.toString(inStream, StandardCharsets.UTF_8);
+ assertEquals("string read from blob doesn't match", testString, readString);
+ }
+
+ // test read
+ readpFile = hbs.read(validKey2);
+ try (InputStream inStream = readpFile.getInputStream()) {
+ String readString = IOUtils.toString(inStream, StandardCharsets.UTF_8);
+ assertEquals("string read from blob doesn't match", testString2, readString);
+ }
+
+ hbs.deleteKey(validKey);
+ assertFalse("key not deleted", hbs.exists(validKey));
+ hbs.deleteKey(validKey2);
+ assertFalse("key not deleted", hbs.exists(validKey2));
}
- assertEquals("number of keys is wrong", 2, total);
- assertTrue("blobstore missing key1", key1Found);
- assertTrue("blobstore missing key2", key2Found);
-
- // test read
- readpFile = hbs.read(validKey);
- readString = IOUtils.toString(readpFile.getInputStream(), "UTF-8");
- assertEquals("string read from blob doesn't match", testString, readString);
-
- // test read
- readpFile = hbs.read(validKey2);
- readString = IOUtils.toString(readpFile.getInputStream(), "UTF-8");
- assertEquals("string read from blob doesn't match", testString2, readString);
-
- hbs.deleteKey(validKey);
- assertFalse("key not deleted", hbs.exists(validKey));
- hbs.deleteKey(validKey2);
- assertFalse("key not deleted", hbs.exists(validKey2));
}
@Test
- public void testGetFileLength() throws IOException {
- FileSystem fs = dfscluster.getFileSystem();
- Map conf = new HashMap();
+ public void testGetFileLength() throws Exception {
+ Map<String, Object> conf = new HashMap<>();
String validKey = "validkeyBasic";
String testString = "testingblob";
- TestHdfsBlobStoreImpl hbs = new TestHdfsBlobStoreImpl(blobDir, conf, hadoopConf);
- BlobStoreFile pfile = hbs.write(validKey, false);
- // Adding metadata to avoid null pointer exception
- SettableBlobMeta meta = new SettableBlobMeta();
- meta.set_replication_factor(1);
- pfile.setMetadata(meta);
- OutputStream ios = pfile.getOutputStream();
- ios.write(testString.getBytes(Charset.forName("UTF-8")));
- ios.close();
- assertEquals(testString.getBytes(Charset.forName("UTF-8")).length, pfile.getFileLength());
+ try (TestHdfsBlobStoreImpl hbs = new TestHdfsBlobStoreImpl(blobDir, conf, DFS_CLUSTER_RULE.getHadoopConf())) {
+ BlobStoreFile pfile = hbs.write(validKey, false);
+ // Adding metadata to avoid null pointer exception
+ SettableBlobMeta meta = new SettableBlobMeta();
+ meta.set_replication_factor(1);
+ pfile.setMetadata(meta);
+ try (OutputStream ios = pfile.getOutputStream()) {
+ ios.write(testString.getBytes(StandardCharsets.UTF_8));
+ }
+ assertEquals(testString.getBytes(StandardCharsets.UTF_8).length, pfile.getFileLength());
+ }
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/83adfd9d/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/bolt/AvroGenericRecordBoltTest.java
----------------------------------------------------------------------
diff --git a/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/bolt/AvroGenericRecordBoltTest.java b/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/bolt/AvroGenericRecordBoltTest.java
index cd828da..0b08e1b 100644
--- a/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/bolt/AvroGenericRecordBoltTest.java
+++ b/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/bolt/AvroGenericRecordBoltTest.java
@@ -1,3 +1,4 @@
+
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
@@ -40,49 +41,71 @@ import org.apache.storm.hdfs.bolt.sync.CountSyncPolicy;
import org.apache.storm.hdfs.bolt.sync.SyncPolicy;
import org.junit.Before;
import org.junit.After;
-import org.junit.Ignore;
import org.junit.Test;
import org.junit.Assert;
import org.mockito.Mock;
-import org.mockito.MockitoAnnotations;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.fs.FSDataInputStream;
import java.io.File;
import java.io.FileOutputStream;
import java.io.IOException;
import java.util.HashMap;
-
+import org.apache.hadoop.hdfs.DistributedFileSystem;
+import org.apache.storm.hdfs.testing.MiniDFSClusterRule;
+import org.apache.storm.hdfs.testing.MiniDFSClusterRule.Java7Supplier;
+import org.junit.BeforeClass;
+import org.junit.Rule;
+import org.junit.runner.RunWith;
+import org.mockito.runners.MockitoJUnitRunner;
+
+@RunWith(MockitoJUnitRunner.class)
public class AvroGenericRecordBoltTest {
- private String hdfsURI;
+ @Rule
+ public MiniDFSClusterRule dfsClusterRule = new MiniDFSClusterRule(new Java7Supplier<Configuration>() {
+ @Override
+ public Configuration get() {
+ Configuration conf = new Configuration();
+ conf.set("fs.trash.interval", "10");
+ conf.setBoolean("dfs.permissions", true);
+ File baseDir = new File("./target/hdfs/").getAbsoluteFile();
+ FileUtil.fullyDelete(baseDir);
+ conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, baseDir.getAbsolutePath());
+ return conf;
+ }
+ });
+ @Mock
+ private OutputCollector collector;
+ @Mock
+ private TopologyContext topologyContext;
+
private DistributedFileSystem fs;
- private MiniDFSCluster hdfsCluster;
+ private String hdfsURI;
private static final String testRoot = "/unittest";
- private static final Schema schema1;
- private static final Schema schema2;
- private static final Tuple tuple1;
- private static final Tuple tuple2;
- private static final String schemaV1 = "{\"type\":\"record\"," +
- "\"name\":\"myrecord\"," +
- "\"fields\":[{\"name\":\"foo1\",\"type\":\"string\"}," +
- "{ \"name\":\"int1\", \"type\":\"int\" }]}";
-
- private static final String schemaV2 = "{\"type\":\"record\"," +
- "\"name\":\"myrecord\"," +
- "\"fields\":[{\"name\":\"foo1\",\"type\":\"string\"}," +
- "{ \"name\":\"bar\", \"type\":\"string\", \"default\":\"baz\" }," +
- "{ \"name\":\"int1\", \"type\":\"int\" }]}";
-
- static {
-
+ private static Schema schema1;
+ private static Schema schema2;
+ private static Tuple tuple1;
+ private static Tuple tuple2;
+ private static final String schemaV1 = "{\"type\":\"record\","
+ + "\"name\":\"myrecord\","
+ + "\"fields\":[{\"name\":\"foo1\",\"type\":\"string\"},"
+ + "{ \"name\":\"int1\", \"type\":\"int\" }]}";
+
+ private static final String schemaV2 = "{\"type\":\"record\","
+ + "\"name\":\"myrecord\","
+ + "\"fields\":[{\"name\":\"foo1\",\"type\":\"string\"},"
+ + "{ \"name\":\"bar\", \"type\":\"string\", \"default\":\"baz\" },"
+ + "{ \"name\":\"int1\", \"type\":\"int\" }]}";
+
+ @BeforeClass
+ public static void setupClass() {
Schema.Parser parser = new Schema.Parser();
schema1 = parser.parse(schemaV1);
@@ -100,33 +123,19 @@ public class AvroGenericRecordBoltTest {
tuple2 = generateTestTuple(builder2.build());
}
- @Mock private OutputCollector collector;
- @Mock private TopologyContext topologyContext;
-
@Before
public void setup() throws Exception {
- MockitoAnnotations.initMocks(this);
- Configuration conf = new Configuration();
- conf.set("fs.trash.interval", "10");
- conf.setBoolean("dfs.permissions", true);
- File baseDir = new File("./target/hdfs/").getAbsoluteFile();
- FileUtil.fullyDelete(baseDir);
- conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, baseDir.getAbsolutePath());
-
- MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(conf);
- hdfsCluster = builder.build();
- fs = hdfsCluster.getFileSystem();
+ fs = dfsClusterRule.getDfscluster().getFileSystem();
hdfsURI = fs.getUri() + "/";
}
@After
public void shutDown() throws IOException {
fs.close();
- hdfsCluster.shutdown();
}
- @Test public void multipleTuplesOneFile() throws IOException
- {
+ @Test
+ public void multipleTuplesOneFile() throws IOException {
AvroGenericRecordBolt bolt = makeAvroBolt(hdfsURI, 1, 1f, schemaV1);
bolt.prepare(new Config(), topologyContext, collector);
@@ -139,8 +148,8 @@ public class AvroGenericRecordBoltTest {
verifyAllAvroFiles(testRoot);
}
- @Test public void multipleTuplesMutliplesFiles() throws IOException
- {
+ @Test
+ public void multipleTuplesMutliplesFiles() throws IOException {
AvroGenericRecordBolt bolt = makeAvroBolt(hdfsURI, 1, .0001f, schemaV1);
bolt.prepare(new Config(), topologyContext, collector);
@@ -153,8 +162,8 @@ public class AvroGenericRecordBoltTest {
verifyAllAvroFiles(testRoot);
}
- @Test public void forwardSchemaChangeWorks() throws IOException
- {
+ @Test
+ public void forwardSchemaChangeWorks() throws IOException {
AvroGenericRecordBolt bolt = makeAvroBolt(hdfsURI, 1, 1000f, schemaV1);
bolt.prepare(new Config(), topologyContext, collector);
@@ -167,8 +176,8 @@ public class AvroGenericRecordBoltTest {
verifyAllAvroFiles(testRoot);
}
- @Test public void backwardSchemaChangeWorks() throws IOException
- {
+ @Test
+ public void backwardSchemaChangeWorks() throws IOException {
AvroGenericRecordBolt bolt = makeAvroBolt(hdfsURI, 1, 1000f, schemaV2);
bolt.prepare(new Config(), topologyContext, collector);
@@ -180,8 +189,8 @@ public class AvroGenericRecordBoltTest {
verifyAllAvroFiles(testRoot);
}
- @Test public void schemaThrashing() throws IOException
- {
+ @Test
+ public void schemaThrashing() throws IOException {
AvroGenericRecordBolt bolt = makeAvroBolt(hdfsURI, 1, 1000f, schemaV2);
bolt.prepare(new Config(), topologyContext, collector);
@@ -206,19 +215,19 @@ public class AvroGenericRecordBoltTest {
FileNameFormat fieldsFileNameFormat = new DefaultFileNameFormat().withPath(testRoot);
FileRotationPolicy rotationPolicy =
- new FileSizeRotationPolicy(rotationSizeMB, FileSizeRotationPolicy.Units.MB);
+ new FileSizeRotationPolicy(rotationSizeMB, FileSizeRotationPolicy.Units.MB);
return new AvroGenericRecordBolt()
- .withFsUrl(nameNodeAddr)
- .withFileNameFormat(fieldsFileNameFormat)
- .withRotationPolicy(rotationPolicy)
- .withSyncPolicy(fieldsSyncPolicy);
+ .withFsUrl(nameNodeAddr)
+ .withFileNameFormat(fieldsFileNameFormat)
+ .withRotationPolicy(rotationPolicy)
+ .withSyncPolicy(fieldsSyncPolicy);
}
private static Tuple generateTestTuple(GenericRecord record) {
TopologyBuilder builder = new TopologyBuilder();
GeneralTopologyContext topologyContext = new GeneralTopologyContext(builder.createTopology(),
- new Config(), new HashMap(), new HashMap(), new HashMap(), "") {
+ new Config(), new HashMap(), new HashMap(), new HashMap(), "") {
@Override
public Fields getComponentOutputFields(String componentId, String streamId) {
return new Fields("record");
@@ -250,24 +259,23 @@ public class AvroGenericRecordBoltTest {
return nonZero;
}
- private void fileIsGoodAvro (Path path) throws IOException {
+ private void fileIsGoodAvro(Path path) throws IOException {
DatumReader<GenericRecord> datumReader = new GenericDatumReader<>();
- FSDataInputStream in = fs.open(path, 0);
- FileOutputStream out = new FileOutputStream("target/FOO.avro");
-
- byte[] buffer = new byte[100];
- int bytesRead;
- while ((bytesRead = in.read(buffer)) > 0) {
- out.write(buffer, 0, bytesRead);
+ try (FSDataInputStream in = fs.open(path, 0); FileOutputStream out = new FileOutputStream("target/FOO.avro")) {
+ byte[] buffer = new byte[100];
+ int bytesRead;
+ while ((bytesRead = in.read(buffer)) > 0) {
+ out.write(buffer, 0, bytesRead);
+ }
}
- out.close();
java.io.File file = new File("target/FOO.avro");
- DataFileReader<GenericRecord> dataFileReader = new DataFileReader<>(file, datumReader);
- GenericRecord user = null;
- while (dataFileReader.hasNext()) {
- user = dataFileReader.next(user);
+ try (DataFileReader<GenericRecord> dataFileReader = new DataFileReader<>(file, datumReader)) {
+ GenericRecord user = null;
+ while (dataFileReader.hasNext()) {
+ user = dataFileReader.next(user);
+ }
}
file.delete();
http://git-wip-us.apache.org/repos/asf/storm/blob/83adfd9d/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/bolt/TestHdfsBolt.java
----------------------------------------------------------------------
diff --git a/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/bolt/TestHdfsBolt.java b/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/bolt/TestHdfsBolt.java
index e8f0702..9386c9f 100644
--- a/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/bolt/TestHdfsBolt.java
+++ b/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/bolt/TestHdfsBolt.java
@@ -1,3 +1,4 @@
+
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
@@ -44,7 +45,7 @@ import org.junit.Assert;
import org.junit.rules.ExpectedException;
import org.mockito.Mock;
-import org.mockito.MockitoAnnotations;
+
import static org.mockito.Mockito.*;
import org.apache.hadoop.conf.Configuration;
@@ -59,40 +60,50 @@ import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
import java.io.File;
import java.io.IOException;
import java.util.HashMap;
+import org.apache.storm.hdfs.testing.MiniDFSClusterRule;
+import org.junit.runner.RunWith;
+import org.mockito.Mockito;
+import org.mockito.runners.MockitoJUnitRunner;
+@RunWith(MockitoJUnitRunner.class)
public class TestHdfsBolt {
+ @Rule
+ public MiniDFSClusterRule dfsClusterRule = new MiniDFSClusterRule(new MiniDFSClusterRule.Java7Supplier<Configuration>() {
+ @Override
+ public Configuration get() {
+ Configuration conf = new Configuration();
+ conf.set("fs.trash.interval", "10");
+ conf.setBoolean("dfs.permissions", true);
+ File baseDir = new File("./target/hdfs/").getAbsoluteFile();
+ FileUtil.fullyDelete(baseDir);
+ conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, baseDir.getAbsolutePath());
+ return conf;
+ }
+ });
+
private String hdfsURI;
private DistributedFileSystem fs;
- private MiniDFSCluster hdfsCluster;
private static final String testRoot = "/unittest";
Tuple tuple1 = generateTestTuple(1, "First Tuple", "SFO", "CA");
Tuple tuple2 = generateTestTuple(1, "Second Tuple", "SJO", "CA");
- @Mock private OutputCollector collector;
- @Mock private TopologyContext topologyContext;
- @Rule public ExpectedException thrown = ExpectedException.none();
+ @Mock
+ private OutputCollector collector;
+ @Mock
+ private TopologyContext topologyContext;
+ @Rule
+ public ExpectedException thrown = ExpectedException.none();
@Before
public void setup() throws Exception {
- MockitoAnnotations.initMocks(this);
- Configuration conf = new Configuration();
- conf.set("fs.trash.interval", "10");
- conf.setBoolean("dfs.permissions", true);
- File baseDir = new File("./target/hdfs/").getAbsoluteFile();
- FileUtil.fullyDelete(baseDir);
- conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, baseDir.getAbsolutePath());
-
- MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(conf);
- hdfsCluster = builder.build();
- fs = hdfsCluster.getFileSystem();
- hdfsURI = "hdfs://localhost:" + hdfsCluster.getNameNodePort() + "/";
+ fs = dfsClusterRule.getDfscluster().getFileSystem();
+ hdfsURI = "hdfs://localhost:" + dfsClusterRule.getDfscluster().getNameNodePort() + "/";
}
@After
public void shutDown() throws IOException {
fs.close();
- hdfsCluster.shutdown();
}
@Test
@@ -134,8 +145,7 @@ public class TestHdfsBolt {
}
@Test
- public void testTwoTuplesOneFile() throws IOException
- {
+ public void testTwoTuplesOneFile() throws IOException {
HdfsBolt bolt = makeHdfsBolt(hdfsURI, 2, 10000f);
bolt.prepare(new Config(), topologyContext, collector);
bolt.execute(tuple1);
@@ -150,8 +160,7 @@ public class TestHdfsBolt {
}
@Test
- public void testFailedSync() throws IOException
- {
+ public void testFailedSync() throws IOException {
HdfsBolt bolt = makeHdfsBolt(hdfsURI, 2, 10000f);
bolt.prepare(new Config(), topologyContext, collector);
bolt.execute(tuple1);
@@ -167,27 +176,23 @@ public class TestHdfsBolt {
// One tuple and one rotation should yield one file with data
// The failed executions should not cause rotations and any new files
@Test
- public void testFailureFilecount() throws IOException, InterruptedException
- {
+ public void testFailureFilecount() throws IOException, InterruptedException {
HdfsBolt bolt = makeHdfsBolt(hdfsURI, 1, .000001f);
bolt.prepare(new Config(), topologyContext, collector);
bolt.execute(tuple1);
fs.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
- try
- {
+ try {
bolt.execute(tuple2);
} catch (RuntimeException e) {
//
}
- try
- {
+ try {
bolt.execute(tuple2);
} catch (RuntimeException e) {
//
}
- try
- {
+ try {
bolt.execute(tuple2);
} catch (RuntimeException e) {
//
@@ -198,8 +203,7 @@ public class TestHdfsBolt {
}
@Test
- public void testTickTuples() throws IOException
- {
+ public void testTickTuples() throws IOException {
HdfsBolt bolt = makeHdfsBolt(hdfsURI, 10, 10000f);
bolt.prepare(new Config(), topologyContext, collector);
@@ -226,36 +230,28 @@ public class TestHdfsBolt {
SyncPolicy fieldsSyncPolicy = new CountSyncPolicy(countSync);
FileRotationPolicy fieldsRotationPolicy =
- new FileSizeRotationPolicy(rotationSizeMB, FileSizeRotationPolicy.Units.MB);
+ new FileSizeRotationPolicy(rotationSizeMB, FileSizeRotationPolicy.Units.MB);
FileNameFormat fieldsFileNameFormat = new DefaultFileNameFormat().withPath(testRoot);
return new HdfsBolt()
- .withFsUrl(nameNodeAddr)
- .withFileNameFormat(fieldsFileNameFormat)
- .withRecordFormat(fieldsFormat)
- .withRotationPolicy(fieldsRotationPolicy)
- .withSyncPolicy(fieldsSyncPolicy);
+ .withFsUrl(nameNodeAddr)
+ .withFileNameFormat(fieldsFileNameFormat)
+ .withRecordFormat(fieldsFormat)
+ .withRotationPolicy(fieldsRotationPolicy)
+ .withSyncPolicy(fieldsSyncPolicy);
}
- private Tuple generateTestTuple(Object id, Object msg,Object city,Object state) {
+ private Tuple generateTestTuple(Object id, Object msg, Object city, Object state) {
TopologyBuilder builder = new TopologyBuilder();
GeneralTopologyContext topologyContext = new GeneralTopologyContext(builder.createTopology(),
- new Config(), new HashMap(), new HashMap(), new HashMap(), "") {
+ new Config(), new HashMap(), new HashMap(), new HashMap(), "") {
@Override
public Fields getComponentOutputFields(String componentId, String streamId) {
- return new Fields("id", "msg","city","state");
+ return new Fields("id", "msg", "city", "state");
}
};
- return new TupleImpl(topologyContext, new Values(id, msg,city,state), 1, "");
- }
-
- private void printFiles(String path) throws IOException {
- Path p = new Path(path);
- FileStatus[] fileStatuses = fs.listStatus(p);
- for (FileStatus file : fileStatuses) {
- System.out.println("@@@ " + file.getPath() + " [" + file.getLen() + "]");
- }
+ return new TupleImpl(topologyContext, new Values(id, msg, city, state), 1, "");
}
// Generally used to compare how files were actually written and compare to expectations based on total
@@ -264,9 +260,11 @@ public class TestHdfsBolt {
Path p = new Path(path);
int nonZero = 0;
- for (FileStatus file : fs.listStatus(p))
- if (file.getLen() > 0)
+ for (FileStatus file : fs.listStatus(p)) {
+ if (file.getLen() > 0) {
nonZero++;
+ }
+ }
return nonZero;
}
@@ -275,9 +273,11 @@ public class TestHdfsBolt {
Path p = new Path(path);
int zeroLength = 0;
- for (FileStatus file : fs.listStatus(p))
- if (file.getLen() == 0)
+ for (FileStatus file : fs.listStatus(p)) {
+ if (file.getLen() == 0) {
zeroLength++;
+ }
+ }
return zeroLength;
}
http://git-wip-us.apache.org/repos/asf/storm/blob/83adfd9d/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/bolt/TestSequenceFileBolt.java
----------------------------------------------------------------------
diff --git a/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/bolt/TestSequenceFileBolt.java b/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/bolt/TestSequenceFileBolt.java
index 9913d9d..7fee578 100644
--- a/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/bolt/TestSequenceFileBolt.java
+++ b/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/bolt/TestSequenceFileBolt.java
@@ -1,3 +1,4 @@
+
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
@@ -17,8 +18,10 @@
*/
package org.apache.storm.hdfs.bolt;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.verifyZeroInteractions;
+
import org.apache.storm.Config;
-import org.apache.storm.Constants;
import org.apache.storm.task.GeneralTopologyContext;
import org.apache.storm.task.OutputCollector;
import org.apache.storm.task.TopologyContext;
@@ -39,13 +42,10 @@ import org.junit.*;
import org.junit.rules.ExpectedException;
import org.mockito.Mock;
-import org.mockito.MockitoAnnotations;
-import static org.mockito.Mockito.*;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.FileUtil;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.DistributedFileSystem;
@@ -56,42 +56,51 @@ import org.slf4j.LoggerFactory;
import java.io.File;
import java.io.IOException;
import java.util.HashMap;
+import org.apache.storm.hdfs.testing.MiniDFSClusterRule;
+import org.junit.runner.RunWith;
+import org.mockito.runners.MockitoJUnitRunner;
+@RunWith(MockitoJUnitRunner.class)
public class TestSequenceFileBolt {
private static final Logger LOG = LoggerFactory.getLogger(TestSequenceFileBolt.class);
+ @Rule
+ public MiniDFSClusterRule dfsClusterRule = new MiniDFSClusterRule(new MiniDFSClusterRule.Java7Supplier<Configuration>() {
+ @Override
+ public Configuration get() {
+ Configuration conf = new Configuration();
+ conf.set("fs.trash.interval", "10");
+ conf.setBoolean("dfs.permissions", true);
+ File baseDir = new File("./target/hdfs/").getAbsoluteFile();
+ FileUtil.fullyDelete(baseDir);
+ conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, baseDir.getAbsolutePath());
+ return conf;
+ }
+ });
+
private String hdfsURI;
private DistributedFileSystem fs;
- private MiniDFSCluster hdfsCluster;
private static final String testRoot = "/unittest";
Tuple tuple1 = generateTestTuple(1l, "first tuple");
Tuple tuple2 = generateTestTuple(2l, "second tuple");
- @Mock private OutputCollector collector;
- @Mock private TopologyContext topologyContext;
- @Rule public ExpectedException thrown = ExpectedException.none();
+ @Mock
+ private OutputCollector collector;
+ @Mock
+ private TopologyContext topologyContext;
+ @Rule
+ public ExpectedException thrown = ExpectedException.none();
@Before
public void setup() throws Exception {
- MockitoAnnotations.initMocks(this);
- Configuration conf = new Configuration();
- conf.set("fs.trash.interval", "10");
- conf.setBoolean("dfs.permissions", true);
- File baseDir = new File("./target/hdfs/").getAbsoluteFile();
- FileUtil.fullyDelete(baseDir);
- conf.set(MiniDFSCluster.HDFS_MINIDFS_BASEDIR, baseDir.getAbsolutePath());
-
- MiniDFSCluster.Builder builder = new MiniDFSCluster.Builder(conf);
- hdfsCluster = builder.build();
- fs = hdfsCluster.getFileSystem();
- hdfsURI = "hdfs://localhost:" + hdfsCluster.getNameNodePort() + "/";
+ fs = dfsClusterRule.getDfscluster().getFileSystem();
+ hdfsURI = "hdfs://localhost:" + dfsClusterRule.getDfscluster().getNameNodePort() + "/";
}
@After
public void shutDown() throws IOException {
fs.close();
- hdfsCluster.shutdown();
}
@Test
@@ -109,8 +118,7 @@ public class TestSequenceFileBolt {
}
@Test
- public void testTwoTuplesOneFile() throws IOException
- {
+ public void testTwoTuplesOneFile() throws IOException {
SequenceFileBolt bolt = makeSeqBolt(hdfsURI, 2, 10000f);
bolt.prepare(new Config(), topologyContext, collector);
bolt.execute(tuple1);
@@ -125,8 +133,7 @@ public class TestSequenceFileBolt {
}
@Test
- public void testFailedSync() throws IOException
- {
+ public void testFailedSync() throws IOException {
SequenceFileBolt bolt = makeSeqBolt(hdfsURI, 2, 10000f);
bolt.prepare(new Config(), topologyContext, collector);
bolt.execute(tuple1);
@@ -142,24 +149,24 @@ public class TestSequenceFileBolt {
SyncPolicy fieldsSyncPolicy = new CountSyncPolicy(countSync);
FileRotationPolicy fieldsRotationPolicy =
- new FileSizeRotationPolicy(rotationSizeMB, FileSizeRotationPolicy.Units.MB);
+ new FileSizeRotationPolicy(rotationSizeMB, FileSizeRotationPolicy.Units.MB);
FileNameFormat fieldsFileNameFormat = new DefaultFileNameFormat().withPath(testRoot);
SequenceFormat seqFormat = new DefaultSequenceFormat("key", "value");
return new SequenceFileBolt()
- .withFsUrl(nameNodeAddr)
- .withFileNameFormat(fieldsFileNameFormat)
- .withRotationPolicy(fieldsRotationPolicy)
- .withSequenceFormat(seqFormat)
- .withSyncPolicy(fieldsSyncPolicy);
+ .withFsUrl(nameNodeAddr)
+ .withFileNameFormat(fieldsFileNameFormat)
+ .withRotationPolicy(fieldsRotationPolicy)
+ .withSequenceFormat(seqFormat)
+ .withSyncPolicy(fieldsSyncPolicy);
}
private Tuple generateTestTuple(Long key, String value) {
TopologyBuilder builder = new TopologyBuilder();
GeneralTopologyContext topologyContext = new GeneralTopologyContext(builder.createTopology(),
- new Config(), new HashMap(), new HashMap(), new HashMap(), "") {
+ new Config(), new HashMap(), new HashMap(), new HashMap(), "") {
@Override
public Fields getComponentOutputFields(String componentId, String streamId) {
return new Fields("key", "value");
[42/50] [abbrv] storm git commit: STORM-2810: Fix resource leaks in
storm-hdfs tests
Posted by pt...@apache.org.
STORM-2810: Fix resource leaks in storm-hdfs tests
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/83adfd9d
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/83adfd9d
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/83adfd9d
Branch: refs/heads/metrics_v2
Commit: 83adfd9d2c5c352efeb076303cacab38f9f5c01c
Parents: 7e17712
Author: Stig Rohde Døssing <sr...@apache.org>
Authored: Sat Nov 11 18:23:00 2017 +0100
Committer: Stig Rohde Døssing <sr...@apache.org>
Committed: Tue Nov 14 07:50:09 2017 +0100
----------------------------------------------------------------------
.../storm/hdfs/blobstore/HdfsBlobStoreImpl.java | 14 +-
.../storm/hdfs/bolt/AbstractHdfsBolt.java | 48 +-
.../org/apache/storm/hdfs/spout/HdfsSpout.java | 1484 +++++++++---------
.../hdfs/avro/TestFixedAvroSerializer.java | 13 +-
.../hdfs/avro/TestGenericAvroSerializer.java | 11 +-
.../storm/hdfs/blobstore/BlobStoreTest.java | 897 ++++++-----
.../hdfs/blobstore/HdfsBlobStoreImplTest.java | 288 ++--
.../hdfs/bolt/AvroGenericRecordBoltTest.java | 142 +-
.../apache/storm/hdfs/bolt/TestHdfsBolt.java | 108 +-
.../storm/hdfs/bolt/TestSequenceFileBolt.java | 71 +-
.../apache/storm/hdfs/spout/TestDirLock.java | 281 ++--
.../apache/storm/hdfs/spout/TestFileLock.java | 679 ++++----
.../storm/hdfs/spout/TestHdfsSemantics.java | 311 ++--
.../apache/storm/hdfs/spout/TestHdfsSpout.java | 1235 +++++++--------
.../storm/hdfs/spout/TestProgressTracker.java | 179 +--
.../storm/hdfs/testing/MiniDFSClusterRule.java | 78 +
16 files changed, 2955 insertions(+), 2884 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/83adfd9d/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/blobstore/HdfsBlobStoreImpl.java
----------------------------------------------------------------------
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/blobstore/HdfsBlobStoreImpl.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/blobstore/HdfsBlobStoreImpl.java
index a4c88ce..e90e2d3 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/blobstore/HdfsBlobStoreImpl.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/blobstore/HdfsBlobStoreImpl.java
@@ -45,9 +45,10 @@ public class HdfsBlobStoreImpl {
private static final long FULL_CLEANUP_FREQ = 60 * 60 * 1000l;
private static final int BUCKETS = 1024;
- private static final Timer timer = new Timer("HdfsBlobStore cleanup thread", true);
private static final String BLOBSTORE_DATA = "data";
+ private Timer timer;
+
public class KeyInHashDirIterator implements Iterator<String> {
private int currentBucket = 0;
private Iterator<String> it = null;
@@ -112,7 +113,6 @@ public class HdfsBlobStoreImpl {
private Path _fullPath;
private FileSystem _fs;
- private TimerTask _cleanup = null;
private Configuration _hadoopConf;
// blobstore directory is private!
@@ -141,7 +141,7 @@ public class HdfsBlobStoreImpl {
Object shouldCleanup = conf.get(Config.BLOBSTORE_CLEANUP_ENABLE);
if (Utils.getBoolean(shouldCleanup, false)) {
LOG.debug("Starting hdfs blobstore cleaner");
- _cleanup = new TimerTask() {
+ TimerTask cleanup = new TimerTask() {
@Override
public void run() {
try {
@@ -151,7 +151,8 @@ public class HdfsBlobStoreImpl {
}
}
};
- timer.scheduleAtFixedRate(_cleanup, 0, FULL_CLEANUP_FREQ);
+ timer = new Timer("HdfsBlobStore cleanup thread", true);
+ timer.scheduleAtFixedRate(cleanup, 0, FULL_CLEANUP_FREQ);
}
}
@@ -304,9 +305,8 @@ public class HdfsBlobStoreImpl {
}
public void shutdown() {
- if (_cleanup != null) {
- _cleanup.cancel();
- _cleanup = null;
+ if (timer != null) {
+ timer.cancel();
}
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/83adfd9d/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/AbstractHdfsBolt.java
----------------------------------------------------------------------
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/AbstractHdfsBolt.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/AbstractHdfsBolt.java
index 4503191..aabd2cc 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/AbstractHdfsBolt.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/bolt/AbstractHdfsBolt.java
@@ -1,3 +1,4 @@
+
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
@@ -48,6 +49,7 @@ import java.util.Timer;
import java.util.TimerTask;
public abstract class AbstractHdfsBolt extends BaseRichBolt {
+
private static final Logger LOG = LoggerFactory.getLogger(AbstractHdfsBolt.class);
private static final Integer DEFAULT_RETRY_COUNT = 3;
/**
@@ -94,14 +96,19 @@ public abstract class AbstractHdfsBolt extends BaseRichBolt {
/**
* Marked as final to prevent override. Subclasses should implement the doPrepare() method.
+ *
* @param conf
* @param topologyContext
* @param collector
*/
- public final void prepare(Map conf, TopologyContext topologyContext, OutputCollector collector){
+ public final void prepare(Map conf, TopologyContext topologyContext, OutputCollector collector) {
this.writeLock = new Object();
- if (this.syncPolicy == null) throw new IllegalStateException("SyncPolicy must be specified.");
- if (this.rotationPolicy == null) throw new IllegalStateException("RotationPolicy must be specified.");
+ if (this.syncPolicy == null) {
+ throw new IllegalStateException("SyncPolicy must be specified.");
+ }
+ if (this.rotationPolicy == null) {
+ throw new IllegalStateException("RotationPolicy must be specified.");
+ }
if (this.fsUrl == null) {
throw new IllegalStateException("File system URL must be specified.");
}
@@ -111,21 +118,21 @@ public abstract class AbstractHdfsBolt extends BaseRichBolt {
this.collector = collector;
this.fileNameFormat.prepare(conf, topologyContext);
this.hdfsConfig = new Configuration();
- Map<String, Object> map = (Map<String, Object>)conf.get(this.configKey);
- if(map != null){
- for(String key : map.keySet()){
+ Map<String, Object> map = (Map<String, Object>) conf.get(this.configKey);
+ if (map != null) {
+ for (String key : map.keySet()) {
this.hdfsConfig.set(key, String.valueOf(map.get(key)));
}
}
- try{
+ try {
HdfsSecurityUtil.login(conf, hdfsConfig);
doPrepare(conf, topologyContext, collector);
- } catch (Exception e){
+ } catch (Exception e) {
throw new RuntimeException("Error preparing HdfsBolt: " + e.getMessage(), e);
}
- if(this.rotationPolicy instanceof TimedRotationPolicy){
+ if (this.rotationPolicy instanceof TimedRotationPolicy) {
startTimedRotationPolicy();
}
}
@@ -214,10 +221,8 @@ public abstract class AbstractHdfsBolt extends BaseRichBolt {
}
/**
- * A tuple must be mapped to a writer based on two factors:
- * - bolt specific logic that must separate tuples into different files in the same directory (see the avro bolt
- * for an example of this)
- * - the directory the tuple will be partioned into
+ * A tuple must be mapped to a writer based on two factors: - bolt specific logic that must separate tuples into different files in the
+ * same directory (see the avro bolt for an example of this) - the directory the tuple will be partioned into
*
* @param tuple
* @return
@@ -251,6 +256,11 @@ public abstract class AbstractHdfsBolt extends BaseRichBolt {
public void declareOutputFields(OutputFieldsDeclarer outputFieldsDeclarer) {
}
+ @Override
+ public void cleanup() {
+ this.rotationTimer.cancel();
+ }
+
private void syncAllWriters() throws IOException {
for (Writer writer : writers.values()) {
writer.sync();
@@ -258,7 +268,7 @@ public abstract class AbstractHdfsBolt extends BaseRichBolt {
}
private void startTimedRotationPolicy() {
- long interval = ((TimedRotationPolicy)this.rotationPolicy).getInterval();
+ long interval = ((TimedRotationPolicy) this.rotationPolicy).getInterval();
this.rotationTimer = new Timer(true);
TimerTask task = new TimerTask() {
@Override
@@ -280,8 +290,7 @@ public abstract class AbstractHdfsBolt extends BaseRichBolt {
final String partitionPath = this.partitioner.getPartitionPath(tuple);
final int rotation;
- if (rotationCounterMap.containsKey(partitionPath))
- {
+ if (rotationCounterMap.containsKey(partitionPath)) {
rotation = rotationCounterMap.get(partitionPath) + 1;
} else {
rotation = 0;
@@ -289,7 +298,7 @@ public abstract class AbstractHdfsBolt extends BaseRichBolt {
rotationCounterMap.put(partitionPath, rotation);
return new Path(this.fsUrl + this.fileNameFormat.getPath() + partitionPath,
- this.fileNameFormat.getName(rotation, System.currentTimeMillis()));
+ this.fileNameFormat.getName(rotation, System.currentTimeMillis()));
}
abstract protected void doPrepare(Map conf, TopologyContext topologyContext, OutputCollector collector) throws IOException;
@@ -299,10 +308,11 @@ public abstract class AbstractHdfsBolt extends BaseRichBolt {
abstract protected Writer makeNewWriter(Path path, Tuple tuple) throws IOException;
static class WritersMap extends LinkedHashMap<String, Writer> {
+
final long maxWriters;
public WritersMap(long maxWriters) {
- super((int)maxWriters, 0.75f, true);
+ super((int) maxWriters, 0.75f, true);
this.maxWriters = maxWriters;
}
@@ -311,4 +321,4 @@ public abstract class AbstractHdfsBolt extends BaseRichBolt {
return this.size() > this.maxWriters;
}
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/83adfd9d/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/HdfsSpout.java
----------------------------------------------------------------------
diff --git a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/HdfsSpout.java b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/HdfsSpout.java
index 1d8bcad..0711623 100644
--- a/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/HdfsSpout.java
+++ b/external/storm-hdfs/src/main/java/org/apache/storm/hdfs/spout/HdfsSpout.java
@@ -1,3 +1,4 @@
+
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
@@ -9,13 +10,10 @@
* <p/>
* http://www.apache.org/licenses/LICENSE-2.0
* <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
-
package org.apache.storm.hdfs.spout;
import java.io.IOException;
@@ -47,800 +45,810 @@ import org.apache.storm.tuple.Fields;
public class HdfsSpout extends BaseRichSpout {
- // user configurable
- private String hdfsUri; // required
- private String readerType; // required
- private Fields outputFields; // required
+ // user configurable
+ private String hdfsUri; // required
+ private String readerType; // required
+ private Fields outputFields; // required
- private String sourceDir; // required
- private Path sourceDirPath; // required
+ private String sourceDir; // required
+ private Path sourceDirPath; // required
- private String archiveDir; // required
- private Path archiveDirPath; // required
+ private String archiveDir; // required
+ private Path archiveDirPath; // required
- private String badFilesDir; // required
- private Path badFilesDirPath; // required
+ private String badFilesDir; // required
+ private Path badFilesDirPath; // required
- private String lockDir;
- private Path lockDirPath;
+ private String lockDir;
+ private Path lockDirPath;
- private int commitFrequencyCount = Configs.DEFAULT_COMMIT_FREQ_COUNT;
- private int commitFrequencySec = Configs.DEFAULT_COMMIT_FREQ_SEC;
- private int maxOutstanding = Configs.DEFAULT_MAX_OUTSTANDING;
- private int lockTimeoutSec = Configs.DEFAULT_LOCK_TIMEOUT;
- private boolean clocksInSync = true;
+ private int commitFrequencyCount = Configs.DEFAULT_COMMIT_FREQ_COUNT;
+ private int commitFrequencySec = Configs.DEFAULT_COMMIT_FREQ_SEC;
+ private int maxOutstanding = Configs.DEFAULT_MAX_OUTSTANDING;
+ private int lockTimeoutSec = Configs.DEFAULT_LOCK_TIMEOUT;
+ private boolean clocksInSync = true;
- private String inprogress_suffix = ".inprogress"; // not configurable to prevent change between topology restarts
- private String ignoreSuffix = ".ignore";
+ private String inprogress_suffix = ".inprogress"; // not configurable to prevent change between topology restarts
+ private String ignoreSuffix = ".ignore";
- private String outputStreamName= null;
+ private String outputStreamName = null;
- // other members
- private static final Logger LOG = LoggerFactory.getLogger(HdfsSpout.class);
+ // other members
+ private static final Logger LOG = LoggerFactory.getLogger(HdfsSpout.class);
- private ProgressTracker tracker = null;
+ private ProgressTracker tracker = null;
- private FileSystem hdfs;
- private FileReader reader;
+ private FileSystem hdfs;
+ private FileReader reader;
- private SpoutOutputCollector collector;
- HashMap<MessageId, List<Object> > inflight = new HashMap<>();
- LinkedBlockingQueue<HdfsUtils.Pair<MessageId, List<Object>>> retryList = new LinkedBlockingQueue<>();
-
- private Configuration hdfsConfig;
-
- private Map conf = null;
- private FileLock lock;
- private String spoutId = null;
-
- HdfsUtils.Pair<Path,FileLock.LogEntry> lastExpiredLock = null;
- private long lastExpiredLockTime = 0;
-
- private long tupleCounter = 0;
- private boolean ackEnabled = false;
- private int acksSinceLastCommit = 0 ;
- private final AtomicBoolean commitTimeElapsed = new AtomicBoolean(false);
- private Timer commitTimer;
- private boolean fileReadCompletely = true;
-
- private String configKey = Configs.DEFAULT_HDFS_CONFIG_KEY; // key for hdfs Kerberos configs
-
- public HdfsSpout() {
- }
-
- public HdfsSpout setHdfsUri(String hdfsUri) {
- this.hdfsUri = hdfsUri;
- return this;
- }
-
- public HdfsSpout setReaderType(String readerType) {
- this.readerType = readerType;
- return this;
- }
-
- public HdfsSpout setSourceDir(String sourceDir) {
- this.sourceDir = sourceDir;
- return this;
- }
-
- public HdfsSpout setArchiveDir(String archiveDir) {
- this.archiveDir = archiveDir;
- return this;
- }
-
- public HdfsSpout setBadFilesDir(String badFilesDir) {
- this.badFilesDir = badFilesDir;
- return this;
- }
-
- public HdfsSpout setLockDir(String lockDir) {
- this.lockDir = lockDir;
- return this;
- }
-
- public HdfsSpout setCommitFrequencyCount(int commitFrequencyCount) {
- this.commitFrequencyCount = commitFrequencyCount;
- return this;
- }
-
- public HdfsSpout setCommitFrequencySec(int commitFrequencySec) {
- this.commitFrequencySec = commitFrequencySec;
- return this;
- }
-
- public HdfsSpout setMaxOutstanding(int maxOutstanding) {
- this.maxOutstanding = maxOutstanding;
- return this;
- }
-
- public HdfsSpout setLockTimeoutSec(int lockTimeoutSec) {
- this.lockTimeoutSec = lockTimeoutSec;
- return this;
- }
-
- public HdfsSpout setClocksInSync(boolean clocksInSync) {
- this.clocksInSync = clocksInSync;
- return this;
- }
-
-
- public HdfsSpout setIgnoreSuffix(String ignoreSuffix) {
- this.ignoreSuffix = ignoreSuffix;
- return this;
- }
-
- /** Output field names. Number of fields depends upon the reader type */
- public HdfsSpout withOutputFields(String... fields) {
- outputFields = new Fields(fields);
- return this;
- }
-
- /** set key name under which HDFS options are placed. (similar to HDFS bolt).
- * default key name is 'hdfs.config' */
- public HdfsSpout withConfigKey(String configKey) {
- this.configKey = configKey;
- return this;
- }
-
- /**
- * Set output stream name
- */
- public HdfsSpout withOutputStream(String streamName) {
- this.outputStreamName = streamName;
- return this;
- }
-
- public Path getLockDirPath() {
- return lockDirPath;
- }
-
- public SpoutOutputCollector getCollector() {
- return collector;
- }
-
- public void nextTuple() {
- LOG.trace("Next Tuple {}", spoutId);
- // 1) First re-emit any previously failed tuples (from retryList)
- if (!retryList.isEmpty()) {
- LOG.debug("Sending tuple from retry list");
- HdfsUtils.Pair<MessageId, List<Object>> pair = retryList.remove();
- emitData(pair.getValue(), pair.getKey());
- return;
- }
-
- if ( ackEnabled && tracker.size()>= maxOutstanding ) {
- LOG.warn("Waiting for more ACKs before generating new tuples. " +
- "Progress tracker size has reached limit {}, SpoutID {}"
- , maxOutstanding, spoutId);
- // Don't emit anything .. allow configured spout wait strategy to kick in
- return;
- }
-
- // 2) If no failed tuples to be retried, then send tuples from hdfs
- while (true) {
- try {
- // 3) Select a new file if one is not open already
- if (reader == null) {
- reader = pickNextFile();
- if (reader == null) {
- LOG.debug("Currently no new files to process under : " + sourceDirPath);
- return;
- } else {
- fileReadCompletely=false;
- }
- }
- if ( fileReadCompletely ) { // wait for more ACKs before proceeding
- return;
- }
- // 4) Read record from file, emit to collector and record progress
- List<Object> tuple = reader.next();
- if (tuple != null) {
- fileReadCompletely= false;
- ++tupleCounter;
- MessageId msgId = new MessageId(tupleCounter, reader.getFilePath(), reader.getFileOffset());
- emitData(tuple, msgId);
-
- if ( !ackEnabled ) {
- ++acksSinceLastCommit; // assume message is immediately ACKed in non-ack mode
- commitProgress(reader.getFileOffset());
- } else {
- commitProgress(tracker.getCommitPosition());
- }
- return;
- } else {
- fileReadCompletely = true;
- if ( !ackEnabled ) {
- markFileAsDone(reader.getFilePath());
- }
- }
- } catch (IOException e) {
- LOG.error("I/O Error processing at file location " + getFileProgress(reader), e);
- // don't emit anything .. allow configured spout wait strategy to kick in
- return;
- } catch (ParseException e) {
- LOG.error("Parsing error when processing at file location " + getFileProgress(reader) +
- ". Skipping remainder of file.", e);
- markFileAsBad(reader.getFilePath());
- // Note: We don't return from this method on ParseException to avoid triggering the
- // spout wait strategy (due to no emits). Instead we go back into the loop and
- // generate a tuple from next file
- }
- } // while
- }
-
- // will commit progress into lock file if commit threshold is reached
- private void commitProgress(FileOffset position) {
- if ( position==null ) {
- return;
- }
- if ( lock!=null && canCommitNow() ) {
- try {
- String pos = position.toString();
- lock.heartbeat(pos);
- LOG.debug("{} Committed progress. {}", spoutId, pos);
- acksSinceLastCommit = 0;
- commitTimeElapsed.set(false);
- setupCommitElapseTimer();
- } catch (IOException e) {
- LOG.error("Unable to commit progress Will retry later. Spout ID = " + spoutId, e);
- }
- }
- }
-
- private void setupCommitElapseTimer() {
- if ( commitFrequencySec<=0 ) {
- return;
- }
- TimerTask timerTask = new TimerTask() {
- @Override
- public void run() {
- commitTimeElapsed.set(true);
- }
- };
- commitTimer.schedule(timerTask, commitFrequencySec * 1000);
- }
-
- private static String getFileProgress(FileReader reader) {
- return reader.getFilePath() + " " + reader.getFileOffset();
- }
-
- private void markFileAsDone(Path filePath) {
- try {
- Path newFile = renameCompletedFile(reader.getFilePath());
- LOG.info("Completed processing {}. Spout Id = {}", newFile, spoutId);
- } catch (IOException e) {
- LOG.error("Unable to archive completed file" + filePath + " Spout ID " + spoutId, e);
- }
- closeReaderAndResetTrackers();
- }
-
- private void markFileAsBad(Path file) {
- String fileName = file.toString();
- String fileNameMinusSuffix = fileName.substring(0, fileName.indexOf(inprogress_suffix));
- String originalName = new Path(fileNameMinusSuffix).getName();
- Path newFile = new Path( badFilesDirPath + Path.SEPARATOR + originalName);
-
- LOG.info("Moving bad file {} to {}. Processed it till offset {}. SpoutID= {}", originalName, newFile, tracker.getCommitPosition(), spoutId);
- try {
- if (!hdfs.rename(file, newFile) ) { // seems this can fail by returning false or throwing exception
- throw new IOException("Move failed for bad file: " + file); // convert false ret value to exception
- }
- } catch (IOException e) {
- LOG.warn("Error moving bad file: " + file + " to destination " + newFile + " SpoutId =" + spoutId, e);
- }
- closeReaderAndResetTrackers();
- }
-
- private void closeReaderAndResetTrackers() {
- inflight.clear();
- tracker.offsets.clear();
- retryList.clear();
-
- reader.close();
- reader = null;
- releaseLockAndLog(lock, spoutId);
- lock = null;
- }
-
- private static void releaseLockAndLog(FileLock fLock, String spoutId) {
- try {
- if ( fLock!=null ) {
- fLock.release();
- LOG.debug("Spout {} released FileLock. SpoutId = {}", fLock.getLockFile(), spoutId);
- }
- } catch (IOException e) {
- LOG.error("Unable to delete lock file : " +fLock.getLockFile() + " SpoutId =" + spoutId, e);
- }
- }
-
- protected void emitData(List<Object> tuple, MessageId id) {
- LOG.trace("Emitting - {}", id);
-
- if ( outputStreamName==null )
- collector.emit( tuple, id );
- else
- collector.emit( outputStreamName, tuple, id );
-
- inflight.put(id, tuple);
- }
-
- public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {
- LOG.info("Opening HDFS Spout");
- this.conf = conf;
- this.commitTimer = new Timer();
- this.tracker = new ProgressTracker();
- this.hdfsConfig = new Configuration();
-
- this.collector = collector;
- this.hdfsConfig = new Configuration();
- this.tupleCounter = 0;
-
- // Hdfs related settings
- if ( this.hdfsUri==null && conf.containsKey(Configs.HDFS_URI) ) {
- this.hdfsUri = conf.get(Configs.HDFS_URI).toString();
- }
- if ( this.hdfsUri==null ) {
- throw new RuntimeException("HDFS Uri not set on spout");
- }
-
- try {
- this.hdfs = FileSystem.get(URI.create(hdfsUri), hdfsConfig);
- } catch (IOException e) {
- LOG.error("Unable to instantiate file system", e);
- throw new RuntimeException("Unable to instantiate file system", e);
- }
-
-
- if ( conf.containsKey(configKey) ) {
- Map<String, Object> map = (Map<String, Object>)conf.get(configKey);
- if ( map != null ) {
- for(String keyName : map.keySet()){
- LOG.info("HDFS Config override : {} = {} ", keyName, String.valueOf(map.get(keyName)));
- this.hdfsConfig.set(keyName, String.valueOf(map.get(keyName)));
- }
- try {
- HdfsSecurityUtil.login(conf, hdfsConfig);
- } catch (IOException e) {
- LOG.error("HDFS Login failed ", e);
- throw new RuntimeException(e);
- }
- } // if (map != null)
- }
+ private SpoutOutputCollector collector;
+ HashMap<MessageId, List<Object>> inflight = new HashMap<>();
+ LinkedBlockingQueue<HdfsUtils.Pair<MessageId, List<Object>>> retryList = new LinkedBlockingQueue<>();
+
+ private Configuration hdfsConfig;
+
+ private Map conf = null;
+ private FileLock lock;
+ private String spoutId = null;
+
+ HdfsUtils.Pair<Path, FileLock.LogEntry> lastExpiredLock = null;
+ private long lastExpiredLockTime = 0;
+
+ private long tupleCounter = 0;
+ private boolean ackEnabled = false;
+ private int acksSinceLastCommit = 0;
+ private final AtomicBoolean commitTimeElapsed = new AtomicBoolean(false);
+ private Timer commitTimer;
+ private boolean fileReadCompletely = true;
+
+ private String configKey = Configs.DEFAULT_HDFS_CONFIG_KEY; // key for hdfs Kerberos configs
- // Reader type config
- if ( readerType==null && conf.containsKey(Configs.READER_TYPE) ) {
- readerType = conf.get(Configs.READER_TYPE).toString();
+ public HdfsSpout() {
}
- checkValidReader(readerType);
- // -- source dir config
- if ( sourceDir==null && conf.containsKey(Configs.SOURCE_DIR) ) {
- sourceDir = conf.get(Configs.SOURCE_DIR).toString();
+ public HdfsSpout setHdfsUri(String hdfsUri) {
+ this.hdfsUri = hdfsUri;
+ return this;
}
- if ( sourceDir==null ) {
- LOG.error(Configs.SOURCE_DIR + " setting is required");
- throw new RuntimeException(Configs.SOURCE_DIR + " setting is required");
+
+ public HdfsSpout setReaderType(String readerType) {
+ this.readerType = readerType;
+ return this;
+ }
+
+ public HdfsSpout setSourceDir(String sourceDir) {
+ this.sourceDir = sourceDir;
+ return this;
}
- this.sourceDirPath = new Path( sourceDir );
- // -- archive dir config
- if ( archiveDir==null && conf.containsKey(Configs.ARCHIVE_DIR) ) {
- archiveDir = conf.get(Configs.ARCHIVE_DIR).toString();
+ public HdfsSpout setArchiveDir(String archiveDir) {
+ this.archiveDir = archiveDir;
+ return this;
}
- if ( archiveDir==null ) {
- LOG.error(Configs.ARCHIVE_DIR + " setting is required");
- throw new RuntimeException(Configs.ARCHIVE_DIR + " setting is required");
+
+ public HdfsSpout setBadFilesDir(String badFilesDir) {
+ this.badFilesDir = badFilesDir;
+ return this;
+ }
+
+ public HdfsSpout setLockDir(String lockDir) {
+ this.lockDir = lockDir;
+ return this;
}
- this.archiveDirPath = new Path( archiveDir );
- validateOrMakeDir(hdfs, archiveDirPath, "Archive");
- // -- bad files dir config
- if ( badFilesDir==null && conf.containsKey(Configs.BAD_DIR) ) {
- badFilesDir = conf.get(Configs.BAD_DIR).toString();
+ public HdfsSpout setCommitFrequencyCount(int commitFrequencyCount) {
+ this.commitFrequencyCount = commitFrequencyCount;
+ return this;
}
- if ( badFilesDir==null ) {
- LOG.error(Configs.BAD_DIR + " setting is required");
- throw new RuntimeException(Configs.BAD_DIR + " setting is required");
+
+ public HdfsSpout setCommitFrequencySec(int commitFrequencySec) {
+ this.commitFrequencySec = commitFrequencySec;
+ return this;
+ }
+
+ public HdfsSpout setMaxOutstanding(int maxOutstanding) {
+ this.maxOutstanding = maxOutstanding;
+ return this;
+ }
+
+ public HdfsSpout setLockTimeoutSec(int lockTimeoutSec) {
+ this.lockTimeoutSec = lockTimeoutSec;
+ return this;
}
- this.badFilesDirPath = new Path(badFilesDir);
- validateOrMakeDir(hdfs, badFilesDirPath, "bad files");
- // -- ignore file names config
- if ( conf.containsKey(Configs.IGNORE_SUFFIX) ) {
- this.ignoreSuffix = conf.get(Configs.IGNORE_SUFFIX).toString();
+ public HdfsSpout setClocksInSync(boolean clocksInSync) {
+ this.clocksInSync = clocksInSync;
+ return this;
}
- // -- lock dir config
- if ( lockDir==null && conf.containsKey(Configs.LOCK_DIR) ) {
- lockDir = conf.get(Configs.LOCK_DIR).toString();
+ public HdfsSpout setIgnoreSuffix(String ignoreSuffix) {
+ this.ignoreSuffix = ignoreSuffix;
+ return this;
}
- if ( lockDir==null ) {
- lockDir = getDefaultLockDir(sourceDirPath);
+
+ /**
+ * Output field names. Number of fields depends upon the reader type
+ */
+ public HdfsSpout withOutputFields(String... fields) {
+ outputFields = new Fields(fields);
+ return this;
}
- this.lockDirPath = new Path(lockDir);
- validateOrMakeDir(hdfs,lockDirPath, "locks");
+ /**
+ * set key name under which HDFS options are placed. (similar to HDFS bolt). default key name is 'hdfs.config'
+ */
+ public HdfsSpout withConfigKey(String configKey) {
+ this.configKey = configKey;
+ return this;
+ }
- // -- lock timeout
- if ( conf.get(Configs.LOCK_TIMEOUT) !=null ) {
- this.lockTimeoutSec = Integer.parseInt(conf.get(Configs.LOCK_TIMEOUT).toString());
+ /**
+ * Set output stream name
+ */
+ public HdfsSpout withOutputStream(String streamName) {
+ this.outputStreamName = streamName;
+ return this;
}
- // -- enable/disable ACKing
- Object ackers = conf.get(Config.TOPOLOGY_ACKER_EXECUTORS);
- if ( ackers!=null ) {
- int ackerCount = Integer.parseInt(ackers.toString());
- this.ackEnabled = (ackerCount>0);
- LOG.debug("ACKer count = {}", ackerCount);
+ public Path getLockDirPath() {
+ return lockDirPath;
}
- else { // ackers==null when ackerCount not explicitly set on the topology
- this.ackEnabled = true;
- LOG.debug("ACK count not explicitly set on topology.");
+
+ public SpoutOutputCollector getCollector() {
+ return collector;
}
- LOG.info("ACK mode is {}", ackEnabled ? "enabled" : "disabled");
+ public void nextTuple() {
+ LOG.trace("Next Tuple {}", spoutId);
+ // 1) First re-emit any previously failed tuples (from retryList)
+ if (!retryList.isEmpty()) {
+ LOG.debug("Sending tuple from retry list");
+ HdfsUtils.Pair<MessageId, List<Object>> pair = retryList.remove();
+ emitData(pair.getValue(), pair.getKey());
+ return;
+ }
- // -- commit frequency - count
- if ( conf.get(Configs.COMMIT_FREQ_COUNT) != null ) {
- commitFrequencyCount = Integer.parseInt(conf.get(Configs.COMMIT_FREQ_COUNT).toString());
+ if (ackEnabled && tracker.size() >= maxOutstanding) {
+ LOG.warn("Waiting for more ACKs before generating new tuples. "
+ + "Progress tracker size has reached limit {}, SpoutID {}",
+ maxOutstanding, spoutId);
+ // Don't emit anything .. allow configured spout wait strategy to kick in
+ return;
+ }
+
+ // 2) If no failed tuples to be retried, then send tuples from hdfs
+ while (true) {
+ try {
+ // 3) Select a new file if one is not open already
+ if (reader == null) {
+ reader = pickNextFile();
+ if (reader == null) {
+ LOG.debug("Currently no new files to process under : " + sourceDirPath);
+ return;
+ } else {
+ fileReadCompletely = false;
+ }
+ }
+ if (fileReadCompletely) { // wait for more ACKs before proceeding
+ return;
+ }
+ // 4) Read record from file, emit to collector and record progress
+ List<Object> tuple = reader.next();
+ if (tuple != null) {
+ fileReadCompletely = false;
+ ++tupleCounter;
+ MessageId msgId = new MessageId(tupleCounter, reader.getFilePath(), reader.getFileOffset());
+ emitData(tuple, msgId);
+
+ if (!ackEnabled) {
+ ++acksSinceLastCommit; // assume message is immediately ACKed in non-ack mode
+ commitProgress(reader.getFileOffset());
+ } else {
+ commitProgress(tracker.getCommitPosition());
+ }
+ return;
+ } else {
+ fileReadCompletely = true;
+ if (!ackEnabled) {
+ markFileAsDone(reader.getFilePath());
+ }
+ }
+ } catch (IOException e) {
+ LOG.error("I/O Error processing at file location " + getFileProgress(reader), e);
+ // don't emit anything .. allow configured spout wait strategy to kick in
+ return;
+ } catch (ParseException e) {
+ LOG.error("Parsing error when processing at file location " + getFileProgress(reader)
+ + ". Skipping remainder of file.", e);
+ markFileAsBad(reader.getFilePath());
+ // Note: We don't return from this method on ParseException to avoid triggering the
+ // spout wait strategy (due to no emits). Instead we go back into the loop and
+ // generate a tuple from next file
+ }
+ } // while
+ }
+
+ // will commit progress into lock file if commit threshold is reached
+ private void commitProgress(FileOffset position) {
+ if (position == null) {
+ return;
+ }
+ if (lock != null && canCommitNow()) {
+ try {
+ String pos = position.toString();
+ lock.heartbeat(pos);
+ LOG.debug("{} Committed progress. {}", spoutId, pos);
+ acksSinceLastCommit = 0;
+ commitTimeElapsed.set(false);
+ setupCommitElapseTimer();
+ } catch (IOException e) {
+ LOG.error("Unable to commit progress Will retry later. Spout ID = " + spoutId, e);
+ }
+ }
}
- // -- commit frequency - seconds
- if ( conf.get(Configs.COMMIT_FREQ_SEC) != null ) {
- commitFrequencySec = Integer.parseInt(conf.get(Configs.COMMIT_FREQ_SEC).toString());
- if ( commitFrequencySec<=0 ) {
- throw new RuntimeException(Configs.COMMIT_FREQ_SEC + " setting must be greater than 0");
- }
- }
-
- // -- max outstanding tuples
- if ( conf.get(Configs.MAX_OUTSTANDING) !=null ) {
- maxOutstanding = Integer.parseInt(conf.get(Configs.MAX_OUTSTANDING).toString());
- }
+ private void setupCommitElapseTimer() {
+ if (commitFrequencySec <= 0) {
+ return;
+ }
+ TimerTask timerTask = new TimerTask() {
+ @Override
+ public void run() {
+ commitTimeElapsed.set(true);
+ }
+ };
+ commitTimer.schedule(timerTask, commitFrequencySec * 1000);
+ }
- // -- clocks in sync
- if ( conf.get(Configs.CLOCKS_INSYNC) !=null ) {
- clocksInSync = Boolean.parseBoolean(conf.get(Configs.CLOCKS_INSYNC).toString());
+ private static String getFileProgress(FileReader reader) {
+ return reader.getFilePath() + " " + reader.getFileOffset();
}
- // -- spout id
- spoutId = context.getThisComponentId();
+ private void markFileAsDone(Path filePath) {
+ try {
+ Path newFile = renameCompletedFile(reader.getFilePath());
+ LOG.info("Completed processing {}. Spout Id = {}", newFile, spoutId);
+ } catch (IOException e) {
+ LOG.error("Unable to archive completed file" + filePath + " Spout ID " + spoutId, e);
+ }
+ closeReaderAndResetTrackers();
+ }
- // setup timer for commit elapse time tracking
- setupCommitElapseTimer();
- }
+ private void markFileAsBad(Path file) {
+ String fileName = file.toString();
+ String fileNameMinusSuffix = fileName.substring(0, fileName.indexOf(inprogress_suffix));
+ String originalName = new Path(fileNameMinusSuffix).getName();
+ Path newFile = new Path(badFilesDirPath + Path.SEPARATOR + originalName);
- private static void validateOrMakeDir(FileSystem fs, Path dir, String dirDescription) {
- try {
- if ( fs.exists(dir) ) {
- if ( !fs.isDirectory(dir) ) {
- LOG.error(dirDescription + " directory is a file, not a dir. " + dir);
- throw new RuntimeException(dirDescription + " directory is a file, not a dir. " + dir);
+ LOG.info("Moving bad file {} to {}. Processed it till offset {}. SpoutID= {}", originalName, newFile, tracker.getCommitPosition(), spoutId);
+ try {
+ if (!hdfs.rename(file, newFile)) { // seems this can fail by returning false or throwing exception
+ throw new IOException("Move failed for bad file: " + file); // convert false ret value to exception
+ }
+ } catch (IOException e) {
+ LOG.warn("Error moving bad file: " + file + " to destination " + newFile + " SpoutId =" + spoutId, e);
}
- } else if ( ! fs.mkdirs(dir) ) {
- LOG.error("Unable to create " + dirDescription + " directory " + dir);
- throw new RuntimeException("Unable to create " + dirDescription + " directory " + dir);
- }
- } catch (IOException e) {
- LOG.error("Unable to create " + dirDescription + " directory " + dir, e);
- throw new RuntimeException("Unable to create " + dirDescription + " directory " + dir, e);
+ closeReaderAndResetTrackers();
}
- }
- private String getDefaultLockDir(Path sourceDirPath) {
- return sourceDirPath.toString() + Path.SEPARATOR + Configs.DEFAULT_LOCK_DIR;
- }
+ private void closeReaderAndResetTrackers() {
+ inflight.clear();
+ tracker.offsets.clear();
+ retryList.clear();
+
+ reader.close();
+ reader = null;
+ releaseLockAndLog(lock, spoutId);
+ lock = null;
+ }
- private static void checkValidReader(String readerType) {
- if ( readerType.equalsIgnoreCase(Configs.TEXT) || readerType.equalsIgnoreCase(Configs.SEQ) )
- return;
- try {
- Class<?> classType = Class.forName(readerType);
- classType.getConstructor(FileSystem.class, Path.class, Map.class);
- return;
- } catch (ClassNotFoundException e) {
- LOG.error(readerType + " not found in classpath.", e);
- throw new IllegalArgumentException(readerType + " not found in classpath.", e);
- } catch (NoSuchMethodException e) {
- LOG.error(readerType + " is missing the expected constructor for Readers.", e);
- throw new IllegalArgumentException(readerType + " is missing the expected constuctor for Readers.");
+ private static void releaseLockAndLog(FileLock fLock, String spoutId) {
+ try {
+ if (fLock != null) {
+ fLock.release();
+ LOG.debug("Spout {} released FileLock. SpoutId = {}", fLock.getLockFile(), spoutId);
+ }
+ } catch (IOException e) {
+ LOG.error("Unable to delete lock file : " + fLock.getLockFile() + " SpoutId =" + spoutId, e);
+ }
}
- }
- @Override
- public void ack(Object msgId) {
- LOG.trace("Ack received for msg {} on spout {}", msgId, spoutId);
- if ( !ackEnabled ) {
- return;
+ protected void emitData(List<Object> tuple, MessageId id) {
+ LOG.trace("Emitting - {}", id);
+
+ if (outputStreamName == null) {
+ collector.emit(tuple, id);
+ } else {
+ collector.emit(outputStreamName, tuple, id);
+ }
+
+ inflight.put(id, tuple);
}
- MessageId id = (MessageId) msgId;
- inflight.remove(id);
- ++acksSinceLastCommit;
- tracker.recordAckedOffset(id.offset);
- commitProgress(tracker.getCommitPosition());
- if ( fileReadCompletely && inflight.isEmpty() ) {
- markFileAsDone(reader.getFilePath());
- reader = null;
- }
- super.ack(msgId);
- }
-
- private boolean canCommitNow() {
-
- if ( commitFrequencyCount>0 && acksSinceLastCommit >= commitFrequencyCount ) {
- return true;
- }
- return commitTimeElapsed.get();
- }
-
- @Override
- public void fail(Object msgId) {
- LOG.trace("Fail received for msg id {} on spout {}", msgId, spoutId);
- super.fail(msgId);
- if ( ackEnabled ) {
- HdfsUtils.Pair<MessageId, List<Object>> item = HdfsUtils.Pair.of(msgId, inflight.remove(msgId));
- retryList.add(item);
- }
- }
-
- private FileReader pickNextFile() {
- try {
- // 1) If there are any abandoned files, pick oldest one
- lock = getOldestExpiredLock();
- if ( lock!=null ) {
- LOG.debug("Spout {} now took over ownership of abandoned FileLock {}", spoutId, lock.getLockFile());
- Path file = getFileForLockFile(lock.getLockFile(), sourceDirPath);
- String resumeFromOffset = lock.getLastLogEntry().fileOffset;
- LOG.info("Resuming processing of abandoned file : {}", file);
- return createFileReader(file, resumeFromOffset);
- }
-
- // 2) If no abandoned files, then pick oldest file in sourceDirPath, lock it and rename it
- Collection<Path> listing = HdfsUtils.listFilesByModificationTime(hdfs, sourceDirPath, 0);
-
- for (Path file : listing) {
- if (file.getName().endsWith(inprogress_suffix)) {
- continue;
- }
- if (file.getName().endsWith(ignoreSuffix)) {
- continue;
- }
- lock = FileLock.tryLock(hdfs, file, lockDirPath, spoutId);
- if (lock == null) {
- LOG.debug("Unable to get FileLock for {}, so skipping it.", file);
- continue; // could not lock, so try another file.
+
+ public void open(Map conf, TopologyContext context, SpoutOutputCollector collector) {
+ LOG.info("Opening HDFS Spout");
+ this.conf = conf;
+ this.commitTimer = new Timer();
+ this.tracker = new ProgressTracker();
+ this.hdfsConfig = new Configuration();
+
+ this.collector = collector;
+ this.hdfsConfig = new Configuration();
+ this.tupleCounter = 0;
+
+ // Hdfs related settings
+ if (this.hdfsUri == null && conf.containsKey(Configs.HDFS_URI)) {
+ this.hdfsUri = conf.get(Configs.HDFS_URI).toString();
+ }
+ if (this.hdfsUri == null) {
+ throw new RuntimeException("HDFS Uri not set on spout");
}
+
try {
- Path newFile = renameToInProgressFile(file);
- FileReader result = createFileReader(newFile);
- LOG.info("Processing : {} ", file);
- return result;
+ this.hdfs = FileSystem.get(URI.create(hdfsUri), hdfsConfig);
+ } catch (IOException e) {
+ LOG.error("Unable to instantiate file system", e);
+ throw new RuntimeException("Unable to instantiate file system", e);
+ }
+
+ if (conf.containsKey(configKey)) {
+ Map<String, Object> map = (Map<String, Object>) conf.get(configKey);
+ if (map != null) {
+ for (String keyName : map.keySet()) {
+ LOG.info("HDFS Config override : {} = {} ", keyName, String.valueOf(map.get(keyName)));
+ this.hdfsConfig.set(keyName, String.valueOf(map.get(keyName)));
+ }
+ try {
+ HdfsSecurityUtil.login(conf, hdfsConfig);
+ } catch (IOException e) {
+ LOG.error("HDFS Login failed ", e);
+ throw new RuntimeException(e);
+ }
+ } // if (map != null)
+ }
+
+ // Reader type config
+ if (readerType == null && conf.containsKey(Configs.READER_TYPE)) {
+ readerType = conf.get(Configs.READER_TYPE).toString();
+ }
+ checkValidReader(readerType);
+
+ // -- source dir config
+ if (sourceDir == null && conf.containsKey(Configs.SOURCE_DIR)) {
+ sourceDir = conf.get(Configs.SOURCE_DIR).toString();
+ }
+ if (sourceDir == null) {
+ LOG.error(Configs.SOURCE_DIR + " setting is required");
+ throw new RuntimeException(Configs.SOURCE_DIR + " setting is required");
+ }
+ this.sourceDirPath = new Path(sourceDir);
+
+ // -- archive dir config
+ if (archiveDir == null && conf.containsKey(Configs.ARCHIVE_DIR)) {
+ archiveDir = conf.get(Configs.ARCHIVE_DIR).toString();
+ }
+ if (archiveDir == null) {
+ LOG.error(Configs.ARCHIVE_DIR + " setting is required");
+ throw new RuntimeException(Configs.ARCHIVE_DIR + " setting is required");
+ }
+ this.archiveDirPath = new Path(archiveDir);
+ validateOrMakeDir(hdfs, archiveDirPath, "Archive");
+
+ // -- bad files dir config
+ if (badFilesDir == null && conf.containsKey(Configs.BAD_DIR)) {
+ badFilesDir = conf.get(Configs.BAD_DIR).toString();
+ }
+ if (badFilesDir == null) {
+ LOG.error(Configs.BAD_DIR + " setting is required");
+ throw new RuntimeException(Configs.BAD_DIR + " setting is required");
+ }
+ this.badFilesDirPath = new Path(badFilesDir);
+ validateOrMakeDir(hdfs, badFilesDirPath, "bad files");
+
+ // -- ignore file names config
+ if (conf.containsKey(Configs.IGNORE_SUFFIX)) {
+ this.ignoreSuffix = conf.get(Configs.IGNORE_SUFFIX).toString();
+ }
+
+ // -- lock dir config
+ if (lockDir == null && conf.containsKey(Configs.LOCK_DIR)) {
+ lockDir = conf.get(Configs.LOCK_DIR).toString();
+ }
+ if (lockDir == null) {
+ lockDir = getDefaultLockDir(sourceDirPath);
+ }
+ this.lockDirPath = new Path(lockDir);
+ validateOrMakeDir(hdfs, lockDirPath, "locks");
+
+ // -- lock timeout
+ if (conf.get(Configs.LOCK_TIMEOUT) != null) {
+ this.lockTimeoutSec = Integer.parseInt(conf.get(Configs.LOCK_TIMEOUT).toString());
+ }
+
+ // -- enable/disable ACKing
+ Object ackers = conf.get(Config.TOPOLOGY_ACKER_EXECUTORS);
+ if (ackers != null) {
+ int ackerCount = Integer.parseInt(ackers.toString());
+ this.ackEnabled = (ackerCount > 0);
+ LOG.debug("ACKer count = {}", ackerCount);
+ } else { // ackers==null when ackerCount not explicitly set on the topology
+ this.ackEnabled = true;
+ LOG.debug("ACK count not explicitly set on topology.");
+ }
+
+ LOG.info("ACK mode is {}", ackEnabled ? "enabled" : "disabled");
+
+ // -- commit frequency - count
+ if (conf.get(Configs.COMMIT_FREQ_COUNT) != null) {
+ commitFrequencyCount = Integer.parseInt(conf.get(Configs.COMMIT_FREQ_COUNT).toString());
+ }
+
+ // -- commit frequency - seconds
+ if (conf.get(Configs.COMMIT_FREQ_SEC) != null) {
+ commitFrequencySec = Integer.parseInt(conf.get(Configs.COMMIT_FREQ_SEC).toString());
+ if (commitFrequencySec <= 0) {
+ throw new RuntimeException(Configs.COMMIT_FREQ_SEC + " setting must be greater than 0");
+ }
+ }
+
+ // -- max outstanding tuples
+ if (conf.get(Configs.MAX_OUTSTANDING) != null) {
+ maxOutstanding = Integer.parseInt(conf.get(Configs.MAX_OUTSTANDING).toString());
+ }
+
+ // -- clocks in sync
+ if (conf.get(Configs.CLOCKS_INSYNC) != null) {
+ clocksInSync = Boolean.parseBoolean(conf.get(Configs.CLOCKS_INSYNC).toString());
+ }
+
+ // -- spout id
+ spoutId = context.getThisComponentId();
+
+ // setup timer for commit elapse time tracking
+ setupCommitElapseTimer();
+ }
+
+ @Override
+ public void close() {
+ this.commitTimer.cancel();
+ }
+
+ private static void validateOrMakeDir(FileSystem fs, Path dir, String dirDescription) {
+ try {
+ if (fs.exists(dir)) {
+ if (!fs.isDirectory(dir)) {
+ LOG.error(dirDescription + " directory is a file, not a dir. " + dir);
+ throw new RuntimeException(dirDescription + " directory is a file, not a dir. " + dir);
+ }
+ } else if (!fs.mkdirs(dir)) {
+ LOG.error("Unable to create " + dirDescription + " directory " + dir);
+ throw new RuntimeException("Unable to create " + dirDescription + " directory " + dir);
+ }
+ } catch (IOException e) {
+ LOG.error("Unable to create " + dirDescription + " directory " + dir, e);
+ throw new RuntimeException("Unable to create " + dirDescription + " directory " + dir, e);
+ }
+ }
+
+ private String getDefaultLockDir(Path sourceDirPath) {
+ return sourceDirPath.toString() + Path.SEPARATOR + Configs.DEFAULT_LOCK_DIR;
+ }
+
+ private static void checkValidReader(String readerType) {
+ if (readerType.equalsIgnoreCase(Configs.TEXT) || readerType.equalsIgnoreCase(Configs.SEQ)) {
+ return;
+ }
+ try {
+ Class<?> classType = Class.forName(readerType);
+ classType.getConstructor(FileSystem.class, Path.class, Map.class);
+ return;
+ } catch (ClassNotFoundException e) {
+ LOG.error(readerType + " not found in classpath.", e);
+ throw new IllegalArgumentException(readerType + " not found in classpath.", e);
+ } catch (NoSuchMethodException e) {
+ LOG.error(readerType + " is missing the expected constructor for Readers.", e);
+ throw new IllegalArgumentException(readerType + " is missing the expected constuctor for Readers.");
+ }
+ }
+
+ @Override
+ public void ack(Object msgId) {
+ LOG.trace("Ack received for msg {} on spout {}", msgId, spoutId);
+ if (!ackEnabled) {
+ return;
+ }
+ MessageId id = (MessageId) msgId;
+ inflight.remove(id);
+ ++acksSinceLastCommit;
+ tracker.recordAckedOffset(id.offset);
+ commitProgress(tracker.getCommitPosition());
+ if (fileReadCompletely && inflight.isEmpty()) {
+ markFileAsDone(reader.getFilePath());
+ reader = null;
+ }
+ super.ack(msgId);
+ }
+
+ private boolean canCommitNow() {
+
+ if (commitFrequencyCount > 0 && acksSinceLastCommit >= commitFrequencyCount) {
+ return true;
+ }
+ return commitTimeElapsed.get();
+ }
+
+ @Override
+ public void fail(Object msgId) {
+ LOG.trace("Fail received for msg id {} on spout {}", msgId, spoutId);
+ super.fail(msgId);
+ if (ackEnabled) {
+ HdfsUtils.Pair<MessageId, List<Object>> item = HdfsUtils.Pair.of(msgId, inflight.remove(msgId));
+ retryList.add(item);
+ }
+ }
+
+ private FileReader pickNextFile() {
+ try {
+ // 1) If there are any abandoned files, pick oldest one
+ lock = getOldestExpiredLock();
+ if (lock != null) {
+ LOG.debug("Spout {} now took over ownership of abandoned FileLock {}", spoutId, lock.getLockFile());
+ Path file = getFileForLockFile(lock.getLockFile(), sourceDirPath);
+ String resumeFromOffset = lock.getLastLogEntry().fileOffset;
+ LOG.info("Resuming processing of abandoned file : {}", file);
+ return createFileReader(file, resumeFromOffset);
+ }
+
+ // 2) If no abandoned files, then pick oldest file in sourceDirPath, lock it and rename it
+ Collection<Path> listing = HdfsUtils.listFilesByModificationTime(hdfs, sourceDirPath, 0);
+
+ for (Path file : listing) {
+ if (file.getName().endsWith(inprogress_suffix)) {
+ continue;
+ }
+ if (file.getName().endsWith(ignoreSuffix)) {
+ continue;
+ }
+ lock = FileLock.tryLock(hdfs, file, lockDirPath, spoutId);
+ if (lock == null) {
+ LOG.debug("Unable to get FileLock for {}, so skipping it.", file);
+ continue; // could not lock, so try another file.
+ }
+ try {
+ Path newFile = renameToInProgressFile(file);
+ FileReader result = createFileReader(newFile);
+ LOG.info("Processing : {} ", file);
+ return result;
+ } catch (Exception e) {
+ LOG.error("Skipping file " + file, e);
+ releaseLockAndLog(lock, spoutId);
+ continue;
+ }
+ }
+
+ return null;
+ } catch (IOException e) {
+ LOG.error("Unable to select next file for consumption " + sourceDirPath, e);
+ return null;
+ }
+ }
+
+ /**
+ * If clocks in sync, then acquires the oldest expired lock Else, on first call, just remembers the oldest expired lock, on next call
+ * check if the lock is updated. if not updated then acquires the lock
+ *
+ * @return a lock object
+ * @throws IOException
+ */
+ private FileLock getOldestExpiredLock() throws IOException {
+ // 1 - acquire lock on dir
+ DirLock dirlock = DirLock.tryLock(hdfs, lockDirPath);
+ if (dirlock == null) {
+ dirlock = DirLock.takeOwnershipIfStale(hdfs, lockDirPath, lockTimeoutSec);
+ if (dirlock == null) {
+ LOG.debug("Spout {} could not take over ownership of DirLock for {}", spoutId, lockDirPath);
+ return null;
+ }
+ LOG.debug("Spout {} now took over ownership of abandoned DirLock for {}", spoutId, lockDirPath);
+ } else {
+ LOG.debug("Spout {} now owns DirLock for {}", spoutId, lockDirPath);
+ }
+
+ try {
+ // 2 - if clocks are in sync then simply take ownership of the oldest expired lock
+ if (clocksInSync) {
+ return FileLock.acquireOldestExpiredLock(hdfs, lockDirPath, lockTimeoutSec, spoutId);
+ }
+
+ // 3 - if clocks are not in sync ..
+ if (lastExpiredLock == null) {
+ // just make a note of the oldest expired lock now and check if its still unmodified after lockTimeoutSec
+ lastExpiredLock = FileLock.locateOldestExpiredLock(hdfs, lockDirPath, lockTimeoutSec);
+ lastExpiredLockTime = System.currentTimeMillis();
+ return null;
+ }
+ // see if lockTimeoutSec time has elapsed since we last selected the lock file
+ if (hasExpired(lastExpiredLockTime)) {
+ return null;
+ }
+
+ // If lock file has expired, then own it
+ FileLock.LogEntry lastEntry = FileLock.getLastEntry(hdfs, lastExpiredLock.getKey());
+ if (lastEntry.equals(lastExpiredLock.getValue())) {
+ FileLock result = FileLock.takeOwnership(hdfs, lastExpiredLock.getKey(), lastEntry, spoutId);
+ lastExpiredLock = null;
+ return result;
+ } else {
+ // if lock file has been updated since last time, then leave this lock file alone
+ lastExpiredLock = null;
+ return null;
+ }
+ } finally {
+ dirlock.release();
+ LOG.debug("Released DirLock {}, SpoutID {} ", dirlock.getLockFile(), spoutId);
+ }
+ }
+
+ private boolean hasExpired(long lastModifyTime) {
+ return (System.currentTimeMillis() - lastModifyTime) < lockTimeoutSec * 1000;
+ }
+
+ /**
+ * Creates a reader that reads from beginning of file
+ *
+ * @param file file to read
+ * @return
+ * @throws IOException
+ */
+ private FileReader createFileReader(Path file)
+ throws IOException {
+ if (readerType.equalsIgnoreCase(Configs.SEQ)) {
+ return new SequenceFileReader(this.hdfs, file, conf);
+ }
+ if (readerType.equalsIgnoreCase(Configs.TEXT)) {
+ return new TextFileReader(this.hdfs, file, conf);
+ }
+ try {
+ Class<?> clsType = Class.forName(readerType);
+ Constructor<?> constructor = clsType.getConstructor(FileSystem.class, Path.class, Map.class);
+ return (FileReader) constructor.newInstance(this.hdfs, file, conf);
} catch (Exception e) {
- LOG.error("Skipping file " + file, e);
- releaseLockAndLog(lock, spoutId);
- continue;
- }
- }
-
- return null;
- } catch (IOException e) {
- LOG.error("Unable to select next file for consumption " + sourceDirPath, e);
- return null;
- }
- }
-
- /**
- * If clocks in sync, then acquires the oldest expired lock
- * Else, on first call, just remembers the oldest expired lock, on next call check if the lock is updated. if not updated then acquires the lock
- * @return a lock object
- * @throws IOException
- */
- private FileLock getOldestExpiredLock() throws IOException {
- // 1 - acquire lock on dir
- DirLock dirlock = DirLock.tryLock(hdfs, lockDirPath);
- if (dirlock == null) {
- dirlock = DirLock.takeOwnershipIfStale(hdfs, lockDirPath, lockTimeoutSec);
- if (dirlock == null) {
- LOG.debug("Spout {} could not take over ownership of DirLock for {}", spoutId, lockDirPath);
- return null;
- }
- LOG.debug("Spout {} now took over ownership of abandoned DirLock for {}", spoutId, lockDirPath);
- } else {
- LOG.debug("Spout {} now owns DirLock for {}", spoutId, lockDirPath);
- }
-
- try {
- // 2 - if clocks are in sync then simply take ownership of the oldest expired lock
- if (clocksInSync) {
- return FileLock.acquireOldestExpiredLock(hdfs, lockDirPath, lockTimeoutSec, spoutId);
- }
-
- // 3 - if clocks are not in sync ..
- if ( lastExpiredLock == null ) {
- // just make a note of the oldest expired lock now and check if its still unmodified after lockTimeoutSec
- lastExpiredLock = FileLock.locateOldestExpiredLock(hdfs, lockDirPath, lockTimeoutSec);
- lastExpiredLockTime = System.currentTimeMillis();
- return null;
- }
- // see if lockTimeoutSec time has elapsed since we last selected the lock file
- if ( hasExpired(lastExpiredLockTime) ) {
- return null;
- }
-
- // If lock file has expired, then own it
- FileLock.LogEntry lastEntry = FileLock.getLastEntry(hdfs, lastExpiredLock.getKey());
- if ( lastEntry.equals(lastExpiredLock.getValue()) ) {
- FileLock result = FileLock.takeOwnership(hdfs, lastExpiredLock.getKey(), lastEntry, spoutId);
- lastExpiredLock = null;
- return result;
- } else {
- // if lock file has been updated since last time, then leave this lock file alone
- lastExpiredLock = null;
+ LOG.error(e.getMessage(), e);
+ throw new RuntimeException("Unable to instantiate " + readerType + " reader", e);
+ }
+ }
+
+ /**
+ * Creates a reader that starts reading from 'offset'
+ *
+ * @param file the file to read
+ * @param offset the offset string should be understandable by the reader type being used
+ * @return
+ * @throws IOException
+ */
+ private FileReader createFileReader(Path file, String offset)
+ throws IOException {
+ if (readerType.equalsIgnoreCase(Configs.SEQ)) {
+ return new SequenceFileReader(this.hdfs, file, conf, offset);
+ }
+ if (readerType.equalsIgnoreCase(Configs.TEXT)) {
+ return new TextFileReader(this.hdfs, file, conf, offset);
+ }
+
+ try {
+ Class<?> clsType = Class.forName(readerType);
+ Constructor<?> constructor = clsType.getConstructor(FileSystem.class, Path.class, Map.class, String.class);
+ return (FileReader) constructor.newInstance(this.hdfs, file, conf, offset);
+ } catch (Exception e) {
+ LOG.error(e.getMessage(), e);
+ throw new RuntimeException("Unable to instantiate " + readerType, e);
+ }
+ }
+
+ /**
+ * Renames files with .inprogress suffix
+ *
+ * @return path of renamed file
+ * @throws if operation fails
+ */
+ private Path renameToInProgressFile(Path file)
+ throws IOException {
+ Path newFile = new Path(file.toString() + inprogress_suffix);
+ try {
+ if (hdfs.rename(file, newFile)) {
+ return newFile;
+ }
+ throw new RenameException(file, newFile);
+ } catch (IOException e) {
+ throw new RenameException(file, newFile, e);
+ }
+ }
+
+ /**
+ * Returns the corresponding input file in the 'sourceDirPath' for the specified lock file. If no such file is found then returns null
+ */
+ private Path getFileForLockFile(Path lockFile, Path sourceDirPath)
+ throws IOException {
+ String lockFileName = lockFile.getName();
+ Path dataFile = new Path(sourceDirPath + Path.SEPARATOR + lockFileName + inprogress_suffix);
+ if (hdfs.exists(dataFile)) {
+ return dataFile;
+ }
+ dataFile = new Path(sourceDirPath + Path.SEPARATOR + lockFileName);
+ if (hdfs.exists(dataFile)) {
+ return dataFile;
+ }
return null;
- }
- } finally {
- dirlock.release();
- LOG.debug("Released DirLock {}, SpoutID {} ", dirlock.getLockFile(), spoutId);
- }
- }
-
- private boolean hasExpired(long lastModifyTime) {
- return (System.currentTimeMillis() - lastModifyTime ) < lockTimeoutSec*1000;
- }
-
- /**
- * Creates a reader that reads from beginning of file
- * @param file file to read
- * @return
- * @throws IOException
- */
- private FileReader createFileReader(Path file)
- throws IOException {
- if ( readerType.equalsIgnoreCase(Configs.SEQ) ) {
- return new SequenceFileReader(this.hdfs, file, conf);
- }
- if ( readerType.equalsIgnoreCase(Configs.TEXT) ) {
- return new TextFileReader(this.hdfs, file, conf);
- }
- try {
- Class<?> clsType = Class.forName(readerType);
- Constructor<?> constructor = clsType.getConstructor(FileSystem.class, Path.class, Map.class);
- return (FileReader) constructor.newInstance(this.hdfs, file, conf);
- } catch (Exception e) {
- LOG.error(e.getMessage(), e);
- throw new RuntimeException("Unable to instantiate " + readerType + " reader", e);
- }
- }
-
-
- /**
- * Creates a reader that starts reading from 'offset'
- * @param file the file to read
- * @param offset the offset string should be understandable by the reader type being used
- * @return
- * @throws IOException
- */
- private FileReader createFileReader(Path file, String offset)
- throws IOException {
- if ( readerType.equalsIgnoreCase(Configs.SEQ) ) {
- return new SequenceFileReader(this.hdfs, file, conf, offset);
- }
- if ( readerType.equalsIgnoreCase(Configs.TEXT) ) {
- return new TextFileReader(this.hdfs, file, conf, offset);
- }
-
- try {
- Class<?> clsType = Class.forName(readerType);
- Constructor<?> constructor = clsType.getConstructor(FileSystem.class, Path.class, Map.class, String.class);
- return (FileReader) constructor.newInstance(this.hdfs, file, conf, offset);
- } catch (Exception e) {
- LOG.error(e.getMessage(), e);
- throw new RuntimeException("Unable to instantiate " + readerType, e);
- }
- }
-
- /**
- * Renames files with .inprogress suffix
- * @return path of renamed file
- * @throws if operation fails
- */
- private Path renameToInProgressFile(Path file)
- throws IOException {
- Path newFile = new Path( file.toString() + inprogress_suffix );
- try {
- if (hdfs.rename(file, newFile)) {
- return newFile;
- }
- throw new RenameException(file, newFile);
- } catch (IOException e){
- throw new RenameException(file, newFile, e);
- }
- }
-
- /** Returns the corresponding input file in the 'sourceDirPath' for the specified lock file.
- * If no such file is found then returns null
- */
- private Path getFileForLockFile(Path lockFile, Path sourceDirPath)
- throws IOException {
- String lockFileName = lockFile.getName();
- Path dataFile = new Path(sourceDirPath + Path.SEPARATOR + lockFileName + inprogress_suffix);
- if ( hdfs.exists(dataFile) ) {
- return dataFile;
- }
- dataFile = new Path(sourceDirPath + Path.SEPARATOR + lockFileName);
- if ( hdfs.exists(dataFile) ) {
- return dataFile;
- }
- return null;
- }
-
-
- // renames files and returns the new file path
- private Path renameCompletedFile(Path file) throws IOException {
- String fileName = file.toString();
- String fileNameMinusSuffix = fileName.substring(0, fileName.indexOf(inprogress_suffix));
- String newName = new Path(fileNameMinusSuffix).getName();
-
- Path newFile = new Path( archiveDirPath + Path.SEPARATOR + newName );
- LOG.info("Completed consuming file {}", fileNameMinusSuffix);
- if ( !hdfs.rename(file, newFile) ) {
- throw new IOException("Rename failed for file: " + file);
- }
- LOG.debug("Renamed file {} to {} ", file, newFile);
- return newFile;
- }
-
- @Override
- public void declareOutputFields(OutputFieldsDeclarer declarer) {
- if (outputStreamName!=null) {
- declarer.declareStream(outputStreamName, outputFields);
- } else {
- declarer.declare(outputFields);
- }
- }
-
- static class MessageId implements Comparable<MessageId> {
- public long msgNumber; // tracks order in which msg came in
- public String fullPath;
- public FileOffset offset;
-
- public MessageId(long msgNumber, Path fullPath, FileOffset offset) {
- this.msgNumber = msgNumber;
- this.fullPath = fullPath.toString();
- this.offset = offset;
}
- @Override
- public String toString() {
- return "{'" + fullPath + "':" + offset + "}";
+ // renames files and returns the new file path
+ private Path renameCompletedFile(Path file) throws IOException {
+ String fileName = file.toString();
+ String fileNameMinusSuffix = fileName.substring(0, fileName.indexOf(inprogress_suffix));
+ String newName = new Path(fileNameMinusSuffix).getName();
+
+ Path newFile = new Path(archiveDirPath + Path.SEPARATOR + newName);
+ LOG.info("Completed consuming file {}", fileNameMinusSuffix);
+ if (!hdfs.rename(file, newFile)) {
+ throw new IOException("Rename failed for file: " + file);
+ }
+ LOG.debug("Renamed file {} to {} ", file, newFile);
+ return newFile;
}
@Override
- public int compareTo(MessageId rhs) {
- if ( msgNumber<rhs.msgNumber ) {
- return -1;
- }
- if ( msgNumber>rhs.msgNumber ) {
- return 1;
- }
- return 0;
+ public void declareOutputFields(OutputFieldsDeclarer declarer) {
+ if (outputStreamName != null) {
+ declarer.declareStream(outputStreamName, outputFields);
+ } else {
+ declarer.declare(outputFields);
+ }
}
- }
- private static class RenameException extends IOException {
- public final Path oldFile;
- public final Path newFile;
+ static class MessageId implements Comparable<MessageId> {
+
+ public long msgNumber; // tracks order in which msg came in
+ public String fullPath;
+ public FileOffset offset;
- public RenameException(Path oldFile, Path newFile) {
- super("Rename of " + oldFile + " to " + newFile + " failed");
- this.oldFile = oldFile;
- this.newFile = newFile;
+ public MessageId(long msgNumber, Path fullPath, FileOffset offset) {
+ this.msgNumber = msgNumber;
+ this.fullPath = fullPath.toString();
+ this.offset = offset;
+ }
+
+ @Override
+ public String toString() {
+ return "{'" + fullPath + "':" + offset + "}";
+ }
+
+ @Override
+ public int compareTo(MessageId rhs) {
+ if (msgNumber < rhs.msgNumber) {
+ return -1;
+ }
+ if (msgNumber > rhs.msgNumber) {
+ return 1;
+ }
+ return 0;
+ }
}
- public RenameException(Path oldFile, Path newFile, IOException cause) {
- super("Rename of " + oldFile + " to " + newFile + " failed", cause);
- this.oldFile = oldFile;
- this.newFile = newFile;
+ private static class RenameException extends IOException {
+
+ public final Path oldFile;
+ public final Path newFile;
+
+ public RenameException(Path oldFile, Path newFile) {
+ super("Rename of " + oldFile + " to " + newFile + " failed");
+ this.oldFile = oldFile;
+ this.newFile = newFile;
+ }
+
+ public RenameException(Path oldFile, Path newFile, IOException cause) {
+ super("Rename of " + oldFile + " to " + newFile + " failed", cause);
+ this.oldFile = oldFile;
+ this.newFile = newFile;
+ }
}
- }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/83adfd9d/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/avro/TestFixedAvroSerializer.java
----------------------------------------------------------------------
diff --git a/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/avro/TestFixedAvroSerializer.java b/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/avro/TestFixedAvroSerializer.java
index a584f91..b588f5b 100644
--- a/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/avro/TestFixedAvroSerializer.java
+++ b/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/avro/TestFixedAvroSerializer.java
@@ -21,8 +21,7 @@ import org.apache.avro.Schema;
import org.junit.Assert;
import org.junit.Test;
-import java.util.ArrayList;
-import java.util.List;
+import org.junit.BeforeClass;
public class TestFixedAvroSerializer {
//These should match FixedAvroSerializer.config in the test resources
@@ -34,12 +33,13 @@ public class TestFixedAvroSerializer {
"\"name\":\"stormtest2\"," +
"\"fields\":[{\"name\":\"foobar1\",\"type\":\"string\"}," +
"{ \"name\":\"intint1\", \"type\":\"int\" }]}";
- private static final Schema schema1;
- private static final Schema schema2;
+ private static Schema schema1;
+ private static Schema schema2;
final AvroSchemaRegistry reg;
- static {
+ @BeforeClass
+ public static void setupClass() {
Schema.Parser parser = new Schema.Parser();
schema1 = parser.parse(schemaString1);
@@ -58,7 +58,8 @@ public class TestFixedAvroSerializer {
testTheSchema(schema2);
}
- @Test public void testDifferentFPs() {
+ @Test
+ public void testDifferentFPs() {
String fp1 = reg.getFingerprint(schema1);
String fp2 = reg.getFingerprint(schema2);
http://git-wip-us.apache.org/repos/asf/storm/blob/83adfd9d/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/avro/TestGenericAvroSerializer.java
----------------------------------------------------------------------
diff --git a/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/avro/TestGenericAvroSerializer.java b/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/avro/TestGenericAvroSerializer.java
index ddfdcf5..fb97782 100644
--- a/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/avro/TestGenericAvroSerializer.java
+++ b/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/avro/TestGenericAvroSerializer.java
@@ -19,6 +19,7 @@ package org.apache.storm.hdfs.avro;
import org.apache.avro.Schema;
import org.junit.Assert;
+import org.junit.BeforeClass;
import org.junit.Test;
public class TestGenericAvroSerializer {
@@ -30,12 +31,13 @@ public class TestGenericAvroSerializer {
"\"name\":\"stormtest2\"," +
"\"fields\":[{\"name\":\"foobar1\",\"type\":\"string\"}," +
"{ \"name\":\"intint1\", \"type\":\"int\" }]}";
- private static final Schema schema1;
- private static final Schema schema2;
+ private static Schema schema1;
+ private static Schema schema2;
AvroSchemaRegistry reg = new GenericAvroSerializer();
- static {
+ @BeforeClass
+ public static void setupClass() {
Schema.Parser parser = new Schema.Parser();
schema1 = parser.parse(schemaString1);
@@ -50,7 +52,8 @@ public class TestGenericAvroSerializer {
testTheSchema(schema2);
}
- @Test public void testDifferentFPs() {
+ @Test
+ public void testDifferentFPs() {
String fp1 = reg.getFingerprint(schema1);
String fp2 = reg.getFingerprint(schema2);
[21/50] [abbrv] storm git commit: STORM-2784: storm-kafka-client
KafkaTupleListener method onPartitionsReassigned() should be called after
initialization is complete
Posted by pt...@apache.org.
STORM-2784: storm-kafka-client KafkaTupleListener method onPartitionsReassigned() should be called after initialization is complete
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/667ffee1
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/667ffee1
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/667ffee1
Branch: refs/heads/metrics_v2
Commit: 667ffee1580d77469c5eaadd7b5651972a316fc5
Parents: 502ed7e
Author: Hugo Louro <hm...@gmail.com>
Authored: Sun Oct 22 21:35:29 2017 -0700
Committer: Jungtaek Lim <ka...@gmail.com>
Committed: Fri Oct 27 15:12:09 2017 +0900
----------------------------------------------------------------------
.../src/main/java/org/apache/storm/kafka/spout/KafkaSpout.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/667ffee1/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpout.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpout.java b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpout.java
index 68bce11..5022862 100644
--- a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpout.java
+++ b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpout.java
@@ -165,8 +165,8 @@ public class KafkaSpout<K, V> extends BaseRichSpout {
LOG.info("Partitions reassignment. [task-ID={}, consumer-group={}, consumer={}, topic-partitions={}]",
context.getThisTaskId(), kafkaSpoutConfig.getConsumerGroupId(), kafkaConsumer, partitions);
- tupleListener.onPartitionsReassigned(partitions);
initialize(partitions);
+ tupleListener.onPartitionsReassigned(partitions);
}
private void initialize(Collection<TopicPartition> partitions) {
[10/50] [abbrv] storm git commit: Merge branch 'STORM-2666-clean-1.x'
of https://github.com/srdo/storm into STORM-2666-1.x-merge
Posted by pt...@apache.org.
Merge branch 'STORM-2666-clean-1.x' of https://github.com/srdo/storm into STORM-2666-1.x-merge
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/644f5cb9
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/644f5cb9
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/644f5cb9
Branch: refs/heads/metrics_v2
Commit: 644f5cb98d6d5b810ac23a04a01f62edd42c44b7
Parents: e83878a 54fdcbc
Author: Jungtaek Lim <ka...@gmail.com>
Authored: Tue Oct 10 08:10:43 2017 +0900
Committer: Jungtaek Lim <ka...@gmail.com>
Committed: Tue Oct 10 08:10:43 2017 +0900
----------------------------------------------------------------------
.../apache/storm/kafka/spout/KafkaSpout.java | 50 +++-
.../KafkaSpoutRetryExponentialBackoff.java | 32 +-
.../kafka/spout/KafkaSpoutRetryService.java | 6 +-
.../kafka/spout/internal/OffsetManager.java | 8 +-
.../storm/kafka/spout/KafkaSpoutCommitTest.java | 23 +-
.../kafka/spout/KafkaSpoutRebalanceTest.java | 56 +++-
.../KafkaSpoutRetryExponentialBackoffTest.java | 292 +++++++++++++++++++
.../kafka/spout/SingleTopicKafkaSpoutTest.java | 62 +++-
.../kafka/spout/internal/OffsetManagerTest.java | 84 ++++++
9 files changed, 556 insertions(+), 57 deletions(-)
----------------------------------------------------------------------
[32/50] [abbrv] storm git commit: STORM-2535: Replace
test-reset-timeout with a more reliable test
Posted by pt...@apache.org.
STORM-2535: Replace test-reset-timeout with a more reliable test
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/6a41b22b
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/6a41b22b
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/6a41b22b
Branch: refs/heads/metrics_v2
Commit: 6a41b22bd7b633653633b868bcd4ca9cb8b858f5
Parents: c70d7d4
Author: Stig Rohde Døssing <sd...@it-minds.dk>
Authored: Thu May 25 22:32:42 2017 +0200
Committer: Stig Rohde Døssing <sr...@apache.org>
Committed: Sat Nov 11 13:30:56 2017 +0100
----------------------------------------------------------------------
.../org/apache/storm/integration_test.clj | 53 ++++++++++++++------
1 file changed, 37 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/6a41b22b/storm-core/test/clj/integration/org/apache/storm/integration_test.clj
----------------------------------------------------------------------
diff --git a/storm-core/test/clj/integration/org/apache/storm/integration_test.clj b/storm-core/test/clj/integration/org/apache/storm/integration_test.clj
index 775949e..a147713 100644
--- a/storm-core/test/clj/integration/org/apache/storm/integration_test.clj
+++ b/storm-core/test/clj/integration/org/apache/storm/integration_test.clj
@@ -129,19 +129,25 @@
(advance-cluster-time cluster 12)
(assert-failed tracker 2)
)))
-
-(defbolt extend-timeout-twice {} {:prepare true}
+
+(defbolt reset-timeout-bolt {} {:prepare true}
[conf context collector]
- (let [state (atom -1)]
+ (let [tuple-counter (atom 1)
+ first-tuple (atom nil)]
(bolt
(execute [tuple]
- (do
- (Time/sleep (* 8 1000))
- (reset-timeout! collector tuple)
- (Time/sleep (* 8 1000))
- (reset-timeout! collector tuple)
- (Time/sleep (* 8 1000))
- (ack! collector tuple)
+ (do
+ (condp = @tuple-counter
+ 1 (reset! first-tuple tuple)
+ 2 (reset-timeout! collector @first-tuple)
+ 5 (do
+ (ack! collector @first-tuple)
+ (ack! collector tuple)
+ )
+ (do
+ (reset-timeout! collector @first-tuple)
+ (ack! collector tuple)))
+ (swap! tuple-counter inc)
)))))
(deftest test-reset-timeout
@@ -151,18 +157,33 @@
_ (.setAckFailDelegate feeder tracker)
topology (thrift/mk-topology
{"1" (thrift/mk-spout-spec feeder)}
- {"2" (thrift/mk-bolt-spec {"1" :global} extend-timeout-twice)})]
+ {"2" (thrift/mk-bolt-spec {"1" :global} reset-timeout-bolt)})]
(submit-local-topology (:nimbus cluster)
"timeout-tester"
{TOPOLOGY-MESSAGE-TIMEOUT-SECS 10}
topology)
- (advance-cluster-time cluster 11)
+ ;The first tuple will be used to check timeout reset
(.feed feeder ["a"] 1)
- (advance-cluster-time cluster 21)
+ ;The second tuple is used to wait for the spout to rotate the spout's pending map
+ (.feed feeder ["b"] 2)
+ (advance-cluster-time cluster 9)
+ ;The other tuples are used to reset the first tuple's timeout,
+ ;and to wait for the message to get through to the spout (acks use the same path as timeout resets)
+ (.feed feeder ["c"] 3)
+ (assert-acked tracker 3)
+ (advance-cluster-time cluster 9)
+ (.feed feeder ["d"], 4)
+ (assert-acked tracker 4)
+ (advance-cluster-time cluster 2)
+ ;The time is now twice the message timeout, the second tuple should expire since it was not acked
+ ;Waiting for this also ensures that the first tuple gets failed if reset-timeout doesn't work
+ (assert-failed tracker 2)
+ ;Put in a tuple to cause the first tuple to be acked
+ (.feed feeder ["e"], 5)
+ (assert-acked tracker 5)
+ ;The first tuple should be acked and should not have failed
(is (not (.isFailed tracker 1)))
- (is (not (.isAcked tracker 1)))
- (advance-cluster-time cluster 5)
- (assert-acked tracker 1)
+ (is (.isAcked tracker 1))
)))
(defn mk-validate-topology-1 []
[28/50] [abbrv] storm git commit: Merge branch 'master' of
https://github.com/lawrencecraft/storm
Posted by pt...@apache.org.
Merge branch 'master' of https://github.com/lawrencecraft/storm
Fixing markdown for HBase bolt README in Storm
This closes #2396
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/7f22203c
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/7f22203c
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/7f22203c
Branch: refs/heads/metrics_v2
Commit: 7f22203c4e45fbbf131cf8d3856d7f50024aa76e
Parents: 5d0889b
Author: Robert Evans <ev...@yahoo-inc.com>
Authored: Wed Nov 8 08:55:24 2017 -0600
Committer: Robert Evans <ev...@yahoo-inc.com>
Committed: Wed Nov 8 08:56:07 2017 -0600
----------------------------------------------------------------------
external/storm-hbase/README.md | 22 +++++++++++-----------
1 file changed, 11 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/7f22203c/external/storm-hbase/README.md
----------------------------------------------------------------------
diff --git a/external/storm-hbase/README.md b/external/storm-hbase/README.md
index 2013357..8e0ff07 100644
--- a/external/storm-hbase/README.md
+++ b/external/storm-hbase/README.md
@@ -1,4 +1,4 @@
-#Storm HBase
+# Storm HBase
Storm/Trident integration for [Apache HBase](https://hbase.apache.org)
@@ -46,7 +46,7 @@ config.put("storm.kerberos.principal", "$principle");
StormSubmitter.submitTopology("$topologyName", config, builder.createTopology());
```
-##Working with Secure HBASE using delegation tokens.
+## Working with Secure HBASE using delegation tokens.
If your topology is going to interact with secure HBase, your bolts/states needs to be authenticated by HBase.
The approach described above requires that all potential worker hosts have "storm.keytab.file" on them. If you have
multiple topologies on a cluster , each with different hbase user, you will have to create multiple keytabs and distribute
@@ -131,17 +131,17 @@ HBaseBolt hbase = new HBaseBolt("WordCount", mapper);
HBaseBolt params
-|Arg |Description | Type | Default |
-|--- |--- |---
-|writeToWAL | To turn Durability SYNC_WAL or SKIP_WAL | Boolean (Optional) | True |
-|configKey | Any Hbase related configs | Map (Optional) | |
-|batchSize | Max no.of Tuples batched together to write to HBase | Int (Optional) | 15000 |
-|flushIntervalSecs| (In seconds) If > 0 HBase Bolt will periodically flush transaction batches. Enabling this is recommended to avoid tuple timeouts while waiting for a batch to fill up. | Int (Optional) | 0 |
+| Arg | Description | Type | Default |
+| ---------------- |---------------------------------------------------- |------------------- | ------- |
+| writeToWAL | To turn Durability SYNC_WAL or SKIP_WAL | Boolean (Optional) | True |
+| configKey | Any Hbase related configs | Map (Optional) | |
+| batchSize | Max no.of Tuples batched together to write to HBase | Int (Optional) | 15000 |
+| flushIntervalSecs| (In seconds) If > 0 HBase Bolt will periodically flush transaction batches. Enabling this is recommended to avoid tuple timeouts while waiting for a batch to fill up. | Int (Optional) | 0 |
The `HBaseBolt` will delegate to the `mapper` instance to figure out how to persist tuple data to HBase.
-###HBaseValueMapper
+### HBaseValueMapper
This class allows you to transform the HBase lookup result into storm Values that will be emitted by the `HBaseLookupBolt`.
```java
@@ -158,7 +158,7 @@ The `declareOutputFields` should be used to declare the outputFields of the `HBa
There is an example implementation in `src/test/java` directory.
-###HBaseProjectionCriteria
+### HBaseProjectionCriteria
This class allows you to specify the projection criteria for your HBase Get function. This is optional parameter
for the lookupBolt and if you do not specify this instance all the columns will be returned by `HBaseLookupBolt`.
@@ -183,7 +183,7 @@ HBaseProjectionCriteria projectionCriteria = new HBaseProjectionCriteria()
.addColumnFamily("cf2");
```
-###HBaseLookupBolt
+### HBaseLookupBolt
To use the `HBaseLookupBolt`, Construct it with the name of the table to write to, an implementation of `HBaseMapper`
and an implementation of `HBaseRowToStormValueMapper`. You can optionally specify a `HBaseProjectionCriteria`.
[16/50] [abbrv] storm git commit: Merge branch 'STORM-2779-1.x' of
https://github.com/HeartSaVioR/storm into STORM-2779-1.x-merge
Posted by pt...@apache.org.
Merge branch 'STORM-2779-1.x' of https://github.com/HeartSaVioR/storm into STORM-2779-1.x-merge
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/9aff9f91
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/9aff9f91
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/9aff9f91
Branch: refs/heads/metrics_v2
Commit: 9aff9f91baafc6f849aa7febb7d0046162c34816
Parents: 148ee66 40384cf
Author: Jungtaek Lim <ka...@gmail.com>
Authored: Wed Oct 18 13:59:30 2017 +0900
Committer: Jungtaek Lim <ka...@gmail.com>
Committed: Wed Oct 18 13:59:30 2017 +0900
----------------------------------------------------------------------
.../src/jvm/org/apache/storm/topology/WindowedBoltExecutor.java | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
[11/50] [abbrv] storm git commit: Merge branch 'STORM-2771' of
https://github.com/revans2/incubator-storm into STORM-2771
Posted by pt...@apache.org.
Merge branch 'STORM-2771' of https://github.com/revans2/incubator-storm into STORM-2771
STORM-2771: By default don't run any tests as integration tests
This closes #2368
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/90e13852
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/90e13852
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/90e13852
Branch: refs/heads/metrics_v2
Commit: 90e1385264104028ef720df4ee89d4f194a5e233
Parents: 644f5cb
Author: Robert Evans <ev...@yahoo-inc.com>
Authored: Fri Oct 13 10:45:20 2017 -0500
Committer: Robert Evans <ev...@yahoo-inc.com>
Committed: Fri Oct 13 11:00:13 2017 -0500
----------------------------------------------------------------------
pom.xml | 6 +++---
1 file changed, 3 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/90e13852/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index f53171b..21ec2bc 100644
--- a/pom.xml
+++ b/pom.xml
@@ -281,6 +281,7 @@
<!-- Java and clojure build lifecycle test properties are defined here to avoid having to create a default profile -->
<java.unit.test.exclude>org.apache.storm.testing.IntegrationTest</java.unit.test.exclude>
<java.unit.test.include>**/Test*.java, **/*Test.java, **/*TestCase.java</java.unit.test.include> <!--maven surefire plugin default test list-->
+ <java.integration.test.include>no.tests</java.integration.test.include>
<!-- by default the clojure test set are all clojure tests that are not integration tests. This property is overridden in the profiles -->
<clojure.test.set>!integration.*</clojure.test.set>
@@ -553,7 +554,6 @@
<id>all-tests</id>
<properties>
<java.integration.test.include>**/*.java</java.integration.test.include>
- <java.integration.test.group>org.apache.storm.testing.IntegrationTest</java.integration.test.group>
<clojure.test.set>*.*</clojure.test.set>
</properties>
</profile>
@@ -563,7 +563,6 @@
<!--Java-->
<java.unit.test.include>no.unit.tests</java.unit.test.include>
<java.integration.test.include>**/*.java</java.integration.test.include>
- <java.integration.test.group>org.apache.storm.testing.IntegrationTest</java.integration.test.group>
<!--Clojure-->
<clojure.test.set>integration.*</clojure.test.set>
<clojure.test.declared.namespace.only>true</clojure.test.declared.namespace.only>
@@ -1032,10 +1031,11 @@
<artifactId>maven-failsafe-plugin</artifactId>
<version>${maven-surefire.version}</version>
<configuration>
+ <redirectTestOutputToFile>true</redirectTestOutputToFile>
<includes>
<include>${java.integration.test.include}</include>
</includes>
- <groups>${java.integration.test.group}</groups> <!--set in integration-test the profile-->
+ <groups>org.apache.storm.testing.IntegrationTest</groups>
<argLine>-Xmx1536m</argLine>
</configuration>
<executions>
[03/50] [abbrv] storm git commit: Merge branch 'STORM-2756' of
https://github.com/srdo/storm into STORM-2756-1.x-merge
Posted by pt...@apache.org.
Merge branch 'STORM-2756' of https://github.com/srdo/storm into STORM-2756-1.x-merge
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/fca692da
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/fca692da
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/fca692da
Branch: refs/heads/metrics_v2
Commit: fca692da38645686e9e0c199bfb8514ae6a81121
Parents: 5890333 386ab20
Author: Jungtaek Lim <ka...@gmail.com>
Authored: Mon Oct 2 15:33:13 2017 +0900
Committer: Jungtaek Lim <ka...@gmail.com>
Committed: Mon Oct 2 15:33:13 2017 +0900
----------------------------------------------------------------------
.../storm/kafka/spout/KafkaSpoutConfig.java | 27 ++++++++++++++++++++
1 file changed, 27 insertions(+)
----------------------------------------------------------------------
[25/50] [abbrv] storm git commit: STORM-2781: Refactor
storm-kafka-client KafkaSpout Processing Guarantees
Posted by pt...@apache.org.
STORM-2781: Refactor storm-kafka-client KafkaSpout Processing Guarantees
- Define processing guarantees as AT_LEAST_ONCE, AT_MOST_ONCE, NONE
- Refactor method name from setForceEnableTupleTracking to setTupleTrackingEnforced
- Throw IllegalStateException instead of IllegalArgumentException if spout attempts to emit an already committed message
- Update documentation to reflect these changes
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/ac16fe1e
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/ac16fe1e
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/ac16fe1e
Branch: refs/heads/metrics_v2
Commit: ac16fe1ee9d974af64a30769819561c0abae23af
Parents: 3414a8c
Author: Hugo Louro <hm...@gmail.com>
Authored: Sun Oct 22 17:44:54 2017 -0700
Committer: Hugo Louro <hm...@gmail.com>
Committed: Sun Oct 29 16:07:53 2017 -0700
----------------------------------------------------------------------
docs/storm-kafka-client.md | 37 ++++++--
.../apache/storm/kafka/spout/KafkaSpout.java | 94 +++++++++++---------
.../storm/kafka/spout/KafkaSpoutConfig.java | 55 +++++++-----
.../spout/KafkaSpoutMessagingGuaranteeTest.java | 14 +--
4 files changed, 121 insertions(+), 79 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/ac16fe1e/docs/storm-kafka-client.md
----------------------------------------------------------------------
diff --git a/docs/storm-kafka-client.md b/docs/storm-kafka-client.md
index 841e8ef..0a12910 100644
--- a/docs/storm-kafka-client.md
+++ b/docs/storm-kafka-client.md
@@ -316,7 +316,7 @@ use Kafka-clients 0.10.0.0, you would use the following dependency in your `pom.
You can also override the kafka clients version while building from maven, with parameter `storm.kafka.client.version`
e.g. `mvn clean install -Dstorm.kafka.client.version=0.10.0.0`
-When selecting a kafka client version, you should ensure -
+When selecting a kafka client version, you should ensure -
1. kafka api is compatible. storm-kafka-client module only supports **0.10 or newer** kafka client API. For older versions,
you can use storm-kafka module (https://github.com/apache/storm/tree/master/external/storm-kafka).
2. The kafka client selected by you should be wire compatible with the broker. e.g. 0.9.x client will not work with
@@ -348,25 +348,46 @@ Currently the Kafka spout has has the following default values, which have been
* max.uncommitted.offsets = 10000000
<br/>
-# Messaging reliability modes
+# Processing Guarantees
-In some cases you may not need or want the spout to guarantee at-least-once processing of messages. The spout also supports at-most-once and any-times modes. At-most-once guarantees that any tuple emitted to the topology will never be reemitted. Any-times makes no guarantees, but may reduce the overhead of committing offsets to Kafka in cases where you truly don't care how many times a message is processed.
+The `KafkaSpoutConfig.ProcessingGuarantee` enum parameter controls when an offset is committed to Kafka. This is
+conceptually equivalent to marking the tuple with the `ConsumerRecord` for that offset as being successfully processed
+because the tuple won't get re-emitted in case of failure or time out.
+
+For the AT_LEAST_ONCE and AT_MOST_ONCE processing guarantees the spout controls when the commit happens.
+When the guarantee is NONE Kafka controls when the commit happens.
+
+* AT_LEAST_ONCE - an offset is ready to commit only after the corresponding tuple has been processed (at-least-once)
+ and acked. If a tuple fails or times out it will be re-emitted. A tuple can be processed more than once if for instance
+ the ack gets lost.
+
+* AT_MOST_ONCE - Offsets will be committed to Kafka right after being polled but before being emitted to the downstream
+ components of the topology. Offsets are processed at most once because tuples that fail or timeout won't be retried.
+
+* NONE - the polled offsets are committed to Kafka periodically as controlled by the Kafka properties
+ "enable.auto.commit" and "auto.commit.interval.ms". Because the spout does not control when the commit happens
+ it cannot give any message processing guarantees, i.e. a message may be processed 0, 1 or more times.
+ This option requires "enable.auto.commit=true". If "enable.auto.commit=false" an exception will be thrown.
+
+To set the processing guarantee use the `KafkaSpoutConfig.Builder.setProcessingGuarantee` method as follows:
-To set the processing guarantee, use the KafkaSpoutConfig.Builder.setProcessingGuarantee method, e.g.
```java
KafkaSpoutConfig<String, String> kafkaConf = KafkaSpoutConfig
.builder(String bootstrapServers, String ... topics)
.setProcessingGuarantee(ProcessingGuarantee.AT_MOST_ONCE)
```
-The spout will disable tuple tracking for emitted tuples by default when you use at-most-once or any-times. In some cases you may want to enable tracking anyway, because tuple tracking is necessary for some features of Storm, e.g. showing complete latency in Storm UI, or enabling backpressure through the `Config.TOPOLOGY_MAX_SPOUT_PENDING` parameter.
+# Tuple Tracking
+
+By default the spout only tracks emitted tuples when the processing guarantee is AT_LEAST_ONCE. It may be necessary to track
+emitted tuples with other processing guarantees to benefit from Storm features such as showing complete latency in the UI,
+or enabling backpressure with Config.TOPOLOGY_MAX_SPOUT_PENDING.
-If you need to enable tracking, use the KafkaSpoutConfig.Builder.setForceEnableTupleTracking method, e.g.
```java
KafkaSpoutConfig<String, String> kafkaConf = KafkaSpoutConfig
.builder(String bootstrapServers, String ... topics)
.setProcessingGuarantee(ProcessingGuarantee.AT_MOST_ONCE)
- .setForceEnableTupleTracking(true)
+ .setTupleTrackingEnforced(true)
```
-Note that this setting has no effect in at-least-once mode, where tuple tracking is always enabled.
\ No newline at end of file
+Note: This setting has no effect with AT_LEAST_ONCE processing guarantee, where tuple tracking is required and therefore always enabled.
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/storm/blob/ac16fe1e/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpout.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpout.java b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpout.java
index 5022862..0a4d788 100644
--- a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpout.java
+++ b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpout.java
@@ -78,17 +78,17 @@ public class KafkaSpout<K, V> extends BaseRichSpout {
private transient KafkaSpoutRetryService retryService;
// Handles tuple events (emit, ack etc.)
private transient KafkaTupleListener tupleListener;
- // timer == null for modes other than at-least-once
+ // timer == null if processing guarantee is none or at-most-once
private transient Timer commitTimer;
// Flag indicating that the spout is still undergoing initialization process.
private transient boolean initialized;
// Initialization is only complete after the first call to KafkaSpoutConsumerRebalanceListener.onPartitionsAssigned()
// Tuples that were successfully acked/emitted. These tuples will be committed periodically when the commit timer expires,
- //or after a consumer rebalance, or during close/deactivate. Always empty if not using at-least-once mode.
+ // or after a consumer rebalance, or during close/deactivate. Always empty if processing guarantee is none or at-most-once.
private transient Map<TopicPartition, OffsetManager> offsetManagers;
// Tuples that have been emitted but that are "on the wire", i.e. pending being acked or failed.
- // Always empty if not using at-least-once mode.
+ // Always empty if processing guarantee is none or at-most-once
private transient Set<KafkaSpoutMessageId> emitted;
// Records that have been polled and are queued to be emitted in the nextTuple() call. One record is emitted per nextTuple()
private transient Iterator<ConsumerRecord<K, V>> waitingToEmit;
@@ -124,11 +124,9 @@ public class KafkaSpout<K, V> extends BaseRichSpout {
retryService = kafkaSpoutConfig.getRetryService();
tupleListener = kafkaSpoutConfig.getTupleListener();
- tupleListener.open(conf, context);
-
- if (isAtLeastOnce()) {
- // Only used if the spout commits offsets for acked tuples
+ if (isAtLeastOnceProcessing()) {
+ // Only used if the spout should commit an offset to Kafka only after the corresponding tuple has been acked.
commitTimer = new Timer(TIMER_DELAY_MS, kafkaSpoutConfig.getOffsetsCommitPeriodMs(), TimeUnit.MILLISECONDS);
}
refreshSubscriptionTimer = new Timer(TIMER_DELAY_MS, kafkaSpoutConfig.getPartitionRefreshPeriodMs(), TimeUnit.MILLISECONDS);
@@ -137,10 +135,12 @@ public class KafkaSpout<K, V> extends BaseRichSpout {
emitted = new HashSet<>();
waitingToEmit = Collections.emptyListIterator();
+ tupleListener.open(conf, context);
+
LOG.info("Kafka Spout opened with the following configuration: {}", kafkaSpoutConfig);
}
- private boolean isAtLeastOnce() {
+ private boolean isAtLeastOnceProcessing() {
return kafkaSpoutConfig.getProcessingGuarantee() == KafkaSpoutConfig.ProcessingGuarantee.AT_LEAST_ONCE;
}
@@ -154,7 +154,7 @@ public class KafkaSpout<K, V> extends BaseRichSpout {
LOG.info("Partitions revoked. [consumer-group={}, consumer={}, topic-partitions={}]",
kafkaSpoutConfig.getConsumerGroupId(), kafkaConsumer, partitions);
previousAssignment = partitions;
- if (isAtLeastOnce() && initialized) {
+ if (isAtLeastOnceProcessing() && initialized) {
initialized = false;
commitOffsetsForAckedTuples();
}
@@ -165,13 +165,14 @@ public class KafkaSpout<K, V> extends BaseRichSpout {
LOG.info("Partitions reassignment. [task-ID={}, consumer-group={}, consumer={}, topic-partitions={}]",
context.getThisTaskId(), kafkaSpoutConfig.getConsumerGroupId(), kafkaConsumer, partitions);
- initialize(partitions);
tupleListener.onPartitionsReassigned(partitions);
+ initialize(partitions);
}
private void initialize(Collection<TopicPartition> partitions) {
- if (isAtLeastOnce()) {
- offsetManagers.keySet().retainAll(partitions); // remove from acked all partitions that are no longer assigned to this spout
+ if (isAtLeastOnceProcessing()) {
+ // remove from acked all partitions that are no longer assigned to this spout
+ offsetManagers.keySet().retainAll(partitions);
retryService.retainAll(partitions);
/*
@@ -193,7 +194,7 @@ public class KafkaSpout<K, V> extends BaseRichSpout {
final OffsetAndMetadata committedOffset = kafkaConsumer.committed(tp);
final long fetchOffset = doSeek(tp, committedOffset);
// If this partition was previously assigned to this spout, leave the acked offsets as they were to resume where it left off
- if (isAtLeastOnce() && !offsetManagers.containsKey(tp)) {
+ if (isAtLeastOnceProcessing() && !offsetManagers.containsKey(tp)) {
offsetManagers.put(tp, new OffsetManager(tp, fetchOffset));
}
}
@@ -260,26 +261,27 @@ public class KafkaSpout<K, V> extends BaseRichSpout {
}
private boolean commit() {
- return isAtLeastOnce() && commitTimer.isExpiredResetOnTrue(); // timer != null for non auto commit mode
+ return isAtLeastOnceProcessing() && commitTimer.isExpiredResetOnTrue(); // timer != null for non auto commit mode
}
private boolean poll() {
final int maxUncommittedOffsets = kafkaSpoutConfig.getMaxUncommittedOffsets();
final int readyMessageCount = retryService.readyMessageCount();
final boolean poll = !waitingToEmit()
- //Check that the number of uncommitted, nonretriable tuples is less than the maxUncommittedOffsets limit
- //Accounting for retriable tuples this way still guarantees that the limit is followed on a per partition basis,
- //and prevents locking up the spout when there are too many retriable tuples
- && (numUncommittedOffsets - readyMessageCount < maxUncommittedOffsets
- || !isAtLeastOnce());
+ // Check that the number of uncommitted, non-retriable tuples is less than the maxUncommittedOffsets limit.
+ // Accounting for retriable tuples in this way still guarantees that the limit is followed on a per partition basis,
+ // and prevents locking up the spout when there are too many retriable tuples
+ && (numUncommittedOffsets - readyMessageCount < maxUncommittedOffsets || !isAtLeastOnceProcessing());
if (!poll) {
if (waitingToEmit()) {
- LOG.debug("Not polling. Tuples waiting to be emitted. [{}] uncommitted offsets across all topic partitions", numUncommittedOffsets);
+ LOG.debug("Not polling. Tuples waiting to be emitted."
+ + " [{}] uncommitted offsets across all topic partitions", numUncommittedOffsets);
}
- if (numUncommittedOffsets >= maxUncommittedOffsets && isAtLeastOnce()) {
- LOG.debug("Not polling. [{}] uncommitted offsets across all topic partitions has reached the threshold of [{}]", numUncommittedOffsets, maxUncommittedOffsets);
+ if (numUncommittedOffsets >= maxUncommittedOffsets && isAtLeastOnceProcessing()) {
+ LOG.debug("Not polling. [{}] uncommitted offsets across all topic partitions has reached the threshold of [{}]",
+ numUncommittedOffsets, maxUncommittedOffsets);
}
}
return poll;
@@ -289,8 +291,8 @@ public class KafkaSpout<K, V> extends BaseRichSpout {
return waitingToEmit != null && waitingToEmit.hasNext();
}
- public void setWaitingToEmit(ConsumerRecords<K,V> consumerRecords) {
- List<ConsumerRecord<K,V>> waitingToEmitList = new LinkedList<>();
+ private void setWaitingToEmit(ConsumerRecords<K, V> consumerRecords) {
+ List<ConsumerRecord<K, V>> waitingToEmitList = new LinkedList<>();
for (TopicPartition tp : consumerRecords.partitions()) {
waitingToEmitList.addAll(consumerRecords.records(tp));
}
@@ -305,7 +307,8 @@ public class KafkaSpout<K, V> extends BaseRichSpout {
}
final ConsumerRecords<K, V> consumerRecords = kafkaConsumer.poll(kafkaSpoutConfig.getPollTimeoutMs());
final int numPolledRecords = consumerRecords.count();
- LOG.debug("Polled [{}] records from Kafka. [{}] uncommitted offsets across all topic partitions", numPolledRecords, numUncommittedOffsets);
+ LOG.debug("Polled [{}] records from Kafka. [{}] uncommitted offsets across all topic partitions",
+ numPolledRecords, numUncommittedOffsets);
if (kafkaSpoutConfig.getProcessingGuarantee() == KafkaSpoutConfig.ProcessingGuarantee.AT_MOST_ONCE) {
//Commit polled records immediately to ensure delivery is at-most-once.
kafkaConsumer.commitSync();
@@ -330,7 +333,7 @@ public class KafkaSpout<K, V> extends BaseRichSpout {
}
/**
- * Creates a tuple from the kafka record and emits it if it was not yet emitted
+ * Creates a tuple from the kafka record and emits it if it was not yet emitted.
*
* @param record to be emitted
* @return true if tuple was emitted. False if tuple has been acked or has been emitted and is pending ack or fail
@@ -338,22 +341,25 @@ public class KafkaSpout<K, V> extends BaseRichSpout {
private boolean emitTupleIfNotEmitted(ConsumerRecord<K, V> record) {
final TopicPartition tp = new TopicPartition(record.topic(), record.partition());
final KafkaSpoutMessageId msgId = retryService.getMessageId(record);
+
if (offsetManagers.containsKey(tp) && offsetManagers.get(tp).contains(msgId)) { // has been acked
LOG.trace("Tuple for record [{}] has already been acked. Skipping", record);
- } else if (emitted.contains(msgId)) { // has been emitted and it's pending ack or fail
+ } else if (emitted.contains(msgId)) { // has been emitted and it is pending ack or fail
LOG.trace("Tuple for record [{}] has already been emitted. Skipping", record);
} else {
- Validate.isTrue(kafkaConsumer.committed(tp) == null || kafkaConsumer.committed(tp).offset() < kafkaConsumer.position(tp),
- "The spout is about to emit a message that has already been committed."
- + " This should never occur, and indicates a bug in the spout");
+ if (kafkaConsumer.committed(tp) != null && (kafkaConsumer.committed(tp).offset() >= kafkaConsumer.position(tp))) {
+ throw new IllegalStateException("Attempting to emit a message that has already been committed.");
+ }
+
final List<Object> tuple = kafkaSpoutConfig.getTranslator().apply(record);
if (isEmitTuple(tuple)) {
final boolean isScheduled = retryService.isScheduled(msgId);
// not scheduled <=> never failed (i.e. never emitted), or scheduled and ready to be retried
if (!isScheduled || retryService.isReady(msgId)) {
- String stream = tuple instanceof KafkaTuple ? ((KafkaTuple) tuple).getStream() : Utils.DEFAULT_STREAM_ID;
- if (!isAtLeastOnce()) {
- if (kafkaSpoutConfig.getForceEnableTupleTracking()) {
+ final String stream = tuple instanceof KafkaTuple ? ((KafkaTuple) tuple).getStream() : Utils.DEFAULT_STREAM_ID;
+
+ if (!isAtLeastOnceProcessing()) {
+ if (kafkaSpoutConfig.isTupleTrackingEnforced()) {
collector.emit(stream, tuple, msgId);
LOG.trace("Emitted tuple [{}] for record [{}] with msgId [{}]", tuple, record, msgId);
} else {
@@ -369,7 +375,7 @@ public class KafkaSpout<K, V> extends BaseRichSpout {
numUncommittedOffsets++;
}
collector.emit(stream, tuple, msgId);
- tupleListener.onEmit(tuple, msgId);
+ tupleListener.onEmit(tuple, msgId);
LOG.trace("Emitted tuple [{}] for record [{}] with msgId [{}]", tuple, record, msgId);
}
return true;
@@ -384,7 +390,7 @@ public class KafkaSpout<K, V> extends BaseRichSpout {
}
/**
- * Emits a tuple if it is not a null tuple, or if the spout is configured to emit null tuples
+ * Emits a tuple if it is not a null tuple, or if the spout is configured to emit null tuples.
*/
private boolean isEmitTuple(List<Object> tuple) {
return tuple != null || kafkaSpoutConfig.isEmitNullTuples();
@@ -440,11 +446,12 @@ public class KafkaSpout<K, V> extends BaseRichSpout {
// ======== Ack =======
@Override
public void ack(Object messageId) {
- if (!isAtLeastOnce()) {
- // Only need to keep track of acked tuples if commits are done based on acks
+ if (!isAtLeastOnceProcessing()) {
return;
}
+ // Only need to keep track of acked tuples if commits to Kafka are controlled by
+ // tuple acks, which happens only for at-least-once processing semantics
final KafkaSpoutMessageId msgId = (KafkaSpoutMessageId) messageId;
if (!emitted.contains(msgId)) {
if (msgId.isEmitted()) {
@@ -466,19 +473,22 @@ public class KafkaSpout<K, V> extends BaseRichSpout {
// ======== Fail =======
@Override
public void fail(Object messageId) {
- if (!isAtLeastOnce()) {
- // Only need to keep track of failed tuples if commits are done based on acks
+ if (!isAtLeastOnceProcessing()) {
return;
}
-
+ // Only need to keep track of failed tuples if commits to Kafka are controlled by
+ // tuple acks, which happens only for at-least-once processing semantics
final KafkaSpoutMessageId msgId = (KafkaSpoutMessageId) messageId;
if (!emitted.contains(msgId)) {
- LOG.debug("Received fail for tuple this spout is no longer tracking. Partitions may have been reassigned. Ignoring message [{}]", msgId);
+ LOG.debug("Received fail for tuple this spout is no longer tracking."
+ + " Partitions may have been reassigned. Ignoring message [{}]", msgId);
return;
}
Validate.isTrue(!retryService.isScheduled(msgId), "The message id " + msgId + " is queued for retry while being failed."
+ " This should never occur barring errors in the RetryService implementation or the spout code.");
+
msgId.incrementNumFails();
+
if (!retryService.schedule(msgId)) {
LOG.debug("Reached maximum number of retries. Message [{}] being marked as acked.", msgId);
// this tuple should be removed from emitted only inside the ack() method. This is to ensure
@@ -527,7 +537,7 @@ public class KafkaSpout<K, V> extends BaseRichSpout {
private void shutdown() {
try {
- if (isAtLeastOnce()) {
+ if (isAtLeastOnceProcessing()) {
commitOffsetsForAckedTuples();
}
} finally {
http://git-wip-us.apache.org/repos/asf/storm/blob/ac16fe1e/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpoutConfig.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpoutConfig.java b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpoutConfig.java
index b7de812..758979c 100644
--- a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpoutConfig.java
+++ b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpoutConfig.java
@@ -56,10 +56,13 @@ public class KafkaSpoutConfig<K, V> implements Serializable {
public static final int DEFAULT_MAX_UNCOMMITTED_OFFSETS = 10_000_000;
// 2s
public static final long DEFAULT_PARTITION_REFRESH_PERIOD_MS = 2_000;
+
public static final FirstPollOffsetStrategy DEFAULT_FIRST_POLL_OFFSET_STRATEGY = FirstPollOffsetStrategy.UNCOMMITTED_EARLIEST;
+
public static final KafkaSpoutRetryService DEFAULT_RETRY_SERVICE =
new KafkaSpoutRetryExponentialBackoff(TimeInterval.seconds(0), TimeInterval.milliSeconds(2),
DEFAULT_MAX_RETRIES, TimeInterval.seconds(10));
+
public static final ProcessingGuarantee DEFAULT_PROCESSING_GUARANTEE = ProcessingGuarantee.AT_LEAST_ONCE;
public static final KafkaTupleListener DEFAULT_TUPLE_LISTENER = new EmptyKafkaTupleListener();
@@ -83,7 +86,7 @@ public class KafkaSpoutConfig<K, V> implements Serializable {
private final SerializableDeserializer<V> valueDes;
private final Class<? extends Deserializer<V>> valueDesClazz;
private final ProcessingGuarantee processingGuarantee;
- private final boolean forceEnableTupleTracking;
+ private final boolean tupleTrackingEnforced;
/**
* Creates a new KafkaSpoutConfig using a Builder.
@@ -108,7 +111,7 @@ public class KafkaSpoutConfig<K, V> implements Serializable {
this.valueDes = builder.valueDes;
this.valueDesClazz = builder.valueDesClazz;
this.processingGuarantee = builder.processingGuarantee;
- this.forceEnableTupleTracking = builder.forceEnableTupleTracking;
+ this.tupleTrackingEnforced = builder.tupleTrackingEnforced;
}
/**
@@ -135,22 +138,29 @@ public class KafkaSpoutConfig<K, V> implements Serializable {
}
/**
- * The processing guarantee supported by the spout. This parameter affects when the spout commits offsets to Kafka, marking them as
- * processed.
+ * This enum controls when the tuple with the {@link ConsumerRecord} for an offset is marked as processed,
+ * i.e. when the offset is committed to Kafka. For AT_LEAST_ONCE and AT_MOST_ONCE the spout controls when
+ * the commit happens. When the guarantee is NONE Kafka controls when the commit happens.
*
* <ul>
- * <li>AT_LEAST_ONCE means that the Kafka spout considers an offset ready for commit once a tuple corresponding to that offset has been
- * acked on the spout. This corresponds to an at-least-once guarantee.</li>
- * <li>ANY_TIMES means that the Kafka spout may commit polled offsets at any time. This means the message may be processed any number of
- * times (including 0), and causes the spout to enable auto offset committing on the underlying consumer.</li>
- * <li>AT_MOST_ONCE means that the spout will commit polled offsets before emitting them to the topology. This guarantees at-most-once
- * processing.</li>
+ * <li>AT_LEAST_ONCE - an offset is ready to commit only after the corresponding tuple has been processed (at-least-once)
+ * and acked. If a tuple fails or times-out it will be re-emitted. A tuple can be processed more than once if for instance
+ * the ack gets lost.</li>
+ * <br/>
+ * <li>AT_MOST_ONCE - every offset will be committed to Kafka right after being polled but before being emitted
+ * to the downstream components of the topology. It guarantees that the offset is processed at-most-once because it
+ * won't retry tuples that fail or timeout after the commit to Kafka has been done.</li>
+ * <br/>
+ * <li>NONE - the polled offsets are committed to Kafka periodically as controlled by the Kafka properties
+ * "enable.auto.commit" and "auto.commit.interval.ms". Because the spout does not control when the commit happens
+ * it cannot give any message processing guarantees, i.e. a message may be processed 0, 1 or more times.
+ * This option requires "enable.auto.commit=true". If "enable.auto.commit=false" an exception will be thrown.</li>
* </ul>
*/
- public static enum ProcessingGuarantee {
+ public enum ProcessingGuarantee {
AT_LEAST_ONCE,
- ANY_TIMES,
- AT_MOST_ONCE
+ AT_MOST_ONCE,
+ NONE,
}
public static class Builder<K, V> {
@@ -171,7 +181,7 @@ public class KafkaSpoutConfig<K, V> implements Serializable {
private long partitionRefreshPeriodMs = DEFAULT_PARTITION_REFRESH_PERIOD_MS;
private boolean emitNullTuples = false;
private ProcessingGuarantee processingGuarantee = DEFAULT_PROCESSING_GUARANTEE;
- private boolean forceEnableTupleTracking = false;
+ private boolean tupleTrackingEnforced = false;
public Builder(String bootstrapServers, String... topics) {
this(bootstrapServers, (SerializableDeserializer) null, (SerializableDeserializer) null, new ManualPartitionSubscription(new RoundRobinManualPartitioner(), new NamedTopicFilter(topics)));
@@ -649,10 +659,10 @@ public class KafkaSpoutConfig<K, V> implements Serializable {
* {@link Config#TOPOLOGY_MAX_SPOUT_PENDING} to have an effect, and enables some spout metrics (e.g. complete-latency) that would
* otherwise be disabled.
*
- * @param forceEnableTupleTracking true if Storm should track emitted tuples, false otherwise
+ * @param tupleTrackingEnforced true if Storm should track emitted tuples, false otherwise
*/
- public Builder<K, V> setForceEnableTupleTracking(boolean forceEnableTupleTracking) {
- this.forceEnableTupleTracking = forceEnableTupleTracking;
+ public Builder<K, V> setTupleTrackingEnforced(boolean tupleTrackingEnforced) {
+ this.tupleTrackingEnforced = tupleTrackingEnforced;
return this;
}
@@ -706,14 +716,15 @@ public class KafkaSpoutConfig<K, V> implements Serializable {
+ " Instead use KafkaSpoutConfig.Builder.setProcessingGuarantee."
+ " This will be treated as an error in the next major release."
+ " For now the spout will be configured to behave like it would have in pre-1.2.0 releases.");
- boolean enableAutoCommit = (boolean)builder.kafkaProps.get(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG);
+
+ final boolean enableAutoCommit = (boolean)builder.kafkaProps.get(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG);
if(enableAutoCommit) {
- builder.processingGuarantee = ProcessingGuarantee.ANY_TIMES;
+ builder.processingGuarantee = ProcessingGuarantee.NONE;
} else {
builder.processingGuarantee = ProcessingGuarantee.AT_LEAST_ONCE;
}
}
- if (builder.processingGuarantee == ProcessingGuarantee.ANY_TIMES) {
+ if (builder.processingGuarantee == ProcessingGuarantee.NONE) {
builder.kafkaProps.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "true");
} else {
builder.kafkaProps.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "false");
@@ -788,8 +799,8 @@ public class KafkaSpoutConfig<K, V> implements Serializable {
return processingGuarantee;
}
- public boolean getForceEnableTupleTracking() {
- return forceEnableTupleTracking;
+ public boolean isTupleTrackingEnforced() {
+ return tupleTrackingEnforced;
}
public String getConsumerGroupId() {
http://git-wip-us.apache.org/repos/asf/storm/blob/ac16fe1e/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutMessagingGuaranteeTest.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutMessagingGuaranteeTest.java b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutMessagingGuaranteeTest.java
index 1f23cc5..8613ce7 100644
--- a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutMessagingGuaranteeTest.java
+++ b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutMessagingGuaranteeTest.java
@@ -111,7 +111,7 @@ public class KafkaSpoutMessagingGuaranteeTest {
public void testAnyTimesModeDisregardsMaxUncommittedOffsets() throws Exception {
//The maxUncommittedOffsets limit should not be enforced, since it is only meaningful in at-least-once mode
KafkaSpoutConfig<String, String> spoutConfig = createKafkaSpoutConfigBuilder(-1)
- .setProcessingGuarantee(KafkaSpoutConfig.ProcessingGuarantee.ANY_TIMES)
+ .setProcessingGuarantee(KafkaSpoutConfig.ProcessingGuarantee.NONE)
.build();
doTestModeDisregardsMaxUncommittedOffsets(spoutConfig);
}
@@ -146,7 +146,7 @@ public class KafkaSpoutMessagingGuaranteeTest {
//When tuple tracking is enabled, the spout must not replay tuples in at-most-once mode
KafkaSpoutConfig<String, String> spoutConfig = createKafkaSpoutConfigBuilder(-1)
.setProcessingGuarantee(KafkaSpoutConfig.ProcessingGuarantee.AT_MOST_ONCE)
- .setForceEnableTupleTracking(true)
+ .setTupleTrackingEnforced(true)
.build();
doTestModeCannotReplayTuples(spoutConfig);
}
@@ -155,8 +155,8 @@ public class KafkaSpoutMessagingGuaranteeTest {
public void testAnyTimesModeCannotReplayTuples() throws Exception {
//When tuple tracking is enabled, the spout must not replay tuples in any-times mode
KafkaSpoutConfig<String, String> spoutConfig = createKafkaSpoutConfigBuilder(-1)
- .setProcessingGuarantee(KafkaSpoutConfig.ProcessingGuarantee.ANY_TIMES)
- .setForceEnableTupleTracking(true)
+ .setProcessingGuarantee(KafkaSpoutConfig.ProcessingGuarantee.NONE)
+ .setTupleTrackingEnforced(true)
.build();
doTestModeCannotReplayTuples(spoutConfig);
}
@@ -189,7 +189,7 @@ public class KafkaSpoutMessagingGuaranteeTest {
//When tuple tracking is enabled, the spout must not commit acked tuples in at-most-once mode because they were committed before being emitted
KafkaSpoutConfig<String, String> spoutConfig = createKafkaSpoutConfigBuilder(-1)
.setProcessingGuarantee(KafkaSpoutConfig.ProcessingGuarantee.AT_MOST_ONCE)
- .setForceEnableTupleTracking(true)
+ .setTupleTrackingEnforced(true)
.build();
doTestModeDoesNotCommitAckedTuples(spoutConfig);
}
@@ -198,8 +198,8 @@ public class KafkaSpoutMessagingGuaranteeTest {
public void testAnyTimesModeDoesNotCommitAckedTuples() throws Exception {
//When tuple tracking is enabled, the spout must not commit acked tuples in any-times mode because committing is managed by the consumer
KafkaSpoutConfig<String, String> spoutConfig = createKafkaSpoutConfigBuilder(-1)
- .setProcessingGuarantee(KafkaSpoutConfig.ProcessingGuarantee.ANY_TIMES)
- .setForceEnableTupleTracking(true)
+ .setProcessingGuarantee(KafkaSpoutConfig.ProcessingGuarantee.NONE)
+ .setTupleTrackingEnforced(true)
.build();
doTestModeDoesNotCommitAckedTuples(spoutConfig);
}
[30/50] [abbrv] storm git commit: STORM-2807: Shut down integration
test topologies immediately after tests,
retry killing each topology for a minute and throw an error if it can't be
done
Posted by pt...@apache.org.
STORM-2807: Shut down integration test topologies immediately after tests, retry killing each topology for a minute and throw an error if it can't be done
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/c4603693
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/c4603693
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/c4603693
Branch: refs/heads/metrics_v2
Commit: c460369305f61bb1bc7aa5f20590f907ffd401dc
Parents: 7f22203
Author: Stig Rohde Døssing <sr...@apache.org>
Authored: Wed Nov 8 21:02:55 2017 +0100
Committer: Stig Rohde Døssing <sr...@apache.org>
Committed: Sat Nov 11 10:45:08 2017 +0100
----------------------------------------------------------------------
integration-test/README.md | 2 +-
.../window/TumblingTimeCorrectness.java | 6 ++---
.../window/TumblingWindowCorrectness.java | 2 +-
.../test/java/org/apache/storm/st/DemoTest.java | 2 +-
.../st/tests/window/SlidingWindowTest.java | 6 ++---
.../st/tests/window/TumblingWindowTest.java | 6 ++---
.../apache/storm/st/wrapper/StormCluster.java | 25 +++++++++++++-------
.../org/apache/storm/st/wrapper/TopoWrap.java | 6 ++---
8 files changed, 32 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/c4603693/integration-test/README.md
----------------------------------------------------------------------
diff --git a/integration-test/README.md b/integration-test/README.md
index 36c70a1..ab78387 100644
--- a/integration-test/README.md
+++ b/integration-test/README.md
@@ -16,7 +16,7 @@ And runs all the tests against it.
Running tests for development & debugging
=========================================
-```vagrant up``` command is steup as a complete auto-pilot.
+```vagrant up``` command is setup as a complete auto-pilot.
Following describes how we can run individual tests against this vagrant cluster or any other cluster.
Configs for running
http://git-wip-us.apache.org/repos/asf/storm/blob/c4603693/integration-test/src/main/java/org/apache/storm/st/topology/window/TumblingTimeCorrectness.java
----------------------------------------------------------------------
diff --git a/integration-test/src/main/java/org/apache/storm/st/topology/window/TumblingTimeCorrectness.java b/integration-test/src/main/java/org/apache/storm/st/topology/window/TumblingTimeCorrectness.java
index a77836f..7cadd18 100644
--- a/integration-test/src/main/java/org/apache/storm/st/topology/window/TumblingTimeCorrectness.java
+++ b/integration-test/src/main/java/org/apache/storm/st/topology/window/TumblingTimeCorrectness.java
@@ -55,9 +55,9 @@ public class TumblingTimeCorrectness implements TestableTopology {
private final String spoutName;
private final String boltName;
- public TumblingTimeCorrectness(int timbleSec) {
- this.tumbleSec = timbleSec;
- final String prefix = this.getClass().getSimpleName() + "-timbleSec" + timbleSec;
+ public TumblingTimeCorrectness(int tumbleSec) {
+ this.tumbleSec = tumbleSec;
+ final String prefix = this.getClass().getSimpleName() + "-tumbleSec" + tumbleSec;
spoutName = prefix + "IncrementingSpout";
boltName = prefix + "VerificationBolt";
}
http://git-wip-us.apache.org/repos/asf/storm/blob/c4603693/integration-test/src/main/java/org/apache/storm/st/topology/window/TumblingWindowCorrectness.java
----------------------------------------------------------------------
diff --git a/integration-test/src/main/java/org/apache/storm/st/topology/window/TumblingWindowCorrectness.java b/integration-test/src/main/java/org/apache/storm/st/topology/window/TumblingWindowCorrectness.java
index 22c6d75..adcb9dd 100644
--- a/integration-test/src/main/java/org/apache/storm/st/topology/window/TumblingWindowCorrectness.java
+++ b/integration-test/src/main/java/org/apache/storm/st/topology/window/TumblingWindowCorrectness.java
@@ -54,7 +54,7 @@ public class TumblingWindowCorrectness implements TestableTopology {
public TumblingWindowCorrectness(final int tumbleSize) {
this.tumbleSize = tumbleSize;
- final String prefix = this.getClass().getSimpleName() + "-tubleSize" + tumbleSize;
+ final String prefix = this.getClass().getSimpleName() + "-tumbleSize" + tumbleSize;
spoutName = prefix + "IncrementingSpout";
boltName = prefix + "VerificationBolt";
}
http://git-wip-us.apache.org/repos/asf/storm/blob/c4603693/integration-test/src/test/java/org/apache/storm/st/DemoTest.java
----------------------------------------------------------------------
diff --git a/integration-test/src/test/java/org/apache/storm/st/DemoTest.java b/integration-test/src/test/java/org/apache/storm/st/DemoTest.java
index b011254..2c62d3f 100644
--- a/integration-test/src/test/java/org/apache/storm/st/DemoTest.java
+++ b/integration-test/src/test/java/org/apache/storm/st/DemoTest.java
@@ -79,7 +79,7 @@ public final class DemoTest extends AbstractTest {
@AfterMethod
public void cleanup() throws Exception {
if (topo != null) {
- topo.killQuietly();
+ topo.killOrThrow();
}
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/c4603693/integration-test/src/test/java/org/apache/storm/st/tests/window/SlidingWindowTest.java
----------------------------------------------------------------------
diff --git a/integration-test/src/test/java/org/apache/storm/st/tests/window/SlidingWindowTest.java b/integration-test/src/test/java/org/apache/storm/st/tests/window/SlidingWindowTest.java
index c4200a0..29afe7a 100644
--- a/integration-test/src/test/java/org/apache/storm/st/tests/window/SlidingWindowTest.java
+++ b/integration-test/src/test/java/org/apache/storm/st/tests/window/SlidingWindowTest.java
@@ -66,7 +66,7 @@ public final class SlidingWindowTest extends AbstractTest {
@Test(dataProvider = "generateCountWindows")
public void testWindowCount(int windowSize, int slideSize) throws Exception {
final SlidingWindowCorrectness testable = new SlidingWindowCorrectness(windowSize, slideSize);
- final String topologyName = this.getClass().getSimpleName() + "w" + windowSize + "s" + slideSize;
+ final String topologyName = this.getClass().getSimpleName() + "-window" + windowSize + "-slide" + slideSize;
if (windowSize <= 0 || slideSize <= 0) {
try {
testable.newTopology();
@@ -130,7 +130,7 @@ public final class SlidingWindowTest extends AbstractTest {
@Test(dataProvider = "generateTimeWindows")
public void testTimeWindow(int windowSec, int slideSec) throws Exception {
final SlidingTimeCorrectness testable = new SlidingTimeCorrectness(windowSec, slideSec);
- final String topologyName = this.getClass().getSimpleName() + "w" + windowSec + "s" + slideSec;
+ final String topologyName = this.getClass().getSimpleName() + "-window" + windowSec + "-slide" + slideSec;
if (windowSec <= 0 || slideSec <= 0) {
try {
testable.newTopology();
@@ -181,7 +181,7 @@ public final class SlidingWindowTest extends AbstractTest {
@AfterMethod
public void cleanup() throws Exception {
if (topo != null) {
- topo.killQuietly();
+ topo.killOrThrow();
}
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/c4603693/integration-test/src/test/java/org/apache/storm/st/tests/window/TumblingWindowTest.java
----------------------------------------------------------------------
diff --git a/integration-test/src/test/java/org/apache/storm/st/tests/window/TumblingWindowTest.java b/integration-test/src/test/java/org/apache/storm/st/tests/window/TumblingWindowTest.java
index 450219d..866e4c8 100644
--- a/integration-test/src/test/java/org/apache/storm/st/tests/window/TumblingWindowTest.java
+++ b/integration-test/src/test/java/org/apache/storm/st/tests/window/TumblingWindowTest.java
@@ -48,7 +48,7 @@ public final class TumblingWindowTest extends AbstractTest {
@Test(dataProvider = "generateWindows")
public void testTumbleCount(int tumbleSize) throws Exception {
final TumblingWindowCorrectness testable = new TumblingWindowCorrectness(tumbleSize);
- final String topologyName = this.getClass().getSimpleName() + "t" + tumbleSize;
+ final String topologyName = this.getClass().getSimpleName() + "-size" + tumbleSize;
if (tumbleSize <= 0) {
try {
testable.newTopology();
@@ -77,7 +77,7 @@ public final class TumblingWindowTest extends AbstractTest {
@Test(dataProvider = "generateTumbleTimes")
public void testTumbleTime(int tumbleSec) throws Exception {
final TumblingTimeCorrectness testable = new TumblingTimeCorrectness(tumbleSec);
- final String topologyName = this.getClass().getSimpleName() + "t" + tumbleSec;
+ final String topologyName = this.getClass().getSimpleName() + "-sec" + tumbleSec;
if (tumbleSec <= 0) {
try {
testable.newTopology();
@@ -93,7 +93,7 @@ public final class TumblingWindowTest extends AbstractTest {
@AfterMethod
public void cleanup() throws Exception {
if (topo != null) {
- topo.killQuietly();
+ topo.killOrThrow();
}
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/c4603693/integration-test/src/test/java/org/apache/storm/st/wrapper/StormCluster.java
----------------------------------------------------------------------
diff --git a/integration-test/src/test/java/org/apache/storm/st/wrapper/StormCluster.java b/integration-test/src/test/java/org/apache/storm/st/wrapper/StormCluster.java
index 7311d5b..51226e1 100644
--- a/integration-test/src/test/java/org/apache/storm/st/wrapper/StormCluster.java
+++ b/integration-test/src/test/java/org/apache/storm/st/wrapper/StormCluster.java
@@ -38,6 +38,7 @@ import java.util.ArrayList;
import java.util.Collection;
import java.util.List;
import java.util.Map;
+import java.util.concurrent.TimeUnit;
public class StormCluster {
private static Logger log = LoggerFactory.getLogger(StormCluster.class);
@@ -83,13 +84,21 @@ public class StormCluster {
return new ArrayList<>(filteredSummary);
}
- public void killSilently(String topologyName) {
- try {
- client.killTopologyWithOpts(topologyName, new KillOptions());
- log.info("Topology killed: " + topologyName);
- } catch (Throwable e){
- log.warn("Couldn't kill topology: " + topologyName + " Exception: " + ExceptionUtils.getFullStackTrace(e));
+ public void killOrThrow(String topologyName) throws Exception {
+ long start = System.currentTimeMillis();
+ while (System.currentTimeMillis() < start + TimeUnit.SECONDS.toMillis(60)) {
+ try {
+ KillOptions killOptions = new KillOptions();
+ killOptions.set_wait_secs(0);
+ client.killTopologyWithOpts(topologyName, killOptions);
+ log.info("Topology killed: " + topologyName);
+ return;
+ } catch (Throwable e) {
+ log.warn("Couldn't kill topology: " + topologyName + ", going to retry soon. Exception: " + ExceptionUtils.getFullStackTrace(e));
+ Thread.sleep(TimeUnit.SECONDS.toMillis(2));
+ }
}
+ throw new RuntimeException("Failed to kill topology " + topologyName + ". Subsequent tests may fail because worker slots are occupied");
}
public TopologySummary getOneActive() throws TException {
@@ -107,10 +116,10 @@ public class StormCluster {
return client;
}
- public void killActiveTopologies() throws TException {
+ public void killActiveTopologies() throws Exception {
List<TopologySummary> activeTopologies = getActive();
for (TopologySummary activeTopology : activeTopologies) {
- killSilently(activeTopology.get_name());
+ killOrThrow(activeTopology.get_name());
}
AssertUtil.empty(getActive());
http://git-wip-us.apache.org/repos/asf/storm/blob/c4603693/integration-test/src/test/java/org/apache/storm/st/wrapper/TopoWrap.java
----------------------------------------------------------------------
diff --git a/integration-test/src/test/java/org/apache/storm/st/wrapper/TopoWrap.java b/integration-test/src/test/java/org/apache/storm/st/wrapper/TopoWrap.java
index 2be0402..5e58637 100644
--- a/integration-test/src/test/java/org/apache/storm/st/wrapper/TopoWrap.java
+++ b/integration-test/src/test/java/org/apache/storm/st/wrapper/TopoWrap.java
@@ -223,7 +223,7 @@ public class TopoWrap {
public void assertProgress(int minEmits, String componentName, int maxWaitSec) throws TException {
waitForProgress(minEmits, componentName, maxWaitSec);
long emitCount = getAllTimeEmittedCount(componentName);
- Assert.assertTrue(emitCount >= minEmits, "Count for component " + componentName + " is " + emitCount + " min is " + minEmits);
+ Assert.assertTrue(emitCount >= minEmits, "Emit count for component '" + componentName + "' is " + emitCount + ", min is " + minEmits);
}
public static class ExecutorURL {
@@ -372,7 +372,7 @@ public class TopoWrap {
return retVal;
}
- public void killQuietly() {
- cluster.killSilently(name);
+ public void killOrThrow() throws Exception {
+ cluster.killOrThrow(name);
}
}
[47/50] [abbrv] storm git commit: Merge branch 'STORM-2546-1.x' of
https://github.com/srdo/storm into asfgit-1.x-branch
Posted by pt...@apache.org.
Merge branch 'STORM-2546-1.x' of https://github.com/srdo/storm into asfgit-1.x-branch
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/818ed16f
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/818ed16f
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/818ed16f
Branch: refs/heads/metrics_v2
Commit: 818ed16f2281a6ee0ff65768a36fc8b8354b3ef5
Parents: 3badc16 d9fe768
Author: Stig Rohde Døssing <sr...@apache.org>
Authored: Thu Nov 16 07:13:20 2017 +0100
Committer: Stig Rohde Døssing <sr...@apache.org>
Committed: Thu Nov 16 07:13:20 2017 +0100
----------------------------------------------------------------------
.../apache/storm/kafka/spout/KafkaSpout.java | 48 +++-
.../storm/kafka/spout/KafkaSpoutConfig.java | 21 ++
.../storm/kafka/spout/KafkaSpoutCommitTest.java | 113 ----------
.../storm/kafka/spout/KafkaSpoutConfigTest.java | 24 ++
.../storm/kafka/spout/KafkaSpoutEmitTest.java | 9 +-
.../KafkaSpoutLogCompactionSupportTest.java | 223 +++++++++++++++++++
.../spout/KafkaSpoutMessagingGuaranteeTest.java | 8 +-
.../kafka/spout/KafkaSpoutRetryLimitTest.java | 3 +-
.../SpoutWithMockedConsumerSetupHelper.java | 66 +++++-
.../kafka/spout/internal/OffsetManagerTest.java | 1 -
10 files changed, 378 insertions(+), 138 deletions(-)
----------------------------------------------------------------------
[07/50] [abbrv] storm git commit: Backport to Java 7 and revert
changes to RetryService API
Posted by pt...@apache.org.
Backport to Java 7 and revert changes to RetryService API
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/54fdcbc0
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/54fdcbc0
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/54fdcbc0
Branch: refs/heads/metrics_v2
Commit: 54fdcbc03be47c95b8e9d177f525e031afb5e946
Parents: 9b95f3d
Author: Stig Rohde Døssing <sr...@apache.org>
Authored: Wed Oct 4 20:46:46 2017 +0200
Committer: Stig Rohde Døssing <sr...@apache.org>
Committed: Wed Oct 4 20:46:46 2017 +0200
----------------------------------------------------------------------
.../apache/storm/kafka/spout/KafkaSpout.java | 10 ++++--
.../KafkaSpoutRetryExponentialBackoff.java | 5 +--
.../kafka/spout/KafkaSpoutRetryService.java | 13 ++++----
.../kafka/spout/KafkaSpoutRebalanceTest.java | 4 +--
.../KafkaSpoutRetryExponentialBackoffTest.java | 33 +++++++++++---------
.../kafka/spout/SingleTopicKafkaSpoutTest.java | 4 +--
6 files changed, 41 insertions(+), 28 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/54fdcbc0/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpout.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpout.java b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpout.java
index 4f9dacb..3582bdb 100644
--- a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpout.java
+++ b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpout.java
@@ -178,7 +178,13 @@ public class KafkaSpout<K, V> extends BaseRichSpout {
* Emitted messages for partitions that are no longer assigned to this spout can't
* be acked and should not be retried, hence remove them from emitted collection.
*/
- emitted.removeIf(msgId -> !partitions.contains(msgId.getTopicPartition()));
+ Iterator<KafkaSpoutMessageId> msgIdIterator = emitted.iterator();
+ while (msgIdIterator.hasNext()) {
+ KafkaSpoutMessageId msgId = msgIdIterator.next();
+ if (!partitions.contains(msgId.getTopicPartition())) {
+ msgIdIterator.remove();
+ }
+ }
}
Set<TopicPartition> newPartitions = new HashSet<>(partitions);
@@ -331,7 +337,7 @@ public class KafkaSpout<K, V> extends BaseRichSpout {
*/
private boolean emitTupleIfNotEmitted(ConsumerRecord<K, V> record) {
final TopicPartition tp = new TopicPartition(record.topic(), record.partition());
- final KafkaSpoutMessageId msgId = retryService.getMessageId(tp, record.offset());
+ final KafkaSpoutMessageId msgId = retryService.getMessageId(record);
if (offsetManagers.containsKey(tp) && offsetManagers.get(tp).contains(msgId)) { // has been acked
LOG.trace("Tuple for record [{}] has already been acked. Skipping", record);
} else if (emitted.contains(msgId)) { // has been emitted and it's pending ack or fail
http://git-wip-us.apache.org/repos/asf/storm/blob/54fdcbc0/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpoutRetryExponentialBackoff.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpoutRetryExponentialBackoff.java b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpoutRetryExponentialBackoff.java
index 81ef986..68a6f3f 100644
--- a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpoutRetryExponentialBackoff.java
+++ b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpoutRetryExponentialBackoff.java
@@ -33,6 +33,7 @@ import java.util.Set;
import java.util.TreeSet;
import java.util.concurrent.TimeUnit;
import org.apache.commons.lang.Validate;
+import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.storm.utils.Time;
/**
@@ -289,8 +290,8 @@ public class KafkaSpoutRetryExponentialBackoff implements KafkaSpoutRetryService
}
@Override
- public KafkaSpoutMessageId getMessageId(TopicPartition tp, long offset) {
- KafkaSpoutMessageId msgId = new KafkaSpoutMessageId(tp, offset);
+ public KafkaSpoutMessageId getMessageId(ConsumerRecord<?, ?> record) {
+ KafkaSpoutMessageId msgId = new KafkaSpoutMessageId(record);
if (isScheduled(msgId)) {
for (KafkaSpoutMessageId originalMsgId : toRetryMsgs) {
if (originalMsgId.equals(msgId)) {
http://git-wip-us.apache.org/repos/asf/storm/blob/54fdcbc0/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpoutRetryService.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpoutRetryService.java b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpoutRetryService.java
index b70acd7..12d26da 100644
--- a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpoutRetryService.java
+++ b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpoutRetryService.java
@@ -18,11 +18,11 @@
package org.apache.storm.kafka.spout;
-import org.apache.kafka.common.TopicPartition;
import java.io.Serializable;
import java.util.Collection;
import java.util.Map;
+import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.common.TopicPartition;
/**
@@ -84,10 +84,11 @@ public interface KafkaSpoutRetryService extends Serializable {
int readyMessageCount();
/**
- * Gets the {@link KafkaSpoutMessageId} for the record on the given topic partition and offset.
- * @param topicPartition The topic partition of the record
- * @param offset The offset of the record
- * @return The id the record was scheduled for retry with, or a new {@link KafkaSpoutMessageId} if the record was not scheduled for retry.
+ * Gets the {@link KafkaSpoutMessageId} for the given record.
+ *
+ * @param record The record to fetch the id for
+ * @return The id the record was scheduled for retry with, or a new {@link KafkaSpoutMessageId} if the record was not scheduled for
+ * retry.
*/
- KafkaSpoutMessageId getMessageId(TopicPartition topicPartition, long offset);
+ KafkaSpoutMessageId getMessageId(ConsumerRecord<?, ?> record);
}
http://git-wip-us.apache.org/repos/asf/storm/blob/54fdcbc0/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutRebalanceTest.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutRebalanceTest.java b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutRebalanceTest.java
index b16ba5d..3e50506 100644
--- a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutRebalanceTest.java
+++ b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutRebalanceTest.java
@@ -213,10 +213,10 @@ public class KafkaSpoutRebalanceTest {
Subscription subscriptionMock = mock(Subscription.class);
doNothing()
.when(subscriptionMock)
- .subscribe(any(), rebalanceListenerCapture.capture(), any());
+ .subscribe(any(KafkaConsumer.class), rebalanceListenerCapture.capture(), any(TopologyContext.class));
KafkaSpout<String, String> spout = new KafkaSpout<>(createKafkaSpoutConfigBuilder(subscriptionMock, -1)
.setFirstPollOffsetStrategy(KafkaSpoutConfig.FirstPollOffsetStrategy.UNCOMMITTED_EARLIEST)
- .build(), consumerFactory);
+ .build(), consumerFactoryMock);
String topic = SingleTopicKafkaSpoutConfiguration.TOPIC;
TopicPartition assignedPartition = new TopicPartition(topic, 1);
TopicPartition newPartition = new TopicPartition(topic, 2);
http://git-wip-us.apache.org/repos/asf/storm/blob/54fdcbc0/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutRetryExponentialBackoffTest.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutRetryExponentialBackoffTest.java b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutRetryExponentialBackoffTest.java
index c543f8b..f6de6a8 100644
--- a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutRetryExponentialBackoffTest.java
+++ b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutRetryExponentialBackoffTest.java
@@ -25,6 +25,7 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.common.TopicPartition;
import org.apache.storm.kafka.spout.KafkaSpoutRetryExponentialBackoff.TimeInterval;
import org.apache.storm.utils.Time;
@@ -44,17 +45,21 @@ public class KafkaSpoutRetryExponentialBackoffTest {
return new KafkaSpoutRetryExponentialBackoff(TimeInterval.seconds(1), TimeInterval.seconds(0), 1, TimeInterval.seconds(1));
}
+ private ConsumerRecord<String, String> createRecord(TopicPartition tp, long offset) {
+ return new ConsumerRecord<>(tp.topic(), tp.partition(), offset, null, null);
+ }
+
@Test
public void testCanScheduleRetry() {
KafkaSpoutRetryExponentialBackoff retryService = createNoWaitRetryService();
long offset = 0;
- KafkaSpoutMessageId msgId = retryService.getMessageId(testTopic, offset);
+ KafkaSpoutMessageId msgId = retryService.getMessageId(createRecord(testTopic, offset));
msgId.incrementNumFails();
boolean scheduled = retryService.schedule(msgId);
assertThat("The service must schedule the message for retry", scheduled, is(true));
- KafkaSpoutMessageId retrievedMessageId = retryService.getMessageId(testTopic, offset);
+ KafkaSpoutMessageId retrievedMessageId = retryService.getMessageId(createRecord(testTopic, offset));
assertThat("The service should return the original message id when asked for the same tp/offset twice", retrievedMessageId, sameInstance(msgId));
assertThat(retryService.isScheduled(msgId), is(true));
assertThat(retryService.isReady(msgId), is(true));
@@ -68,7 +73,7 @@ public class KafkaSpoutRetryExponentialBackoffTest {
KafkaSpoutRetryExponentialBackoff retryService = createOneSecondWaitRetryService();
long offset = 0;
- KafkaSpoutMessageId msgId = retryService.getMessageId(testTopic, offset);
+ KafkaSpoutMessageId msgId = retryService.getMessageId(createRecord(testTopic, offset));
msgId.incrementNumFails();
retryService.schedule(msgId);
@@ -79,7 +84,7 @@ public class KafkaSpoutRetryExponentialBackoffTest {
Time.advanceTime(500);
assertThat("The message should not be ready for retry yet since it was rescheduled", retryService.isReady(msgId), is(false));
assertThat(retryService.isScheduled(msgId), is(true));
- assertThat(retryService.earliestRetriableOffsets(), is(Collections.emptyMap()));
+ assertThat(retryService.earliestRetriableOffsets(), is(Collections.<TopicPartition, Long>emptyMap()));
assertThat(retryService.readyMessageCount(), is(0));
Time.advanceTime(500);
assertThat("The message should be ready for retry once the full delay has passed", retryService.isReady(msgId), is(true));
@@ -95,7 +100,7 @@ public class KafkaSpoutRetryExponentialBackoffTest {
KafkaSpoutRetryExponentialBackoff retryService = createOneSecondWaitRetryService();
long offset = 0;
- KafkaSpoutMessageId msgId = retryService.getMessageId(testTopic, offset);
+ KafkaSpoutMessageId msgId = retryService.getMessageId(createRecord(testTopic, offset));
msgId.incrementNumFails();
retryService.schedule(msgId);
@@ -113,7 +118,7 @@ public class KafkaSpoutRetryExponentialBackoffTest {
public void testCanRemoveRetry() {
KafkaSpoutRetryExponentialBackoff retryService = createNoWaitRetryService();
long offset = 0;
- KafkaSpoutMessageId msgId = retryService.getMessageId(testTopic, offset);
+ KafkaSpoutMessageId msgId = retryService.getMessageId(createRecord(testTopic, offset));
msgId.incrementNumFails();
retryService.schedule(msgId);
@@ -122,7 +127,7 @@ public class KafkaSpoutRetryExponentialBackoffTest {
assertThat(removed, is(true));
assertThat(retryService.isScheduled(msgId), is(false));
assertThat(retryService.isReady(msgId), is(false));
- assertThat(retryService.earliestRetriableOffsets(), is(Collections.emptyMap()));
+ assertThat(retryService.earliestRetriableOffsets(), is(Collections.<TopicPartition, Long>emptyMap()));
assertThat(retryService.readyMessageCount(), is(0));
}
@@ -133,8 +138,8 @@ public class KafkaSpoutRetryExponentialBackoffTest {
KafkaSpoutRetryExponentialBackoff retryService = createOneSecondWaitRetryService();
long offset = 0;
- KafkaSpoutMessageId msgIdTp1 = retryService.getMessageId(testTopic, offset);
- KafkaSpoutMessageId msgIdTp2 = retryService.getMessageId(testTopic2, offset);
+ KafkaSpoutMessageId msgIdTp1 = retryService.getMessageId(createRecord(testTopic, offset));
+ KafkaSpoutMessageId msgIdTp2 = retryService.getMessageId(createRecord(testTopic2, offset));
msgIdTp1.incrementNumFails();
msgIdTp2.incrementNumFails();
@@ -179,8 +184,8 @@ public class KafkaSpoutRetryExponentialBackoffTest {
KafkaSpoutRetryExponentialBackoff retryService = createOneSecondWaitRetryService();
long offset = 0;
- KafkaSpoutMessageId msgIdEarliest = retryService.getMessageId(testTopic, offset);
- KafkaSpoutMessageId msgIdLatest = retryService.getMessageId(testTopic, offset + 1);
+ KafkaSpoutMessageId msgIdEarliest = retryService.getMessageId(createRecord(testTopic, offset));
+ KafkaSpoutMessageId msgIdLatest = retryService.getMessageId(createRecord(testTopic, offset + 1));
msgIdEarliest.incrementNumFails();
msgIdLatest.incrementNumFails();
@@ -213,7 +218,7 @@ public class KafkaSpoutRetryExponentialBackoffTest {
KafkaSpoutRetryExponentialBackoff retryService = new KafkaSpoutRetryExponentialBackoff(TimeInterval.seconds(0), TimeInterval.seconds(0), maxRetries, TimeInterval.seconds(0));
long offset = 0;
- KafkaSpoutMessageId msgId = retryService.getMessageId(testTopic, offset);
+ KafkaSpoutMessageId msgId = retryService.getMessageId(createRecord(testTopic, offset));
for (int i = 0; i < maxRetries; i++) {
msgId.incrementNumFails();
}
@@ -240,7 +245,7 @@ public class KafkaSpoutRetryExponentialBackoffTest {
KafkaSpoutRetryExponentialBackoff retryService = new KafkaSpoutRetryExponentialBackoff(TimeInterval.seconds(500), TimeInterval.seconds(0), 1, TimeInterval.seconds(maxDelaySecs));
long offset = 0;
- KafkaSpoutMessageId msgId = retryService.getMessageId(testTopic, offset);
+ KafkaSpoutMessageId msgId = retryService.getMessageId(createRecord(testTopic, offset));
msgId.incrementNumFails();
retryService.schedule(msgId);
@@ -263,7 +268,7 @@ public class KafkaSpoutRetryExponentialBackoffTest {
KafkaSpoutRetryExponentialBackoff retryService = new KafkaSpoutRetryExponentialBackoff(TimeInterval.seconds(0), TimeInterval.seconds(4), Integer.MAX_VALUE, TimeInterval.seconds(Integer.MAX_VALUE));
long offset = 0;
- KafkaSpoutMessageId msgId = retryService.getMessageId(testTopic, offset);
+ KafkaSpoutMessageId msgId = retryService.getMessageId(createRecord(testTopic, offset));
msgId.incrementNumFails();
msgId.incrementNumFails(); //First failure is the initial delay, so not interesting
http://git-wip-us.apache.org/repos/asf/storm/blob/54fdcbc0/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/SingleTopicKafkaSpoutTest.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/SingleTopicKafkaSpoutTest.java b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/SingleTopicKafkaSpoutTest.java
index 8d7da7f..75fa6b8 100644
--- a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/SingleTopicKafkaSpoutTest.java
+++ b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/SingleTopicKafkaSpoutTest.java
@@ -132,7 +132,7 @@ public class SingleTopicKafkaSpoutTest {
spout.nextTuple();
}
ArgumentCaptor<KafkaSpoutMessageId> messageIdCaptor = ArgumentCaptor.forClass(KafkaSpoutMessageId.class);
- verify(collector, times(messageCount)).emit(anyObject(), anyObject(), messageIdCaptor.capture());
+ verify(collector, times(messageCount)).emit(anyString(), anyList(), messageIdCaptor.capture());
List<KafkaSpoutMessageId> messageIds = messageIdCaptor.getAllValues();
for (int i = 1; i < messageIds.size(); i++) {
spout.ack(messageIds.get(i));
@@ -144,7 +144,7 @@ public class SingleTopicKafkaSpoutTest {
reset(collector);
spout.nextTuple();
ArgumentCaptor<KafkaSpoutMessageId> failedIdReplayCaptor = ArgumentCaptor.forClass(KafkaSpoutMessageId.class);
- verify(collector).emit(anyObject(), anyObject(), failedIdReplayCaptor.capture());
+ verify(collector).emit(anyString(), anyList(), failedIdReplayCaptor.capture());
assertThat("Expected replay of failed tuple", failedIdReplayCaptor.getValue(), is(failedTuple));
[48/50] [abbrv] storm git commit: [STORM-2815] UI HTTP server should
return 403 if the user is unauthorized (1.x)
Posted by pt...@apache.org.
[STORM-2815] UI HTTP server should return 403 if the user is unauthorized (1.x)
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/ad13f457
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/ad13f457
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/ad13f457
Branch: refs/heads/metrics_v2
Commit: ad13f457bf465f14e34133c013da1b18088036dd
Parents: 818ed16
Author: Ethan Li <et...@gmail.com>
Authored: Thu Nov 16 08:29:54 2017 -0600
Committer: Ethan Li <et...@gmail.com>
Committed: Thu Nov 16 08:35:34 2017 -0600
----------------------------------------------------------------------
.../src/clj/org/apache/storm/daemon/logviewer.clj | 10 +++++-----
storm-core/src/clj/org/apache/storm/ui/core.clj | 3 ++-
storm-core/src/clj/org/apache/storm/ui/helpers.clj | 15 ++++++++-------
3 files changed, 15 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/ad13f457/storm-core/src/clj/org/apache/storm/daemon/logviewer.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/org/apache/storm/daemon/logviewer.clj b/storm-core/src/clj/org/apache/storm/daemon/logviewer.clj
index a59cbb9..17bea01 100644
--- a/storm-core/src/clj/org/apache/storm/daemon/logviewer.clj
+++ b/storm-core/src/clj/org/apache/storm/daemon/logviewer.clj
@@ -848,7 +848,7 @@
(str "Search substring must be between 1 and 1024 UTF-8 "
"bytes in size (inclusive)"))))
(catch Exception ex
- (json-response (exception->json ex) callback :status 500))))
+ (json-response (exception->json ex 500) callback :status 500))))
(json-response (unauthorized-user-json user) callback :status 403))
(json-response {"error" "Not Found"
"errorMessage" "The file was not found on this node."}
@@ -1121,7 +1121,7 @@
(.getHeader servlet-request "Origin")))
(catch InvalidRequestException ex
(log-error ex)
- (json-response (exception->json ex) (:callback m) :status 400))))
+ (json-response (exception->json ex 400) (:callback m) :status 400))))
(GET "/deepSearch/:topo-id" [:as {:keys [servlet-request servlet-response log-root]} topo-id & m]
;; We do not use servlet-response here, but do not remove it from the
;; :keys list, or this rule could stop working when an authentication
@@ -1141,7 +1141,7 @@
(.getHeader servlet-request "Origin")))
(catch InvalidRequestException ex
(log-error ex)
- (json-response (exception->json ex) (:callback m) :status 400))))
+ (json-response (exception->json ex 400) (:callback m) :status 400))))
(GET "/searchLogs" [:as req & m]
(try
(let [servlet-request (:servlet-request req)
@@ -1154,7 +1154,7 @@
(.getHeader servlet-request "Origin")))
(catch InvalidRequestException ex
(log-error ex)
- (json-response (exception->json ex) (:callback m) :status 400))))
+ (json-response (exception->json ex 400) (:callback m) :status 400))))
(GET "/listLogs" [:as req & m]
(try
(mark! logviewer:num-list-logs-http-requests)
@@ -1168,7 +1168,7 @@
(.getHeader servlet-request "Origin")))
(catch InvalidRequestException ex
(log-error ex)
- (json-response (exception->json ex) (:callback m) :status 400))))
+ (json-response (exception->json ex 400) (:callback m) :status 400))))
(route/resources "/")
(route/not-found "Page not found"))
http://git-wip-us.apache.org/repos/asf/storm/blob/ad13f457/storm-core/src/clj/org/apache/storm/ui/core.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/org/apache/storm/ui/core.clj b/storm-core/src/clj/org/apache/storm/ui/core.clj
index 4365f27..2c73520 100644
--- a/storm-core/src/clj/org/apache/storm/ui/core.clj
+++ b/storm-core/src/clj/org/apache/storm/ui/core.clj
@@ -1427,7 +1427,8 @@
(try
(handler request)
(catch Exception ex
- (json-response (exception->json ex) ((:query-params request) "callback") :status 500)))))
+ (let [status-code (if (instance? AuthorizationException ex) 403 500)]
+ (json-response (exception->json ex status-code) ((:query-params request) "callback") :status status-code))))))
(def app
(handler/site (-> main-routes
http://git-wip-us.apache.org/repos/asf/storm/blob/ad13f457/storm-core/src/clj/org/apache/storm/ui/helpers.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/org/apache/storm/ui/helpers.clj b/storm-core/src/clj/org/apache/storm/ui/helpers.clj
index 78c78be..1a98287 100644
--- a/storm-core/src/clj/org/apache/storm/ui/helpers.clj
+++ b/storm-core/src/clj/org/apache/storm/ui/helpers.clj
@@ -29,9 +29,10 @@
[org.eclipse.jetty.server.nio SelectChannelConnector]
[org.eclipse.jetty.server.ssl SslSocketConnector]
[org.eclipse.jetty.servlet ServletHolder FilterMapping]
- [org.eclipse.jetty.util.ssl SslContextFactory]
+ [org.eclipse.jetty.util.ssl SslContextFactory]
[org.eclipse.jetty.server DispatcherType]
- [org.eclipse.jetty.servlets CrossOriginFilter])
+ [org.eclipse.jetty.servlets CrossOriginFilter]
+ (org.eclipse.jetty.http HttpStatus))
(:require [ring.util servlet]
[ring.util.response :as response])
(:require [compojure.route :as route]
@@ -269,9 +270,9 @@
(serialize-fn data))})
(defn exception->json
- [ex]
- {"error" "Internal Server Error"
+ [ex status-code]
+ {"error" (str status-code " " (HttpStatus/getMessage status-code))
"errorMessage"
- (let [sw (java.io.StringWriter.)]
- (.printStackTrace ex (java.io.PrintWriter. sw))
- (.toString sw))})
+ (let [sw (java.io.StringWriter.)]
+ (.printStackTrace ex (java.io.PrintWriter. sw))
+ (.toString sw))})
[29/50] [abbrv] storm git commit: STORM-2549: Fix broken enforcement
mechanism for maxUncommittedOffsets in storm-kafka-client spout
Posted by pt...@apache.org.
STORM-2549: Fix broken enforcement mechanism for maxUncommittedOffsets in storm-kafka-client spout
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/07ecff12
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/07ecff12
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/07ecff12
Branch: refs/heads/metrics_v2
Commit: 07ecff1202bff8661fe3fd4b27a22649d33113b4
Parents: 7f22203
Author: Stig Rohde Døssing <st...@gmail.com>
Authored: Sat Jun 10 19:38:22 2017 +0200
Committer: Stig Rohde Døssing <sr...@apache.org>
Committed: Sat Nov 11 08:47:19 2017 +0100
----------------------------------------------------------------------
.../apache/storm/kafka/spout/KafkaSpout.java | 111 +++++++++------
.../storm/kafka/spout/KafkaSpoutConfig.java | 11 +-
.../kafka/spout/internal/OffsetManager.java | 22 +++
.../storm/kafka/spout/KafkaSpoutCommitTest.java | 12 +-
.../storm/kafka/spout/KafkaSpoutEmitTest.java | 138 +++++++------------
.../kafka/spout/KafkaSpoutRebalanceTest.java | 15 +-
.../kafka/spout/KafkaSpoutRetryLimitTest.java | 11 +-
.../kafka/spout/MaxUncommittedOffsetTest.java | 32 +++--
.../kafka/spout/SingleTopicKafkaSpoutTest.java | 52 ++++---
.../SpoutWithMockedConsumerSetupHelper.java | 52 ++++---
.../SingleTopicKafkaSpoutConfiguration.java | 2 +-
.../kafka/spout/internal/OffsetManagerTest.java | 15 ++
12 files changed, 266 insertions(+), 207 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/07ecff12/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpout.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpout.java b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpout.java
index 7decd0f..51d81c7 100644
--- a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpout.java
+++ b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpout.java
@@ -91,9 +91,7 @@ public class KafkaSpout<K, V> extends BaseRichSpout {
// Always empty if processing guarantee is none or at-most-once
private transient Set<KafkaSpoutMessageId> emitted;
// Records that have been polled and are queued to be emitted in the nextTuple() call. One record is emitted per nextTuple()
- private transient Iterator<ConsumerRecord<K, V>> waitingToEmit;
- // Number of offsets that have been polled and emitted but not yet been committed. Not used if auto commit mode is enabled.
- private transient long numUncommittedOffsets;
+ private transient Iterator<ConsumerRecord<K, V>> waitingToEmit;
// Triggers when a subscription should be refreshed
private transient Timer refreshSubscriptionTimer;
private transient TopologyContext context;
@@ -115,7 +113,6 @@ public class KafkaSpout<K, V> extends BaseRichSpout {
// Spout internals
this.collector = collector;
- numUncommittedOffsets = 0;
// Offset management
firstPollOffsetStrategy = kafkaSpoutConfig.getFirstPollOffsetStrategy();
@@ -233,13 +230,19 @@ public class KafkaSpout<K, V> extends BaseRichSpout {
public void nextTuple() {
try {
if (initialized) {
+
+ if (refreshSubscriptionTimer.isExpiredResetOnTrue()) {
+ kafkaSpoutConfig.getSubscription().refreshAssignment();
+ }
+
if (commit()) {
commitOffsetsForAckedTuples();
}
- if (poll()) {
+ Set<TopicPartition> pollablePartitions = poll();
+ if (!pollablePartitions.isEmpty()) {
try {
- setWaitingToEmit(pollKafkaBroker());
+ setWaitingToEmit(pollKafkaBroker(pollablePartitions));
} catch (RetriableException e) {
LOG.error("Failed to poll from kafka.", e);
}
@@ -266,27 +269,38 @@ public class KafkaSpout<K, V> extends BaseRichSpout {
return isAtLeastOnceProcessing() && commitTimer.isExpiredResetOnTrue(); // timer != null for non auto commit mode
}
- private boolean poll() {
+ private Set<TopicPartition> poll() {
final int maxUncommittedOffsets = kafkaSpoutConfig.getMaxUncommittedOffsets();
- final int readyMessageCount = retryService.readyMessageCount();
- final boolean poll = !waitingToEmit()
- // Check that the number of uncommitted, non-retriable tuples is less than the maxUncommittedOffsets limit.
- // Accounting for retriable tuples in this way still guarantees that the limit is followed on a per partition basis,
- // and prevents locking up the spout when there are too many retriable tuples
- && (numUncommittedOffsets - readyMessageCount < maxUncommittedOffsets || !isAtLeastOnceProcessing());
-
- if (!poll) {
- if (waitingToEmit()) {
- LOG.debug("Not polling. Tuples waiting to be emitted."
- + " [{}] uncommitted offsets across all topic partitions", numUncommittedOffsets);
- }
-
- if (numUncommittedOffsets >= maxUncommittedOffsets && isAtLeastOnceProcessing()) {
- LOG.debug("Not polling. [{}] uncommitted offsets across all topic partitions has reached the threshold of [{}]",
- numUncommittedOffsets, maxUncommittedOffsets);
+
+ if (waitingToEmit()) {
+ LOG.debug("Not polling. Tuples waiting to be emitted.");
+ return Collections.emptySet();
+ }
+ Set<TopicPartition> assignment = kafkaConsumer.assignment();
+ if (!isAtLeastOnceProcessing()) {
+ return assignment;
+ }
+ Map<TopicPartition, Long> earliestRetriableOffsets = retryService.earliestRetriableOffsets();
+ Set<TopicPartition> pollablePartitions = new HashSet<>();
+ for (TopicPartition tp : assignment) {
+ OffsetManager offsetManager = offsetManagers.get(tp);
+ int numUncommittedOffsets = offsetManager.getNumUncommittedOffsets();
+ if (numUncommittedOffsets < maxUncommittedOffsets) {
+ //Allow poll if the partition is not at the maxUncommittedOffsets limit
+ pollablePartitions.add(tp);
+ } else {
+ long offsetAtLimit = offsetManager.getNthUncommittedOffsetAfterCommittedOffset(maxUncommittedOffsets);
+ Long earliestRetriableOffset = earliestRetriableOffsets.get(tp);
+ if (earliestRetriableOffset != null && earliestRetriableOffset <= offsetAtLimit) {
+ //Allow poll if there are retriable tuples within the maxUncommittedOffsets limit
+ pollablePartitions.add(tp);
+ } else {
+ LOG.debug("Not polling on partition [{}]. It has [{}] uncommitted offsets, which exceeds the limit of [{}]. ", tp,
+ numUncommittedOffsets, maxUncommittedOffsets);
+ }
}
}
- return poll;
+ return pollablePartitions;
}
private boolean waitingToEmit() {
@@ -302,28 +316,39 @@ public class KafkaSpout<K, V> extends BaseRichSpout {
}
// ======== poll =========
- private ConsumerRecords<K, V> pollKafkaBroker() {
- doSeekRetriableTopicPartitions();
- if (refreshSubscriptionTimer.isExpiredResetOnTrue()) {
- kafkaSpoutConfig.getSubscription().refreshAssignment();
+ private ConsumerRecords<K, V> pollKafkaBroker(Set<TopicPartition> pollablePartitions) {
+ doSeekRetriableTopicPartitions(pollablePartitions);
+ Set<TopicPartition> pausedPartitions = new HashSet<>(kafkaConsumer.assignment());
+ Iterator<TopicPartition> pausedIter = pausedPartitions.iterator();
+ while (pausedIter.hasNext()) {
+ if (pollablePartitions.contains(pausedIter.next())) {
+ pausedIter.remove();
+ }
}
- final ConsumerRecords<K, V> consumerRecords = kafkaConsumer.poll(kafkaSpoutConfig.getPollTimeoutMs());
- final int numPolledRecords = consumerRecords.count();
- LOG.debug("Polled [{}] records from Kafka. [{}] uncommitted offsets across all topic partitions",
- numPolledRecords, numUncommittedOffsets);
- if (kafkaSpoutConfig.getProcessingGuarantee() == KafkaSpoutConfig.ProcessingGuarantee.AT_MOST_ONCE) {
- //Commit polled records immediately to ensure delivery is at-most-once.
- kafkaConsumer.commitSync();
+ try {
+ kafkaConsumer.pause(pausedPartitions);
+ final ConsumerRecords<K, V> consumerRecords = kafkaConsumer.poll(kafkaSpoutConfig.getPollTimeoutMs());
+ final int numPolledRecords = consumerRecords.count();
+ LOG.debug("Polled [{}] records from Kafka.",
+ numPolledRecords);
+ if (kafkaSpoutConfig.getProcessingGuarantee() == KafkaSpoutConfig.ProcessingGuarantee.AT_MOST_ONCE) {
+ //Commit polled records immediately to ensure delivery is at-most-once.
+ kafkaConsumer.commitSync();
+ }
+ return consumerRecords;
+ } finally {
+ kafkaConsumer.resume(pausedPartitions);
}
- return consumerRecords;
}
- private void doSeekRetriableTopicPartitions() {
+ private void doSeekRetriableTopicPartitions(Set<TopicPartition> pollablePartitions) {
final Map<TopicPartition, Long> retriableTopicPartitions = retryService.earliestRetriableOffsets();
for (Entry<TopicPartition, Long> retriableTopicPartitionAndOffset : retriableTopicPartitions.entrySet()) {
- //Seek directly to the earliest retriable message for each retriable topic partition
- kafkaConsumer.seek(retriableTopicPartitionAndOffset.getKey(), retriableTopicPartitionAndOffset.getValue());
+ if (pollablePartitions.contains(retriableTopicPartitionAndOffset.getKey())) {
+ //Seek directly to the earliest retriable message for each retriable topic partition
+ kafkaConsumer.seek(retriableTopicPartitionAndOffset.getKey(), retriableTopicPartitionAndOffset.getValue());
+ }
}
}
@@ -373,8 +398,6 @@ public class KafkaSpout<K, V> extends BaseRichSpout {
offsetManagers.get(tp).addToEmitMsgs(msgId.offset());
if (isScheduled) { // Was scheduled for retry and re-emitted, so remove from schedule.
retryService.remove(msgId);
- } else { //New tuple, hence increment the uncommitted offset counter
- numUncommittedOffsets++;
}
collector.emit(stream, tuple, msgId);
tupleListener.onEmit(tuple, msgId);
@@ -435,10 +458,8 @@ public class KafkaSpout<K, V> extends BaseRichSpout {
final OffsetManager offsetManager = offsetManagers.get(tp);
- long numCommittedOffsets = offsetManager.commit(tpOffset.getValue());
- numUncommittedOffsets -= numCommittedOffsets;
- LOG.debug("[{}] uncommitted offsets across all topic partitions",
- numUncommittedOffsets);
+ offsetManager.commit(tpOffset.getValue());
+ LOG.debug("[{}] uncommitted offsets for partition [{}] after commit", offsetManager.getNumUncommittedOffsets(), tp);
}
} else {
LOG.trace("No offsets to commit. {}", this);
http://git-wip-us.apache.org/repos/asf/storm/blob/07ecff12/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpoutConfig.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpoutConfig.java b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpoutConfig.java
index 758979c..8ff21b6 100644
--- a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpoutConfig.java
+++ b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpoutConfig.java
@@ -533,11 +533,12 @@ public class KafkaSpoutConfig<K, V> implements Serializable {
}
/**
- * Defines the max number of polled offsets (records) that can be pending commit, before another poll can take place. Once this
- * limit is reached, no more offsets (records) can be polled until the next successful commit(s) sets the number of pending offsets
- * below the threshold. The default is {@link #DEFAULT_MAX_UNCOMMITTED_OFFSETS}. Note that this limit can in some cases be exceeded,
- * but no partition will exceed this limit by more than maxPollRecords - 1.
- *
+ * Defines the max number of polled offsets (records) that can be pending commit, before another poll can take place.
+ * Once this limit is reached, no more offsets (records) can be polled until the next successful commit(s) sets the number
+ * of pending offsets below the threshold. The default is {@link #DEFAULT_MAX_UNCOMMITTED_OFFSETS}.
+ * This limit is per partition and may in some cases be exceeded,
+ * but each partition cannot exceed this limit by more than maxPollRecords - 1.
+ *
* <p>This setting only has an effect if the configured {@link ProcessingGuarantee} is {@link ProcessingGuarantee#AT_LEAST_ONCE}.
*
* @param maxUncommittedOffsets max number of records that can be be pending commit
http://git-wip-us.apache.org/repos/asf/storm/blob/07ecff12/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/internal/OffsetManager.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/internal/OffsetManager.java b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/internal/OffsetManager.java
index 7dfe7f6..e2fa70a 100755
--- a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/internal/OffsetManager.java
+++ b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/internal/OffsetManager.java
@@ -18,6 +18,7 @@ package org.apache.storm.kafka.spout.internal;
import java.util.Comparator;
import java.util.Iterator;
import java.util.NavigableSet;
+import java.util.NoSuchElementException;
import java.util.TreeSet;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.OffsetAndMetadata;
@@ -58,6 +59,27 @@ public class OffsetManager {
public void addToEmitMsgs(long offset) {
this.emittedOffsets.add(offset); // O(Log N)
}
+
+ public int getNumUncommittedOffsets() {
+ return this.emittedOffsets.size();
+ }
+
+ /**
+ * Gets the offset of the nth emitted message after the committed offset.
+ * Example: If the committed offset is 0 and offsets 1, 2, 8, 10 have been emitted,
+ * getNthUncommittedOffsetAfterCommittedOffset(3) returns 8.
+ *
+ * @param index The index of the message to get the offset for
+ * @return The offset
+ * @throws NoSuchElementException if the index is out of range
+ */
+ public long getNthUncommittedOffsetAfterCommittedOffset(int index) {
+ Iterator<Long> offsetIter = emittedOffsets.iterator();
+ for (int i = 0; i < index - 1; i++) {
+ offsetIter.next();
+ }
+ return offsetIter.next();
+ }
/**
* An offset can only be committed when all emitted records with lower offset have been
http://git-wip-us.apache.org/repos/asf/storm/blob/07ecff12/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutCommitTest.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutCommitTest.java b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutCommitTest.java
index eb498d4..95e3b60 100644
--- a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutCommitTest.java
+++ b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutCommitTest.java
@@ -25,6 +25,7 @@ import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import org.apache.kafka.clients.consumer.*;
import org.apache.kafka.common.TopicPartition;
@@ -68,16 +69,13 @@ public class KafkaSpoutCommitTest {
public void testCommitSuccessWithOffsetVoids() {
//Verify that the commit logic can handle offset voids
try (SimulatedTime simulatedTime = new SimulatedTime()) {
- KafkaSpout<String, String> spout = SpoutWithMockedConsumerSetupHelper.setupSpout(spoutConfig, conf, contextMock, collectorMock, consumerMock, Collections.singleton(partition));
+ Set<TopicPartition> assignedPartitions = Collections.singleton(partition);
+ KafkaSpout<String, String> spout = SpoutWithMockedConsumerSetupHelper.setupSpout(spoutConfig, conf, contextMock, collectorMock, consumerMock, assignedPartitions);
Map<TopicPartition, List<ConsumerRecord<String, String>>> records = new HashMap<>();
List<ConsumerRecord<String, String>> recordsForPartition = new ArrayList<>();
// Offsets emitted are 0,1,2,3,4,<void>,8,9
- for (int i = 0; i < 5; i++) {
- recordsForPartition.add(new ConsumerRecord<>(partition.topic(), partition.partition(), i, "key", "value"));
- }
- for (int i = 8; i < 10; i++) {
- recordsForPartition.add(new ConsumerRecord<>(partition.topic(), partition.partition(), i, "key", "value"));
- }
+ recordsForPartition.addAll(SpoutWithMockedConsumerSetupHelper.<String, String>createRecords(partition, 0, 5));
+ recordsForPartition.addAll(SpoutWithMockedConsumerSetupHelper.<String, String>createRecords(partition, 8, 2));
records.put(partition, recordsForPartition);
when(consumerMock.poll(anyLong()))
http://git-wip-us.apache.org/repos/asf/storm/blob/07ecff12/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutEmitTest.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutEmitTest.java b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutEmitTest.java
index 3b1ce2d..5d4252e 100755
--- a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutEmitTest.java
+++ b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutEmitTest.java
@@ -26,8 +26,10 @@ import static org.mockito.Mockito.when;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Set;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;
@@ -74,11 +76,7 @@ public class KafkaSpoutEmitTest {
//This is necessary for Storm to be able to throttle the spout according to maxSpoutPending
KafkaSpout<String, String> spout = SpoutWithMockedConsumerSetupHelper.setupSpout(spoutConfig, conf, contextMock, collectorMock, consumerMock, Collections.singleton(partition));
Map<TopicPartition, List<ConsumerRecord<String, String>>> records = new HashMap<>();
- List<ConsumerRecord<String, String>> recordsForPartition = new ArrayList<>();
- for (int i = 0; i < 10; i++) {
- recordsForPartition.add(new ConsumerRecord<>(partition.topic(), partition.partition(), i, "key", "value"));
- }
- records.put(partition, recordsForPartition);
+ records.put(partition, SpoutWithMockedConsumerSetupHelper.<String, String>createRecords(partition, 0, 10));
when(consumerMock.poll(anyLong()))
.thenReturn(new ConsumerRecords<>(records));
@@ -91,27 +89,24 @@ public class KafkaSpoutEmitTest {
@Test
public void testNextTupleEmitsFailedMessagesEvenWhenMaxUncommittedOffsetsIsExceeded() {
//The spout must reemit failed messages waiting for retry even if it is not allowed to poll for new messages due to maxUncommittedOffsets being exceeded
-
+
//Emit maxUncommittedOffsets messages, and fail all of them. Then ensure that the spout will retry them when the retry backoff has passed
try (SimulatedTime simulatedTime = new SimulatedTime()) {
KafkaSpout<String, String> spout = SpoutWithMockedConsumerSetupHelper.setupSpout(spoutConfig, conf, contextMock, collectorMock, consumerMock, Collections.singleton(partition));
Map<TopicPartition, List<ConsumerRecord<String, String>>> records = new HashMap<>();
- List<ConsumerRecord<String, String>> recordsForPartition = new ArrayList<>();
- for (int i = 0; i < spoutConfig.getMaxUncommittedOffsets(); i++) {
- //This is cheating a bit since maxPollRecords would normally spread this across multiple polls
- recordsForPartition.add(new ConsumerRecord<>(partition.topic(), partition.partition(), i, "key", "value"));
- }
- records.put(partition, recordsForPartition);
+ int numRecords = spoutConfig.getMaxUncommittedOffsets();
+ //This is cheating a bit since maxPollRecords would normally spread this across multiple polls
+ records.put(partition, SpoutWithMockedConsumerSetupHelper.<String, String>createRecords(partition, 0, numRecords));
when(consumerMock.poll(anyLong()))
.thenReturn(new ConsumerRecords<>(records));
- for (int i = 0; i < recordsForPartition.size(); i++) {
+ for (int i = 0; i < numRecords; i++) {
spout.nextTuple();
}
ArgumentCaptor<KafkaSpoutMessageId> messageIds = ArgumentCaptor.forClass(KafkaSpoutMessageId.class);
- verify(collectorMock, times(recordsForPartition.size())).emit(anyString(), anyList(), messageIds.capture());
+ verify(collectorMock, times(numRecords)).emit(anyString(), anyList(), messageIds.capture());
for (KafkaSpoutMessageId messageId : messageIds.getAllValues()) {
spout.fail(messageId);
@@ -121,16 +116,16 @@ public class KafkaSpoutEmitTest {
Time.advanceTime(50);
//No backoff for test retry service, just check that messages will retry immediately
- for (int i = 0; i < recordsForPartition.size(); i++) {
+ for (int i = 0; i < numRecords; i++) {
spout.nextTuple();
}
ArgumentCaptor<KafkaSpoutMessageId> retryMessageIds = ArgumentCaptor.forClass(KafkaSpoutMessageId.class);
- verify(collectorMock, times(recordsForPartition.size())).emit(anyString(), anyList(), retryMessageIds.capture());
+ verify(collectorMock, times(numRecords)).emit(anyString(), anyList(), retryMessageIds.capture());
//Verify that the poll started at the earliest retriable tuple offset
List<Long> failedOffsets = new ArrayList<>();
- for(KafkaSpoutMessageId msgId : messageIds.getAllValues()) {
+ for (KafkaSpoutMessageId msgId : messageIds.getAllValues()) {
failedOffsets.add(msgId.offset());
}
InOrder inOrder = inOrder(consumerMock);
@@ -138,93 +133,64 @@ public class KafkaSpoutEmitTest {
inOrder.verify(consumerMock).poll(anyLong());
}
}
-
+
@Test
- public void testNextTupleEmitsAtMostMaxUncommittedOffsetsPlusMaxPollRecordsWhenRetryingTuples() {
- /*
- The spout must reemit failed messages waiting for retry even if it is not allowed to poll for new messages due to maxUncommittedOffsets being exceeded.
- numUncommittedOffsets is equal to numNonRetriableEmittedTuples + numRetriableTuples.
- The spout will only emit if numUncommittedOffsets - numRetriableTuples < maxUncommittedOffsets (i.e. numNonRetriableEmittedTuples < maxUncommittedOffsets)
- This means that the latest offset a poll can start at for a retriable partition,
- counting from the last committed offset, is maxUncommittedOffsets,
- where there are maxUncommittedOffsets - 1 uncommitted tuples "to the left".
- If the retry poll starts at that offset, it at most emits the retried tuple plus maxPollRecords - 1 new tuples.
- The limit on uncommitted offsets for one partition is therefore maxUncommittedOffsets + maxPollRecords - 1.
-
- It is only necessary to test this for a single partition, because partitions can't contribute negatively to numNonRetriableEmittedTuples,
- so if the limit holds for one partition, it will also hold for each individual partition when multiple are involved.
-
- This makes the actual limit numPartitions * (maxUncommittedOffsets + maxPollRecords - 1)
- */
-
- //Emit maxUncommittedOffsets messages, and fail only the last. Then ensure that the spout will allow no more than maxUncommittedOffsets + maxPollRecords - 1 uncommitted offsets when retrying
+ public void testSpoutWillSkipPartitionsAtTheMaxUncommittedOffsetsLimit() {
+ //This verifies that partitions can't prevent each other from retrying tuples due to the maxUncommittedOffsets limit.
try (SimulatedTime simulatedTime = new SimulatedTime()) {
- KafkaSpout<String, String> spout = SpoutWithMockedConsumerSetupHelper.setupSpout(spoutConfig, conf, contextMock, collectorMock, consumerMock, Collections.singleton(partition));
-
- Map<TopicPartition, List<ConsumerRecord<String, String>>> firstPollRecords = new HashMap<>();
- List<ConsumerRecord<String, String>> firstPollRecordsForPartition = new ArrayList<>();
- for (int i = 0; i < spoutConfig.getMaxUncommittedOffsets(); i++) {
- //This is cheating a bit since maxPollRecords would normally spread this across multiple polls
- firstPollRecordsForPartition.add(new ConsumerRecord<>(partition.topic(), partition.partition(), i, "key", "value"));
- }
- firstPollRecords.put(partition, firstPollRecordsForPartition);
-
- int maxPollRecords = 5;
- Map<TopicPartition, List<ConsumerRecord<String, String>>> secondPollRecords = new HashMap<>();
- List<ConsumerRecord<String, String>> secondPollRecordsForPartition = new ArrayList<>();
- for(int i = 0; i < maxPollRecords; i++) {
- secondPollRecordsForPartition.add(new ConsumerRecord<>(partition.topic(), partition.partition(), spoutConfig.getMaxUncommittedOffsets() + i, "key", "value"));
- }
- secondPollRecords.put(partition, secondPollRecordsForPartition);
+ TopicPartition partitionTwo = new TopicPartition(SingleTopicKafkaSpoutConfiguration.TOPIC, 2);
+ Set<TopicPartition> partitions = new HashSet<>();
+ partitions.add(partition);
+ partitions.add(partitionTwo);
+ KafkaSpout<String, String> spout = SpoutWithMockedConsumerSetupHelper.setupSpout(spoutConfig, conf, contextMock, collectorMock, consumerMock, partitions);
+ Map<TopicPartition, List<ConsumerRecord<String, String>>> records = new HashMap<>();
+ //This is cheating a bit since maxPollRecords would normally spread this across multiple polls
+ records.put(partition, SpoutWithMockedConsumerSetupHelper.<String, String>createRecords(partition, 0, spoutConfig.getMaxUncommittedOffsets()));
+ records.put(partitionTwo, SpoutWithMockedConsumerSetupHelper.<String, String>createRecords(partitionTwo, 0, spoutConfig.getMaxUncommittedOffsets()));
when(consumerMock.poll(anyLong()))
- .thenReturn(new ConsumerRecords<>(firstPollRecords))
- .thenReturn(new ConsumerRecords<>(secondPollRecords));
+ .thenReturn(new ConsumerRecords<>(records));
- for (int i = 0; i < spoutConfig.getMaxUncommittedOffsets() + maxPollRecords; i++) {
+ for (int i = 0; i < spoutConfig.getMaxUncommittedOffsets()*2; i++) {
spout.nextTuple();
}
ArgumentCaptor<KafkaSpoutMessageId> messageIds = ArgumentCaptor.forClass(KafkaSpoutMessageId.class);
- verify(collectorMock, times(firstPollRecordsForPartition.size())).emit(anyString(), anyList(), messageIds.capture());
-
- KafkaSpoutMessageId failedMessageId = messageIds.getAllValues().get(messageIds.getAllValues().size() - 1);
+ verify(collectorMock, times(spoutConfig.getMaxUncommittedOffsets()*2)).emit(anyString(), anyList(), messageIds.capture());
+
+ //Now fail a tuple on partition 0 and verify that it is allowed to retry
+ //Partition 1 should be paused, since it is at the uncommitted offsets limit
+ KafkaSpoutMessageId failedMessageId = null;
+ for (KafkaSpoutMessageId msgId : messageIds.getAllValues()) {
+ if (msgId.partition() == partition.partition()) {
+ failedMessageId = msgId;
+ break;
+ }
+ }
+
spout.fail(failedMessageId);
-
+
reset(collectorMock);
-
- //Now make the single failed tuple retriable
+
Time.advanceTime(50);
- //The spout should allow another poll since there are now only maxUncommittedOffsets - 1 nonretriable tuples
- for (int i = 0; i < firstPollRecordsForPartition.size() + maxPollRecords; i++) {
- spout.nextTuple();
- }
-
- ArgumentCaptor<KafkaSpoutMessageId> retryBatchMessageIdsCaptor = ArgumentCaptor.forClass(KafkaSpoutMessageId.class);
- verify(collectorMock, times(maxPollRecords)).emit(anyString(), anyList(), retryBatchMessageIdsCaptor.capture());
- reset(collectorMock);
+ when(consumerMock.poll(anyLong()))
+ .thenReturn(new ConsumerRecords<>(Collections.singletonMap(partition, SpoutWithMockedConsumerSetupHelper.<String, String>createRecords(partition, failedMessageId.offset(), 1))));
+
+ spout.nextTuple();
+
+ verify(collectorMock, times(1)).emit(anyString(), anyList(), anyObject());
- //Check that the consumer started polling at the failed tuple offset
InOrder inOrder = inOrder(consumerMock);
- inOrder.verify(consumerMock).seek(partition, failedMessageId.offset());
+ inOrder.verify(consumerMock).pause(Collections.singleton(partitionTwo));
inOrder.verify(consumerMock).poll(anyLong());
+ inOrder.verify(consumerMock).resume(Collections.singleton(partitionTwo));
- //Now fail all except one of the last batch, and check that the spout won't reemit any tuples because there are more than maxUncommittedOffsets nonretriable tuples
- Time.advanceTime(50);
- List<KafkaSpoutMessageId> retryBatchMessageIds = retryBatchMessageIdsCaptor.getAllValues();
- KafkaSpoutMessageId firstTupleFromRetryBatch = retryBatchMessageIds.remove(0);
- for(KafkaSpoutMessageId msgId : retryBatchMessageIds) {
- spout.fail(msgId);
- }
- for (int i = 0; i < firstPollRecordsForPartition.size() + maxPollRecords; i++) {
- spout.nextTuple();
- }
- verify(collectorMock, never()).emit(anyString(), anyList(), anyObject());
+ reset(collectorMock);
- //Fail the last tuple, which brings the number of nonretriable tuples back under the limit, and check that the spout polls again
- spout.fail(firstTupleFromRetryBatch);
+ //Now also check that no more tuples are polled for, since both partitions are at their limits
spout.nextTuple();
- verify(collectorMock, times(1)).emit(anyString(), anyList(), anyObject());
+
+ verify(collectorMock, never()).emit(anyString(), anyList(), anyObject());
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/07ecff12/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutRebalanceTest.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutRebalanceTest.java b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutRebalanceTest.java
index 010433a..bb9a5bd 100644
--- a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutRebalanceTest.java
+++ b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutRebalanceTest.java
@@ -31,9 +31,10 @@ import static org.mockito.Mockito.when;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
-
+import java.util.Set;
import org.apache.kafka.clients.consumer.ConsumerRebalanceListener;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;
@@ -94,19 +95,16 @@ public class KafkaSpoutRebalanceTest {
//Assign partitions to the spout
ConsumerRebalanceListener consumerRebalanceListener = rebalanceListenerCapture.getValue();
- List<TopicPartition> assignedPartitions = new ArrayList<>();
+ Set<TopicPartition> assignedPartitions = new HashSet<>();
assignedPartitions.add(partitionThatWillBeRevoked);
assignedPartitions.add(assignedPartition);
consumerRebalanceListener.onPartitionsAssigned(assignedPartitions);
+ when(consumerMock.assignment()).thenReturn(assignedPartitions);
//Make the consumer return a single message for each partition
- Map<TopicPartition, List<ConsumerRecord<String, String>>> firstPartitionRecords = new HashMap<>();
- firstPartitionRecords.put(partitionThatWillBeRevoked, Collections.singletonList(new ConsumerRecord<>(partitionThatWillBeRevoked.topic(), partitionThatWillBeRevoked.partition(), 0L, "key", "value")));
- Map<TopicPartition, List<ConsumerRecord<String, String>>> secondPartitionRecords = new HashMap<>();
- secondPartitionRecords.put(assignedPartition, Collections.singletonList(new ConsumerRecord<>(assignedPartition.topic(), assignedPartition.partition(), 0L, "key", "value")));
when(consumerMock.poll(anyLong()))
- .thenReturn(new ConsumerRecords<>(firstPartitionRecords))
- .thenReturn(new ConsumerRecords<>(secondPartitionRecords))
+ .thenReturn(new ConsumerRecords<>(Collections.singletonMap(partitionThatWillBeRevoked, SpoutWithMockedConsumerSetupHelper.<String, String>createRecords(partitionThatWillBeRevoked, 0, 1))))
+ .thenReturn(new ConsumerRecords<>(Collections.singletonMap(assignedPartition, SpoutWithMockedConsumerSetupHelper.<String, String>createRecords(assignedPartition, 0, 1))))
.thenReturn(new ConsumerRecords<>(new HashMap<TopicPartition, List<ConsumerRecord<String, String>>>()));
//Emit the messages
@@ -121,6 +119,7 @@ public class KafkaSpoutRebalanceTest {
//Now rebalance
consumerRebalanceListener.onPartitionsRevoked(assignedPartitions);
consumerRebalanceListener.onPartitionsAssigned(Collections.singleton(assignedPartition));
+ when(consumerMock.assignment()).thenReturn(Collections.singleton(assignedPartition));
List<KafkaSpoutMessageId> emittedMessageIds = new ArrayList<>();
emittedMessageIds.add(messageIdForRevokedPartition.getValue());
http://git-wip-us.apache.org/repos/asf/storm/blob/07ecff12/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutRetryLimitTest.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutRetryLimitTest.java b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutRetryLimitTest.java
index 21997f8..24b1373 100644
--- a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutRetryLimitTest.java
+++ b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutRetryLimitTest.java
@@ -81,22 +81,19 @@ public class KafkaSpoutRetryLimitTest {
try (SimulatedTime simulatedTime = new SimulatedTime()) {
KafkaSpout<String, String> spout = SpoutWithMockedConsumerSetupHelper.setupSpout(spoutConfig, conf, contextMock, collectorMock, consumerMock, Collections.singleton(partition));
Map<TopicPartition, List<ConsumerRecord<String, String>>> records = new HashMap<>();
- List<ConsumerRecord<String, String>> recordsForPartition = new ArrayList<>();
int lastOffset = 3;
- for (int i = 0; i <= lastOffset; i++) {
- recordsForPartition.add(new ConsumerRecord<>(partition.topic(), partition.partition(), i, "key", "value"));
- }
- records.put(partition, recordsForPartition);
+ int numRecords = lastOffset + 1;
+ records.put(partition, SpoutWithMockedConsumerSetupHelper.<String, String>createRecords(partition, 0, numRecords));
when(consumerMock.poll(anyLong()))
.thenReturn(new ConsumerRecords<>(records));
- for (int i = 0; i < recordsForPartition.size(); i++) {
+ for (int i = 0; i < numRecords; i++) {
spout.nextTuple();
}
ArgumentCaptor<KafkaSpoutMessageId> messageIds = ArgumentCaptor.forClass(KafkaSpoutMessageId.class);
- verify(collectorMock, times(recordsForPartition.size())).emit(anyString(), anyList(), messageIds.capture());
+ verify(collectorMock, times(numRecords)).emit(anyString(), anyList(), messageIds.capture());
for (KafkaSpoutMessageId messageId : messageIds.getAllValues()) {
spout.fail(messageId);
http://git-wip-us.apache.org/repos/asf/storm/blob/07ecff12/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/MaxUncommittedOffsetTest.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/MaxUncommittedOffsetTest.java b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/MaxUncommittedOffsetTest.java
index 80a8e1d..0926671 100755
--- a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/MaxUncommittedOffsetTest.java
+++ b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/MaxUncommittedOffsetTest.java
@@ -15,6 +15,7 @@
*/
package org.apache.storm.kafka.spout;
+import static org.hamcrest.CoreMatchers.either;
import static org.hamcrest.CoreMatchers.everyItem;
import static org.hamcrest.CoreMatchers.is;
import static org.hamcrest.Matchers.greaterThanOrEqualTo;
@@ -95,7 +96,7 @@ public class MaxUncommittedOffsetTest {
ArgumentCaptor<KafkaSpoutMessageId> messageIds = ArgumentCaptor.forClass(KafkaSpoutMessageId.class);
for (int i = 0; i < messageCount; i++) {
spout.nextTuple();
- };
+ }
verify(collector, times(maxUncommittedOffsets)).emit(
anyString(),
anyList(),
@@ -171,8 +172,13 @@ public class MaxUncommittedOffsetTest {
@Test
public void testNextTupleWillNotEmitMoreThanMaxUncommittedOffsetsPlusMaxPollRecordsMessages() throws Exception {
- //The upper bound on uncommitted offsets should be maxUncommittedOffsets + maxPollRecords - 1
- //This is reachable by emitting maxUncommittedOffsets messages, acking the first message, then polling.
+ /*
+ For each partition the spout is allowed to retry all tuples between the committed offset, and maxUncommittedOffsets ahead.
+ It is not allowed to retry tuples past that limit.
+ This makes the actual limit per partition maxUncommittedOffsets + maxPollRecords - 1,
+ reached if the tuple at the maxUncommittedOffsets limit is the earliest retriable tuple,
+ or if the spout is 1 tuple below the limit, and receives a full maxPollRecords tuples in the poll.
+ */
try (Time.SimulatedTime simulatedTime = new Time.SimulatedTime()) {
//First check that maxUncommittedOffsets is respected when emitting from scratch
ArgumentCaptor<KafkaSpoutMessageId> messageIds = emitMaxUncommittedOffsetsMessagesAndCheckNoMoreAreEmitted(numMessages);
@@ -180,7 +186,7 @@ public class MaxUncommittedOffsetTest {
failAllExceptTheFirstMessageThenCommit(messageIds);
- //Offset 0 is acked, 1 to maxUncommittedOffsets - 1 are failed
+ //Offset 0 is acked, 1 to maxUncommittedOffsets - 1 are failed but not retriable
//The spout should now emit another maxPollRecords messages
//This is allowed because the acked message brings the numUncommittedOffsets below the cap
for (int i = 0; i < maxUncommittedOffsets; i++) {
@@ -204,18 +210,20 @@ public class MaxUncommittedOffsetTest {
}
assertThat("Expected the newly emitted messages to have no overlap with the first batch", secondRunOffsets.removeAll(firstRunOffsets), is(false));
- //Offset 0 is acked, 1 to maxUncommittedOffsets-1 are failed, maxUncommittedOffsets to maxUncommittedOffsets + maxPollRecords-1 are emitted
- //There are now maxUncommittedOffsets-1 + maxPollRecords records emitted past the last committed offset
- //Advance time so the failed tuples become ready for retry, and check that the spout will emit retriable tuples as long as numNonRetriableEmittedTuples < maxUncommittedOffsets
-
- int numNonRetriableEmittedTuples = maxPollRecords; //The other tuples were failed and are becoming retriable
- int allowedPolls = (int)Math.ceil((maxUncommittedOffsets - numNonRetriableEmittedTuples)/(double)maxPollRecords);
+ //Offset 0 is committed, 1 to maxUncommittedOffsets-1 are failed, maxUncommittedOffsets to maxUncommittedOffsets + maxPollRecords-1 are emitted
+ //Fail the last tuples so only offset 0 is not failed.
+ //Advance time so the failed tuples become ready for retry, and check that the spout will emit retriable tuples
+ //for all the failed tuples that are within maxUncommittedOffsets tuples of the committed offset
+ //This means 1 to maxUncommitteddOffsets, but not maxUncommittedOffsets+1...maxUncommittedOffsets+maxPollRecords-1
+ for(KafkaSpoutMessageId msgId : secondRunMessageIds.getAllValues()) {
+ spout.fail(msgId);
+ }
Time.advanceTimeSecs(initialRetryDelaySecs);
for (int i = 0; i < numMessages; i++) {
spout.nextTuple();
}
ArgumentCaptor<KafkaSpoutMessageId> thirdRunMessageIds = ArgumentCaptor.forClass(KafkaSpoutMessageId.class);
- verify(collector, times(allowedPolls*maxPollRecords)).emit(
+ verify(collector, times(maxUncommittedOffsets)).emit(
anyString(),
anyList(),
thirdRunMessageIds.capture());
@@ -226,7 +234,7 @@ public class MaxUncommittedOffsetTest {
thirdRunOffsets.add(msgId.offset());
}
- assertThat("Expected the emitted messages to be retries of the failed tuples from the first batch", thirdRunOffsets, everyItem(isIn(firstRunOffsets)));
+ assertThat("Expected the emitted messages to be retries of the failed tuples from the first batch, plus the first failed tuple from the second batch", thirdRunOffsets, everyItem(either(isIn(firstRunOffsets)).or(is(secondRunMessageIds.getAllValues().get(0).offset()))));
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/07ecff12/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/SingleTopicKafkaSpoutTest.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/SingleTopicKafkaSpoutTest.java b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/SingleTopicKafkaSpoutTest.java
index 211ae97..ec7f694 100644
--- a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/SingleTopicKafkaSpoutTest.java
+++ b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/SingleTopicKafkaSpoutTest.java
@@ -32,10 +32,8 @@ import java.util.Map;
import static org.hamcrest.CoreMatchers.is;
import static org.hamcrest.MatcherAssert.assertThat;
-import static org.mockito.Matchers.anyList;
import static org.mockito.Matchers.anyListOf;
import static org.mockito.Matchers.anyObject;
-import static org.mockito.Matchers.anyString;
import static org.mockito.Matchers.eq;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.never;
@@ -58,10 +56,11 @@ import org.junit.Before;
import org.mockito.Captor;
import org.mockito.MockitoAnnotations;
-import static org.apache.storm.kafka.spout.builders.SingleTopicKafkaSpoutConfiguration.createKafkaSpoutConfigBuilder;
+import static org.mockito.Matchers.any;
import static org.mockito.Matchers.anyList;
import static org.mockito.Matchers.anyString;
+import java.util.regex.Pattern;
import org.apache.kafka.clients.consumer.ConsumerConfig;
import org.hamcrest.Matchers;
@@ -86,12 +85,15 @@ public class SingleTopicKafkaSpoutTest {
@Before
public void setUp() {
MockitoAnnotations.initMocks(this);
- KafkaSpoutConfig<String, String> spoutConfig = createKafkaSpoutConfigBuilder(kafkaUnitRule.getKafkaUnit().getKafkaPort())
- .setOffsetCommitPeriodMs(commitOffsetPeriodMs)
- .setRetry(new KafkaSpoutRetryExponentialBackoff(KafkaSpoutRetryExponentialBackoff.TimeInterval.seconds(0), KafkaSpoutRetryExponentialBackoff.TimeInterval.seconds(0),
- maxRetries, KafkaSpoutRetryExponentialBackoff.TimeInterval.seconds(0)))
- .setProp(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, maxPollRecords)
- .build();
+ KafkaSpoutConfig<String, String> spoutConfig =
+ SingleTopicKafkaSpoutConfiguration.setCommonSpoutConfig(
+ KafkaSpoutConfig.builder("127.0.0.1:" + kafkaUnitRule.getKafkaUnit().getKafkaPort(),
+ Pattern.compile(SingleTopicKafkaSpoutConfiguration.TOPIC)))
+ .setOffsetCommitPeriodMs(commitOffsetPeriodMs)
+ .setRetry(new KafkaSpoutRetryExponentialBackoff(KafkaSpoutRetryExponentialBackoff.TimeInterval.seconds(0), KafkaSpoutRetryExponentialBackoff.TimeInterval.seconds(0),
+ maxRetries, KafkaSpoutRetryExponentialBackoff.TimeInterval.seconds(0)))
+ .setProp(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, maxPollRecords)
+ .build();
this.consumerSpy = spy(new KafkaConsumerFactoryDefault<String, String>().createConsumer(spoutConfig));
this.consumerFactory = new KafkaConsumerFactory<String, String>() {
@Override
@@ -107,7 +109,7 @@ public class SingleTopicKafkaSpoutTest {
SingleTopicKafkaUnitSetupHelper.populateTopicData(kafkaUnitRule.getKafkaUnit(), SingleTopicKafkaSpoutConfiguration.TOPIC, messageCount);
SingleTopicKafkaUnitSetupHelper.initializeSpout(spout, conf, topologyContext, collector);
}
-
+
/*
* Asserts that commitSync has been called once,
* that there are only commits on one topic,
@@ -175,7 +177,7 @@ public class SingleTopicKafkaSpoutTest {
}
@Test
- public void shouldContinueWithSlowDoubleAcks() throws Exception {
+ public void testShouldContinueWithSlowDoubleAcks() throws Exception {
try (SimulatedTime simulatedTime = new SimulatedTime()) {
int messageCount = 20;
prepareSpout(messageCount);
@@ -216,7 +218,7 @@ public class SingleTopicKafkaSpoutTest {
}
@Test
- public void shouldEmitAllMessages() throws Exception {
+ public void testShouldEmitAllMessages() throws Exception {
try (SimulatedTime simulatedTime = new SimulatedTime()) {
int messageCount = 10;
prepareSpout(messageCount);
@@ -244,7 +246,7 @@ public class SingleTopicKafkaSpoutTest {
}
@Test
- public void shouldReplayInOrderFailedMessages() throws Exception {
+ public void testShouldReplayInOrderFailedMessages() throws Exception {
try (SimulatedTime simulatedTime = new SimulatedTime()) {
int messageCount = 10;
prepareSpout(messageCount);
@@ -287,7 +289,7 @@ public class SingleTopicKafkaSpoutTest {
}
@Test
- public void shouldReplayFirstTupleFailedOutOfOrder() throws Exception {
+ public void testShouldReplayFirstTupleFailedOutOfOrder() throws Exception {
try (SimulatedTime simulatedTime = new SimulatedTime()) {
int messageCount = 10;
prepareSpout(messageCount);
@@ -333,7 +335,7 @@ public class SingleTopicKafkaSpoutTest {
}
@Test
- public void shouldReplayAllFailedTuplesWhenFailedOutOfOrder() throws Exception {
+ public void testShouldReplayAllFailedTuplesWhenFailedOutOfOrder() throws Exception {
//The spout must reemit retriable tuples, even if they fail out of order.
//The spout should be able to skip tuples it has already emitted when retrying messages, even if those tuples are also retries.
int messageCount = 10;
@@ -367,7 +369,7 @@ public class SingleTopicKafkaSpoutTest {
}
@Test
- public void shouldDropMessagesAfterMaxRetriesAreReached() throws Exception {
+ public void testShouldDropMessagesAfterMaxRetriesAreReached() throws Exception {
//Check that if one message fails repeatedly, the retry cap limits how many times the message can be reemitted
int messageCount = 1;
prepareSpout(messageCount);
@@ -387,4 +389,22 @@ public class SingleTopicKafkaSpoutTest {
spout.nextTuple();
verify(collector, never()).emit(anyString(), anyListOf(Object.class), anyObject());
}
+
+ @Test
+ public void testSpoutMustRefreshPartitionsEvenIfNotPolling() throws Exception {
+ try (SimulatedTime time = new SimulatedTime()) {
+ SingleTopicKafkaUnitSetupHelper.initializeSpout(spout, conf, topologyContext, collector);
+
+ //Nothing is assigned yet, should emit nothing
+ spout.nextTuple();
+ verify(collector, never()).emit(anyString(), anyList(), any(KafkaSpoutMessageId.class));
+
+ SingleTopicKafkaUnitSetupHelper.populateTopicData(kafkaUnitRule.getKafkaUnit(), SingleTopicKafkaSpoutConfiguration.TOPIC, 1);
+ Time.advanceTime(KafkaSpoutConfig.DEFAULT_PARTITION_REFRESH_PERIOD_MS + KafkaSpout.TIMER_DELAY_MS);
+
+ //The new partition should be discovered and the message should be emitted
+ spout.nextTuple();
+ verify(collector).emit(anyString(), anyList(), any(KafkaSpoutMessageId.class));
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/07ecff12/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/SpoutWithMockedConsumerSetupHelper.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/SpoutWithMockedConsumerSetupHelper.java b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/SpoutWithMockedConsumerSetupHelper.java
index 3d67ebd..6f6196a 100644
--- a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/SpoutWithMockedConsumerSetupHelper.java
+++ b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/SpoutWithMockedConsumerSetupHelper.java
@@ -51,33 +51,16 @@ public class SpoutWithMockedConsumerSetupHelper {
*/
public static <K, V> KafkaSpout<K, V> setupSpout(KafkaSpoutConfig<K, V> spoutConfig, Map<String, Object> topoConf,
TopologyContext contextMock, SpoutOutputCollector collectorMock, final KafkaConsumer<K, V> consumerMock, Set<TopicPartition> assignedPartitions) {
-
- Map<String, List<PartitionInfo>> partitionInfos = new HashMap<>();
- for(TopicPartition tp : assignedPartitions) {
- PartitionInfo info = new PartitionInfo(tp.topic(), tp.partition(), null, null, null);
- List<PartitionInfo> infos = partitionInfos.get(tp.topic());
- if(infos == null) {
- infos = new ArrayList<>();
- partitionInfos.put(tp.topic(), infos);
- }
- infos.add(info);
- }
- for(String topic : partitionInfos.keySet()) {
- when(consumerMock.partitionsFor(topic))
- .thenReturn(partitionInfos.get(topic));
- }
+
+ stubAssignment(contextMock, consumerMock, assignedPartitions);
KafkaConsumerFactory<K, V> consumerFactory = new KafkaConsumerFactory<K, V>() {
@Override
public KafkaConsumer<K, V> createConsumer(KafkaSpoutConfig<K, V> kafkaSpoutConfig) {
return consumerMock;
}
};
-
KafkaSpout<K, V> spout = new KafkaSpout<>(spoutConfig, consumerFactory);
-
- when(contextMock.getComponentTasks(anyString())).thenReturn(Collections.singletonList(0));
- when(contextMock.getThisTaskIndex()).thenReturn(0);
-
+
spout.open(topoConf, contextMock, collectorMock);
spout.activate();
@@ -85,6 +68,35 @@ public class SpoutWithMockedConsumerSetupHelper {
return spout;
}
+
+ /**
+ * Sets up the mocked context and consumer to appear to have the given partition assignment.
+ *
+ * @param <K> The Kafka key type
+ * @param <V> The Kafka value type
+ * @param contextMock The mocked topology context
+ * @param consumerMock The mocked consumer
+ * @param assignedPartitions The partitions to assign to the consumer
+ */
+ public static <K, V> void stubAssignment(TopologyContext contextMock, KafkaConsumer<K, V> consumerMock, Set<TopicPartition> assignedPartitions) {
+ Map<String, List<PartitionInfo>> partitionInfos = new HashMap<>();
+ for (TopicPartition tp : assignedPartitions) {
+ PartitionInfo info = new PartitionInfo(tp.topic(), tp.partition(), null, null, null);
+ List<PartitionInfo> infos = partitionInfos.get(tp.topic());
+ if (infos == null) {
+ infos = new ArrayList<>();
+ partitionInfos.put(tp.topic(), infos);
+ }
+ infos.add(info);
+ }
+ for (String topic : partitionInfos.keySet()) {
+ when(consumerMock.partitionsFor(topic)).thenReturn(partitionInfos.get(topic));
+ }
+ when(contextMock.getComponentTasks(anyString())).thenReturn(Collections.singletonList(0));
+ when(contextMock.getThisTaskIndex()).thenReturn(0);
+
+ when(consumerMock.assignment()).thenReturn(assignedPartitions);
+ }
/**
* Creates sequential dummy records
http://git-wip-us.apache.org/repos/asf/storm/blob/07ecff12/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/builders/SingleTopicKafkaSpoutConfiguration.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/builders/SingleTopicKafkaSpoutConfiguration.java b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/builders/SingleTopicKafkaSpoutConfiguration.java
index f4cb833..b178687 100644
--- a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/builders/SingleTopicKafkaSpoutConfiguration.java
+++ b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/builders/SingleTopicKafkaSpoutConfiguration.java
@@ -69,7 +69,7 @@ public class SingleTopicKafkaSpoutConfiguration {
return setCommonSpoutConfig(new KafkaSpoutConfig.Builder<String, String>("127.0.0.1:" + port, subscription));
}
- private static KafkaSpoutConfig.Builder<String, String> setCommonSpoutConfig(KafkaSpoutConfig.Builder<String, String> config) {
+ public static KafkaSpoutConfig.Builder<String, String> setCommonSpoutConfig(KafkaSpoutConfig.Builder<String, String> config) {
return config
.setRecordTranslator(TOPIC_KEY_VALUE_FUNC,
new Fields("topic", "key", "value"), STREAM)
http://git-wip-us.apache.org/repos/asf/storm/blob/07ecff12/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/internal/OffsetManagerTest.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/internal/OffsetManagerTest.java b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/internal/OffsetManagerTest.java
index abbacf9..3acc252 100644
--- a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/internal/OffsetManagerTest.java
+++ b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/internal/OffsetManagerTest.java
@@ -170,4 +170,19 @@ public class OffsetManagerTest {
manager.addToAckMsgs(msgId);
}
+ @Test
+ public void testGetNthUncommittedOffsetAfterCommittedOffset() {
+ manager.addToEmitMsgs(initialFetchOffset + 1);
+ manager.addToEmitMsgs(initialFetchOffset + 2);
+ manager.addToEmitMsgs(initialFetchOffset + 5);
+ manager.addToEmitMsgs(initialFetchOffset + 30);
+
+ assertThat("The third uncommitted offset should be 5", manager.getNthUncommittedOffsetAfterCommittedOffset(3), is(initialFetchOffset + 5L));
+ assertThat("The fourth uncommitted offset should be 30", manager.getNthUncommittedOffsetAfterCommittedOffset(4), is(initialFetchOffset + 30L));
+
+ expect.expect(NoSuchElementException.class);
+ manager.getNthUncommittedOffsetAfterCommittedOffset(5);
+
+ }
+
}
[05/50] [abbrv] storm git commit: Merge branch 'STORM-2648-1.x' of
https://github.com/srdo/storm into asfgit-1.x-branch
Posted by pt...@apache.org.
Merge branch 'STORM-2648-1.x' of https://github.com/srdo/storm into asfgit-1.x-branch
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/cb2d7e8b
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/cb2d7e8b
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/cb2d7e8b
Branch: refs/heads/metrics_v2
Commit: cb2d7e8be2423bc7be34e388dee4fbd697724c48
Parents: fca692d 11a7a15
Author: Stig Rohde Døssing <sr...@apache.org>
Authored: Mon Oct 2 18:09:11 2017 +0200
Committer: Stig Rohde Døssing <sr...@apache.org>
Committed: Mon Oct 2 18:09:11 2017 +0200
----------------------------------------------------------------------
docs/storm-kafka-client.md | 31 ++-
.../apache/storm/kafka/spout/KafkaSpout.java | 175 +++++++++-------
.../storm/kafka/spout/KafkaSpoutConfig.java | 99 ++++++++-
.../storm/kafka/spout/KafkaSpoutCommitTest.java | 5 +-
.../storm/kafka/spout/KafkaSpoutEmitTest.java | 8 +-
.../spout/KafkaSpoutMessagingGuaranteeTest.java | 207 +++++++++++++++++++
.../kafka/spout/KafkaSpoutRebalanceTest.java | 8 +-
.../kafka/spout/KafkaSpoutRetryLimitTest.java | 5 +-
.../kafka/spout/MaxUncommittedOffsetTest.java | 32 +--
.../kafka/spout/SingleTopicKafkaSpoutTest.java | 47 ++---
.../spout/SingleTopicKafkaUnitSetupHelper.java | 67 ++++++
.../SingleTopicKafkaSpoutConfiguration.java | 23 +--
12 files changed, 522 insertions(+), 185 deletions(-)
----------------------------------------------------------------------
[02/50] [abbrv] storm git commit: Merge branch 'STORM-2764-1.x' of
https://github.com/revans2/incubator-storm into STORM-2764-1.x-merge
Posted by pt...@apache.org.
Merge branch 'STORM-2764-1.x' of https://github.com/revans2/incubator-storm into STORM-2764-1.x-merge
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/5890333c
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/5890333c
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/5890333c
Branch: refs/heads/metrics_v2
Commit: 5890333c7f1f7c823b2d12194063ca781fb4df52
Parents: 892bf5c 26ff239
Author: Jungtaek Lim <ka...@gmail.com>
Authored: Mon Oct 2 14:45:23 2017 +0900
Committer: Jungtaek Lim <ka...@gmail.com>
Committed: Mon Oct 2 14:45:23 2017 +0900
----------------------------------------------------------------------
.../storm/hdfs/blobstore/HdfsBlobStore.java | 107 ++++++++++---------
1 file changed, 59 insertions(+), 48 deletions(-)
----------------------------------------------------------------------
[24/50] [abbrv] storm git commit: Merge branch 'STORM-2791-1.x' of
https://github.com/danielsd/storm into STORM-2791-1.x-merge
Posted by pt...@apache.org.
Merge branch 'STORM-2791-1.x' of https://github.com/danielsd/storm into STORM-2791-1.x-merge
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/96864e83
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/96864e83
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/96864e83
Branch: refs/heads/metrics_v2
Commit: 96864e8356eba99bed8c61c0a6d96bceda83cab4
Parents: 3414a8c af1fd21
Author: Jungtaek Lim <ka...@gmail.com>
Authored: Sun Oct 29 14:08:04 2017 +0900
Committer: Jungtaek Lim <ka...@gmail.com>
Committed: Sun Oct 29 14:08:04 2017 +0900
----------------------------------------------------------------------
.../org/apache/storm/testing/FixedTupleSpout.java | 18 +++++++++++++-----
1 file changed, 13 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
[13/50] [abbrv] storm git commit: STORM-2779 NPE on shutting down
WindowedBoltExecutor
Posted by pt...@apache.org.
STORM-2779 NPE on shutting down WindowedBoltExecutor
* waterMarkEventGenerator could be null when timestamp field is not specified
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/40384cf5
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/40384cf5
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/40384cf5
Branch: refs/heads/metrics_v2
Commit: 40384cf5f21337cc496660c1bf0c59ad41d30788
Parents: 39e12aa
Author: Jungtaek Lim <ka...@gmail.com>
Authored: Tue Oct 17 12:40:48 2017 +0900
Committer: Jungtaek Lim <ka...@gmail.com>
Committed: Tue Oct 17 12:42:11 2017 +0900
----------------------------------------------------------------------
.../src/jvm/org/apache/storm/topology/WindowedBoltExecutor.java | 4 +++-
1 file changed, 3 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/40384cf5/storm-core/src/jvm/org/apache/storm/topology/WindowedBoltExecutor.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/topology/WindowedBoltExecutor.java b/storm-core/src/jvm/org/apache/storm/topology/WindowedBoltExecutor.java
index 26c69a2..c9afc67 100644
--- a/storm-core/src/jvm/org/apache/storm/topology/WindowedBoltExecutor.java
+++ b/storm-core/src/jvm/org/apache/storm/topology/WindowedBoltExecutor.java
@@ -299,7 +299,9 @@ public class WindowedBoltExecutor implements IRichBolt {
@Override
public void cleanup() {
- waterMarkEventGenerator.shutdown();
+ if (waterMarkEventGenerator != null) {
+ waterMarkEventGenerator.shutdown();
+ }
windowManager.shutdown();
bolt.cleanup();
}
[43/50] [abbrv] storm git commit: Merge branch 'STORM-2810-1.x' of
https://github.com/srdo/storm into asfgit-1.x-branch
Posted by pt...@apache.org.
Merge branch 'STORM-2810-1.x' of https://github.com/srdo/storm into asfgit-1.x-branch
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/6d45875d
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/6d45875d
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/6d45875d
Branch: refs/heads/metrics_v2
Commit: 6d45875dac89be8d3c35f84194a7e1b95e364343
Parents: a5ed0f9 83adfd9
Author: Stig Rohde Døssing <sr...@apache.org>
Authored: Tue Nov 14 07:50:48 2017 +0100
Committer: Stig Rohde Døssing <sr...@apache.org>
Committed: Tue Nov 14 07:50:48 2017 +0100
----------------------------------------------------------------------
.../storm/hdfs/blobstore/HdfsBlobStoreImpl.java | 14 +-
.../storm/hdfs/bolt/AbstractHdfsBolt.java | 48 +-
.../org/apache/storm/hdfs/spout/HdfsSpout.java | 1484 +++++++++---------
.../hdfs/avro/TestFixedAvroSerializer.java | 13 +-
.../hdfs/avro/TestGenericAvroSerializer.java | 11 +-
.../storm/hdfs/blobstore/BlobStoreTest.java | 897 ++++++-----
.../hdfs/blobstore/HdfsBlobStoreImplTest.java | 288 ++--
.../hdfs/bolt/AvroGenericRecordBoltTest.java | 142 +-
.../apache/storm/hdfs/bolt/TestHdfsBolt.java | 108 +-
.../storm/hdfs/bolt/TestSequenceFileBolt.java | 71 +-
.../apache/storm/hdfs/spout/TestDirLock.java | 281 ++--
.../apache/storm/hdfs/spout/TestFileLock.java | 679 ++++----
.../storm/hdfs/spout/TestHdfsSemantics.java | 311 ++--
.../apache/storm/hdfs/spout/TestHdfsSpout.java | 1235 +++++++--------
.../storm/hdfs/spout/TestProgressTracker.java | 179 +--
.../storm/hdfs/testing/MiniDFSClusterRule.java | 78 +
16 files changed, 2955 insertions(+), 2884 deletions(-)
----------------------------------------------------------------------
[40/50] [abbrv] storm git commit: STORM-2810: Fix resource leaks in
storm-hdfs tests
Posted by pt...@apache.org.
http://git-wip-us.apache.org/repos/asf/storm/blob/83adfd9d/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/spout/TestDirLock.java
----------------------------------------------------------------------
diff --git a/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/spout/TestDirLock.java b/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/spout/TestDirLock.java
index b96f1ff..0d1f146 100644
--- a/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/spout/TestDirLock.java
+++ b/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/spout/TestDirLock.java
@@ -1,3 +1,4 @@
+
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
@@ -9,180 +10,170 @@
* <p/>
* http://www.apache.org/licenses/LICENSE-2.0
* <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
-
package org.apache.storm.hdfs.spout;
-
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.HdfsConfiguration;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.junit.After;
-import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.Before;
-import org.junit.BeforeClass;
import org.junit.Test;
import java.io.IOException;
+import org.apache.storm.hdfs.testing.MiniDFSClusterRule;
+import org.junit.Rule;
public class TestDirLock {
- static MiniDFSCluster.Builder builder;
- static MiniDFSCluster hdfsCluster;
- static FileSystem fs;
- static String hdfsURI;
- static HdfsConfiguration conf = new HdfsConfiguration();
- static final int LOCK_EXPIRY_SEC = 1;
- private Path locksDir = new Path("/tmp/lockdir");
-
- @BeforeClass
- public static void setupClass() throws IOException {
- conf.set(CommonConfigurationKeys.IPC_PING_INTERVAL_KEY,"5000");
- builder = new MiniDFSCluster.Builder(new Configuration());
- hdfsCluster = builder.build();
- fs = hdfsCluster.getFileSystem();
- hdfsURI = "hdfs://localhost:" + hdfsCluster.getNameNodePort() + "/";
- }
-
- @AfterClass
- public static void teardownClass() throws IOException {
- fs.close();
- hdfsCluster.shutdown();
- }
-
- @Before
- public void setUp() throws Exception {
- assert fs.mkdirs(locksDir) ;
- }
-
- @After
- public void tearDown() throws Exception {
- fs.delete(locksDir, true);
- }
-
-
- @Test
- public void testBasicLocking() throws Exception {
- // 1 grab lock
- DirLock lock = DirLock.tryLock(fs, locksDir);
- Assert.assertTrue(fs.exists(lock.getLockFile()));
-
- // 2 try to grab another lock while dir is locked
- DirLock lock2 = DirLock.tryLock(fs, locksDir); // should fail
- Assert.assertNull(lock2);
-
- // 3 let go first lock
- lock.release();
- Assert.assertFalse(fs.exists(lock.getLockFile()));
-
- // 4 try locking again
- lock2 = DirLock.tryLock(fs, locksDir);
- Assert.assertTrue(fs.exists(lock2.getLockFile()));
- lock2.release();
- Assert.assertFalse(fs.exists(lock.getLockFile()));
- lock2.release(); // should be throw
- }
-
-
- @Test
- public void testConcurrentLocking() throws Exception {
- DirLockingThread[] thds = startThreads(100, locksDir);
- for (DirLockingThread thd : thds) {
- thd.join();
- if( !thd.cleanExit ) {
- System.err.println(thd.getName() + " did not exit cleanly");
- }
- Assert.assertTrue(thd.cleanExit);
- }
+ @Rule
+ public MiniDFSClusterRule DFS_CLUSTER_RULE = new MiniDFSClusterRule();
- Path lockFile = new Path(locksDir + Path.SEPARATOR + DirLock.DIR_LOCK_FILE);
- Assert.assertFalse(fs.exists(lockFile));
- }
+ private static final int LOCK_EXPIRY_SEC = 1;
- private DirLockingThread[] startThreads(int thdCount, Path dir)
- throws IOException {
- DirLockingThread[] result = new DirLockingThread[thdCount];
- for (int i = 0; i < thdCount; i++) {
- result[i] = new DirLockingThread(i, fs, dir);
+ private FileSystem fs;
+ private HdfsConfiguration conf = new HdfsConfiguration();
+ private final Path locksDir = new Path("/tmp/lockdir");
+
+ @Before
+ public void setUp() throws IOException {
+ conf.set(CommonConfigurationKeys.IPC_PING_INTERVAL_KEY, "5000");
+ fs = DFS_CLUSTER_RULE.getDfscluster().getFileSystem();
+ assert fs.mkdirs(locksDir);
+ }
+
+ @After
+ public void teardownClass() throws IOException {
+ fs.delete(locksDir, true);
+ fs.close();
}
- for (DirLockingThread thd : result) {
- thd.start();
+ @Test
+ public void testBasicLocking() throws Exception {
+ // 1 grab lock
+ DirLock lock = DirLock.tryLock(fs, locksDir);
+ Assert.assertTrue(fs.exists(lock.getLockFile()));
+
+ // 2 try to grab another lock while dir is locked
+ DirLock lock2 = DirLock.tryLock(fs, locksDir); // should fail
+ Assert.assertNull(lock2);
+
+ // 3 let go first lock
+ lock.release();
+ Assert.assertFalse(fs.exists(lock.getLockFile()));
+
+ // 4 try locking again
+ lock2 = DirLock.tryLock(fs, locksDir);
+ Assert.assertTrue(fs.exists(lock2.getLockFile()));
+ lock2.release();
+ Assert.assertFalse(fs.exists(lock.getLockFile()));
+ lock2.release(); // should be throw
}
- return result;
- }
- @Test
- public void testLockRecovery() throws Exception {
- DirLock lock1 = DirLock.tryLock(fs, locksDir); // should pass
- Assert.assertNotNull(lock1);
+ @Test
+ public void testConcurrentLocking() throws Exception {
+ DirLockingThread[] threads = null;
+ try {
+ threads = startThreads(100, locksDir);
+ for (DirLockingThread thd : threads) {
+ thd.join(30_000);
+ Assert.assertTrue(thd.getName() + " did not exit cleanly", thd.cleanExit);
+ }
- DirLock lock2 = DirLock.takeOwnershipIfStale(fs, locksDir, LOCK_EXPIRY_SEC); // should fail
- Assert.assertNull(lock2);
+ Path lockFile = new Path(locksDir + Path.SEPARATOR + DirLock.DIR_LOCK_FILE);
+ Assert.assertFalse(fs.exists(lockFile));
+ } finally {
+ if (threads != null) {
+ for (DirLockingThread thread : threads) {
+ thread.interrupt();
+ thread.join(30_000);
+ if (thread.isAlive()) {
+ throw new RuntimeException("Failed to stop threads within 30 seconds, threads may leak into other tests");
+ }
+ }
+ }
+ }
+ }
- Thread.sleep(LOCK_EXPIRY_SEC*1000 + 500); // wait for lock to expire
- Assert.assertTrue(fs.exists(lock1.getLockFile()));
+ private DirLockingThread[] startThreads(int thdCount, Path dir)
+ throws IOException {
+ DirLockingThread[] result = new DirLockingThread[thdCount];
+ for (int i = 0; i < thdCount; i++) {
+ result[i] = new DirLockingThread(i, fs, dir);
+ }
+
+ for (DirLockingThread thd : result) {
+ thd.start();
+ }
+ return result;
+ }
- DirLock lock3 = DirLock.takeOwnershipIfStale(fs, locksDir, LOCK_EXPIRY_SEC); // should pass now
- Assert.assertNotNull(lock3);
- Assert.assertTrue(fs.exists(lock3.getLockFile()));
- lock3.release();
- Assert.assertFalse(fs.exists(lock3.getLockFile()));
- lock1.release(); // should not throw
- }
+ @Test
+ public void testLockRecovery() throws Exception {
+ DirLock lock1 = DirLock.tryLock(fs, locksDir); // should pass
+ Assert.assertNotNull(lock1);
- class DirLockingThread extends Thread {
+ DirLock lock2 = DirLock.takeOwnershipIfStale(fs, locksDir, LOCK_EXPIRY_SEC); // should fail
+ Assert.assertNull(lock2);
- private int thdNum;
- private final FileSystem fs;
- private final Path dir;
- public boolean cleanExit = false;
+ Thread.sleep(LOCK_EXPIRY_SEC * 1000 + 500); // wait for lock to expire
+ Assert.assertTrue(fs.exists(lock1.getLockFile()));
- public DirLockingThread(int thdNum,FileSystem fs, Path dir)
- throws IOException {
- this.thdNum = thdNum;
- this.fs = fs;
- this.dir = dir;
+ DirLock lock3 = DirLock.takeOwnershipIfStale(fs, locksDir, LOCK_EXPIRY_SEC); // should pass now
+ Assert.assertNotNull(lock3);
+ Assert.assertTrue(fs.exists(lock3.getLockFile()));
+ lock3.release();
+ Assert.assertFalse(fs.exists(lock3.getLockFile()));
+ lock1.release(); // should not throw
}
- @Override
- public void run() {
- Thread.currentThread().setName("DirLockingThread-" + thdNum);
- DirLock lock = null;
- try {
- do {
- System.err.println("Trying lock " + getName());
- lock = DirLock.tryLock(fs, dir);
- System.err.println("Acquired lock " + getName());
- if(lock==null) {
- System.out.println("Retrying lock - " + getName());
- }
- } while (lock==null);
- cleanExit= true;
- } catch (Exception e) {
- e.printStackTrace();
- }
- finally {
- try {
- if(lock!=null) {
- lock.release();
- System.err.println("Released lock " + getName());
+ class DirLockingThread extends Thread {
+
+ private int thdNum;
+ private final FileSystem fs;
+ private final Path dir;
+ public boolean cleanExit = false;
+
+ public DirLockingThread(int thdNum, FileSystem fs, Path dir)
+ throws IOException {
+ this.thdNum = thdNum;
+ this.fs = fs;
+ this.dir = dir;
+ }
+
+ @Override
+ public void run() {
+ Thread.currentThread().setName("DirLockingThread-" + thdNum);
+ DirLock lock = null;
+ try {
+ do {
+ System.err.println("Trying lock " + getName());
+ lock = DirLock.tryLock(fs, dir);
+ System.err.println("Acquired lock " + getName());
+ if (lock == null) {
+ System.out.println("Retrying lock - " + getName());
+ }
+ } while (lock == null && !Thread.currentThread().isInterrupted());
+ cleanExit = true;
+ } catch (IOException e) {
+ e.printStackTrace();
+ } finally {
+ try {
+ if (lock != null) {
+ lock.release();
+ System.err.println("Released lock " + getName());
+ }
+ } catch (IOException e) {
+ e.printStackTrace(System.err);
+ }
}
- } catch (IOException e) {
- e.printStackTrace(System.err);
- }
- }
- System.err.println("Thread exiting " + getName());
- } // run()
-
- } // class DirLockingThread
+ System.err.println("Thread exiting " + getName());
+ } // run()
+
+ } // class DirLockingThread
}
http://git-wip-us.apache.org/repos/asf/storm/blob/83adfd9d/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/spout/TestFileLock.java
----------------------------------------------------------------------
diff --git a/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/spout/TestFileLock.java b/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/spout/TestFileLock.java
index 725fa11..df131a4 100644
--- a/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/spout/TestFileLock.java
+++ b/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/spout/TestFileLock.java
@@ -1,3 +1,4 @@
+
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
@@ -9,388 +10,380 @@
* <p/>
* http://www.apache.org/licenses/LICENSE-2.0
* <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
package org.apache.storm.hdfs.spout;
-
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.HdfsConfiguration;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.storm.hdfs.common.HdfsUtils;
import org.junit.After;
-import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.Before;
-import org.junit.BeforeClass;
import org.junit.Test;
-
import java.io.BufferedReader;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStreamReader;
import java.lang.reflect.Method;
import java.util.ArrayList;
+import org.apache.storm.hdfs.testing.MiniDFSClusterRule;
+import org.junit.Rule;
public class TestFileLock {
- static MiniDFSCluster.Builder builder;
- static MiniDFSCluster hdfsCluster;
- static FileSystem fs;
- static String hdfsURI;
- static HdfsConfiguration conf = new HdfsConfiguration();
-
- private Path filesDir = new Path("/tmp/filesdir");
- private Path locksDir = new Path("/tmp/locskdir");
-
- @BeforeClass
- public static void setupClass() throws IOException {
- conf.set(CommonConfigurationKeys.IPC_PING_INTERVAL_KEY,"5000");
- builder = new MiniDFSCluster.Builder(new Configuration());
- hdfsCluster = builder.build();
- fs = hdfsCluster.getFileSystem();
- hdfsURI = "hdfs://localhost:" + hdfsCluster.getNameNodePort() + "/";
- }
-
- @AfterClass
- public static void teardownClass() throws IOException {
- fs.close();
- hdfsCluster.shutdown();
- }
-
- @Before
- public void setUp() throws Exception {
- assert fs.mkdirs(filesDir) ;
- assert fs.mkdirs(locksDir) ;
- }
-
- @After
- public void tearDown() throws Exception {
- fs.delete(filesDir, true);
- fs.delete(locksDir, true);
- }
-
- @Test
- public void testBasicLocking() throws Exception {
- // create empty files in filesDir
- Path file1 = new Path(filesDir + Path.SEPARATOR + "file1");
- Path file2 = new Path(filesDir + Path.SEPARATOR + "file2");
- fs.create(file1).close();
- fs.create(file2).close(); // create empty file
-
- // acquire lock on file1 and verify if worked
- FileLock lock1a = FileLock.tryLock(fs, file1, locksDir, "spout1");
- Assert.assertNotNull(lock1a);
- Assert.assertTrue(fs.exists(lock1a.getLockFile()));
- Assert.assertEquals(lock1a.getLockFile().getParent(), locksDir); // verify lock file location
- Assert.assertEquals(lock1a.getLockFile().getName(), file1.getName()); // verify lock filename
-
- // acquire another lock on file1 and verify it failed
- FileLock lock1b = FileLock.tryLock(fs, file1, locksDir, "spout1");
- Assert.assertNull(lock1b);
-
- // release lock on file1 and check
- lock1a.release();
- Assert.assertFalse(fs.exists(lock1a.getLockFile()));
-
- // Retry locking and verify
- FileLock lock1c = FileLock.tryLock(fs, file1, locksDir, "spout1");
- Assert.assertNotNull(lock1c);
- Assert.assertTrue(fs.exists(lock1c.getLockFile()));
- Assert.assertEquals(lock1c.getLockFile().getParent(), locksDir); // verify lock file location
- Assert.assertEquals(lock1c.getLockFile().getName(), file1.getName()); // verify lock filename
-
- // try locking another file2 at the same time
- FileLock lock2a = FileLock.tryLock(fs, file2, locksDir, "spout1");
- Assert.assertNotNull(lock2a);
- Assert.assertTrue(fs.exists(lock2a.getLockFile()));
- Assert.assertEquals(lock2a.getLockFile().getParent(), locksDir); // verify lock file location
- Assert.assertEquals(lock2a.getLockFile().getName(), file2.getName()); // verify lock filename
-
- // release both locks
- lock2a.release();
- Assert.assertFalse(fs.exists(lock2a.getLockFile()));
- lock1c.release();
- Assert.assertFalse(fs.exists(lock1c.getLockFile()));
- }
-
- @Test
- public void testHeartbeat() throws Exception {
- Path file1 = new Path(filesDir + Path.SEPARATOR + "file1");
- fs.create(file1).close();
-
- // acquire lock on file1
- FileLock lock1 = FileLock.tryLock(fs, file1, locksDir, "spout1");
- Assert.assertNotNull(lock1);
- Assert.assertTrue(fs.exists(lock1.getLockFile()));
-
- ArrayList<String> lines = readTextFile(lock1.getLockFile());
- Assert.assertEquals("heartbeats appear to be missing", 1, lines.size());
-
- // hearbeat upon it
- lock1.heartbeat("1");
- lock1.heartbeat("2");
- lock1.heartbeat("3");
-
- lines = readTextFile(lock1.getLockFile());
- Assert.assertEquals("heartbeats appear to be missing", 4, lines.size());
-
- lock1.heartbeat("4");
- lock1.heartbeat("5");
- lock1.heartbeat("6");
-
- lines = readTextFile(lock1.getLockFile());
- Assert.assertEquals("heartbeats appear to be missing", 7, lines.size());
-
- lock1.release();
- lines = readTextFile(lock1.getLockFile());
- Assert.assertNull(lines);
- Assert.assertFalse(fs.exists(lock1.getLockFile()));
- }
-
- @Test
- public void testConcurrentLocking() throws IOException, InterruptedException {
- Path file1 = new Path(filesDir + Path.SEPARATOR + "file1");
- fs.create(file1).close();
-
- FileLockingThread[] thds = startThreads(100, file1, locksDir);
- for (FileLockingThread thd : thds) {
- thd.join();
- if( !thd.cleanExit) {
- System.err.println(thd.getName() + " did not exit cleanly");
- }
- Assert.assertTrue(thd.cleanExit);
+ @Rule
+ public MiniDFSClusterRule dfsClusterRule = new MiniDFSClusterRule();
+
+ private FileSystem fs;
+ private HdfsConfiguration conf = new HdfsConfiguration();
+
+ private final Path filesDir = new Path("/tmp/filesdir");
+ private final Path locksDir = new Path("/tmp/locksdir");
+
+ @Before
+ public void setup() throws IOException {
+ conf.set(CommonConfigurationKeys.IPC_PING_INTERVAL_KEY, "5000");
+ fs = dfsClusterRule.getDfscluster().getFileSystem();
+ assert fs.mkdirs(filesDir);
+ assert fs.mkdirs(locksDir);
}
- Path lockFile = new Path(locksDir + Path.SEPARATOR + file1.getName());
- Assert.assertFalse(fs.exists(lockFile));
- }
+ @After
+ public void teardown() throws IOException {
+ fs.delete(filesDir, true);
+ fs.delete(locksDir, true);
+ fs.close();
+ }
- private FileLockingThread[] startThreads(int thdCount, Path fileToLock, Path locksDir)
- throws IOException {
- FileLockingThread[] result = new FileLockingThread[thdCount];
- for (int i = 0; i < thdCount; i++) {
- result[i] = new FileLockingThread(i, fs, fileToLock, locksDir, "spout" + Integer.toString(i));
+ @Test
+ public void testBasicLocking() throws Exception {
+ // create empty files in filesDir
+ Path file1 = new Path(filesDir + Path.SEPARATOR + "file1");
+ Path file2 = new Path(filesDir + Path.SEPARATOR + "file2");
+ fs.create(file1).close();
+ fs.create(file2).close(); // create empty file
+
+ // acquire lock on file1 and verify if worked
+ FileLock lock1a = FileLock.tryLock(fs, file1, locksDir, "spout1");
+ Assert.assertNotNull(lock1a);
+ Assert.assertTrue(fs.exists(lock1a.getLockFile()));
+ Assert.assertEquals(lock1a.getLockFile().getParent(), locksDir); // verify lock file location
+ Assert.assertEquals(lock1a.getLockFile().getName(), file1.getName()); // verify lock filename
+
+ // acquire another lock on file1 and verify it failed
+ FileLock lock1b = FileLock.tryLock(fs, file1, locksDir, "spout1");
+ Assert.assertNull(lock1b);
+
+ // release lock on file1 and check
+ lock1a.release();
+ Assert.assertFalse(fs.exists(lock1a.getLockFile()));
+
+ // Retry locking and verify
+ FileLock lock1c = FileLock.tryLock(fs, file1, locksDir, "spout1");
+ Assert.assertNotNull(lock1c);
+ Assert.assertTrue(fs.exists(lock1c.getLockFile()));
+ Assert.assertEquals(lock1c.getLockFile().getParent(), locksDir); // verify lock file location
+ Assert.assertEquals(lock1c.getLockFile().getName(), file1.getName()); // verify lock filename
+
+ // try locking another file2 at the same time
+ FileLock lock2a = FileLock.tryLock(fs, file2, locksDir, "spout1");
+ Assert.assertNotNull(lock2a);
+ Assert.assertTrue(fs.exists(lock2a.getLockFile()));
+ Assert.assertEquals(lock2a.getLockFile().getParent(), locksDir); // verify lock file location
+ Assert.assertEquals(lock2a.getLockFile().getName(), file2.getName()); // verify lock filename
+
+ // release both locks
+ lock2a.release();
+ Assert.assertFalse(fs.exists(lock2a.getLockFile()));
+ lock1c.release();
+ Assert.assertFalse(fs.exists(lock1c.getLockFile()));
}
- for (FileLockingThread thd : result) {
- thd.start();
+ @Test
+ public void testHeartbeat() throws Exception {
+ Path file1 = new Path(filesDir + Path.SEPARATOR + "file1");
+ fs.create(file1).close();
+
+ // acquire lock on file1
+ FileLock lock1 = FileLock.tryLock(fs, file1, locksDir, "spout1");
+ Assert.assertNotNull(lock1);
+ Assert.assertTrue(fs.exists(lock1.getLockFile()));
+
+ ArrayList<String> lines = readTextFile(lock1.getLockFile());
+ Assert.assertEquals("heartbeats appear to be missing", 1, lines.size());
+
+ // hearbeat upon it
+ lock1.heartbeat("1");
+ lock1.heartbeat("2");
+ lock1.heartbeat("3");
+
+ lines = readTextFile(lock1.getLockFile());
+ Assert.assertEquals("heartbeats appear to be missing", 4, lines.size());
+
+ lock1.heartbeat("4");
+ lock1.heartbeat("5");
+ lock1.heartbeat("6");
+
+ lines = readTextFile(lock1.getLockFile());
+ Assert.assertEquals("heartbeats appear to be missing", 7, lines.size());
+
+ lock1.release();
+ lines = readTextFile(lock1.getLockFile());
+ Assert.assertNull(lines);
+ Assert.assertFalse(fs.exists(lock1.getLockFile()));
}
- return result;
- }
-
-
- @Test
- public void testStaleLockDetection_SingleLock() throws Exception {
- final int LOCK_EXPIRY_SEC = 1;
- final int WAIT_MSEC = 1500;
- Path file1 = new Path(filesDir + Path.SEPARATOR + "file1");
- fs.create(file1).close();
- FileLock lock1 = FileLock.tryLock(fs, file1, locksDir, "spout1");
- try {
- // acquire lock on file1
- Assert.assertNotNull(lock1);
- Assert.assertTrue(fs.exists(lock1.getLockFile()));
- Thread.sleep(WAIT_MSEC); // wait for lock to expire
- HdfsUtils.Pair<Path, FileLock.LogEntry> expired = FileLock.locateOldestExpiredLock(fs, locksDir, LOCK_EXPIRY_SEC);
- Assert.assertNotNull(expired);
-
- // heartbeat, ensure its no longer stale and read back the heartbeat data
- lock1.heartbeat("1");
- expired = FileLock.locateOldestExpiredLock(fs, locksDir, 1);
- Assert.assertNull(expired);
-
- FileLock.LogEntry lastEntry = lock1.getLastLogEntry();
- Assert.assertNotNull(lastEntry);
- Assert.assertEquals("1", lastEntry.fileOffset);
-
- // wait and check for expiry again
- Thread.sleep(WAIT_MSEC);
- expired = FileLock.locateOldestExpiredLock(fs, locksDir, LOCK_EXPIRY_SEC);
- Assert.assertNotNull(expired);
- } finally {
- lock1.release();
- fs.delete(file1, false);
+
+ @Test
+ public void testConcurrentLocking() throws IOException, InterruptedException {
+ Path file1 = new Path(filesDir + Path.SEPARATOR + "file1");
+ fs.create(file1).close();
+
+ FileLockingThread[] threads = null;
+ try {
+ threads = startThreads(100, file1, locksDir);
+ for (FileLockingThread thd : threads) {
+ thd.join(30_000);
+ Assert.assertTrue(thd.getName() + " did not exit cleanly", thd.cleanExit);
+ }
+
+ Path lockFile = new Path(locksDir + Path.SEPARATOR + file1.getName());
+ Assert.assertFalse(fs.exists(lockFile));
+ } finally {
+ if (threads != null) {
+ for (FileLockingThread thread : threads) {
+ thread.interrupt();
+ thread.join(30_000);
+ if (thread.isAlive()) {
+ throw new RuntimeException("Failed to stop threads within 30 seconds, threads may leak into other tests");
+ }
+ }
+ }
+ }
}
- }
-
- @Test
- public void testStaleLockDetection_MultipleLocks() throws Exception {
- final int LOCK_EXPIRY_SEC = 1;
- final int WAIT_MSEC = 1500;
- Path file1 = new Path(filesDir + Path.SEPARATOR + "file1");
- Path file2 = new Path(filesDir + Path.SEPARATOR + "file2");
- Path file3 = new Path(filesDir + Path.SEPARATOR + "file3");
-
- fs.create(file1).close();
- fs.create(file2).close();
- fs.create(file3).close();
-
- // 1) acquire locks on file1,file2,file3
- FileLock lock1 = FileLock.tryLock(fs, file1, locksDir, "spout1");
- FileLock lock2 = FileLock.tryLock(fs, file2, locksDir, "spout2");
- FileLock lock3 = FileLock.tryLock(fs, file3, locksDir, "spout3");
- Assert.assertNotNull(lock1);
- Assert.assertNotNull(lock2);
- Assert.assertNotNull(lock3);
-
- try {
- HdfsUtils.Pair<Path, FileLock.LogEntry> expired = FileLock.locateOldestExpiredLock(fs, locksDir, LOCK_EXPIRY_SEC);
- Assert.assertNull(expired);
-
- // 2) wait for all 3 locks to expire then heart beat on 2 locks and verify stale lock
- Thread.sleep(WAIT_MSEC);
- lock1.heartbeat("1");
- lock2.heartbeat("1");
-
- expired = FileLock.locateOldestExpiredLock(fs, locksDir, LOCK_EXPIRY_SEC);
- Assert.assertNotNull(expired);
- Assert.assertEquals("spout3", expired.getValue().componentID);
- } finally {
- lock1.release();
- lock2.release();
- lock3.release();
- fs.delete(file1, false);
- fs.delete(file2, false);
- fs.delete(file3, false);
+
+ private FileLockingThread[] startThreads(int thdCount, Path fileToLock, Path locksDir)
+ throws IOException {
+ FileLockingThread[] result = new FileLockingThread[thdCount];
+ for (int i = 0; i < thdCount; i++) {
+ result[i] = new FileLockingThread(i, fs, fileToLock, locksDir, "spout" + Integer.toString(i));
+ }
+
+ for (FileLockingThread thd : result) {
+ thd.start();
+ }
+ return result;
}
- }
-
- @Test
- public void testLockRecovery() throws Exception {
- final int LOCK_EXPIRY_SEC = 1;
- final int WAIT_MSEC = LOCK_EXPIRY_SEC*1000 + 500;
- Path file1 = new Path(filesDir + Path.SEPARATOR + "file1");
- Path file2 = new Path(filesDir + Path.SEPARATOR + "file2");
- Path file3 = new Path(filesDir + Path.SEPARATOR + "file3");
-
- fs.create(file1).close();
- fs.create(file2).close();
- fs.create(file3).close();
-
- // 1) acquire locks on file1,file2,file3
- FileLock lock1 = FileLock.tryLock(fs, file1, locksDir, "spout1");
- FileLock lock2 = FileLock.tryLock(fs, file2, locksDir, "spout2");
- FileLock lock3 = FileLock.tryLock(fs, file3, locksDir, "spout3");
- Assert.assertNotNull(lock1);
- Assert.assertNotNull(lock2);
- Assert.assertNotNull(lock3);
-
- try {
- HdfsUtils.Pair<Path, FileLock.LogEntry> expired = FileLock.locateOldestExpiredLock(fs, locksDir, LOCK_EXPIRY_SEC);
- Assert.assertNull(expired);
-
- // 1) Simulate lock file lease expiring and getting closed by HDFS
- closeUnderlyingLockFile(lock3);
-
- // 2) wait for all 3 locks to expire then heart beat on 2 locks
- Thread.sleep(WAIT_MSEC*2); // wait for locks to expire
- lock1.heartbeat("1");
- lock2.heartbeat("1");
-
- // 3) Take ownership of stale lock
- FileLock lock3b = FileLock.acquireOldestExpiredLock(fs, locksDir, LOCK_EXPIRY_SEC, "spout1");
- Assert.assertNotNull(lock3b);
- Assert.assertEquals("Expected lock3 file", Path.getPathWithoutSchemeAndAuthority(lock3b.getLockFile()), lock3.getLockFile());
- } finally {
- lock1.release();
- lock2.release();
- lock3.release();
- fs.delete(file1, false);
- fs.delete(file2, false);
- try {
- fs.delete(file3, false);
- } catch (Exception e) {
- e.printStackTrace();
- }
+
+ @Test
+ public void testStaleLockDetection_SingleLock() throws Exception {
+ final int LOCK_EXPIRY_SEC = 1;
+ final int WAIT_MSEC = 1500;
+ Path file1 = new Path(filesDir + Path.SEPARATOR + "file1");
+ fs.create(file1).close();
+ FileLock lock1 = FileLock.tryLock(fs, file1, locksDir, "spout1");
+ try {
+ // acquire lock on file1
+ Assert.assertNotNull(lock1);
+ Assert.assertTrue(fs.exists(lock1.getLockFile()));
+ Thread.sleep(WAIT_MSEC); // wait for lock to expire
+ HdfsUtils.Pair<Path, FileLock.LogEntry> expired = FileLock.locateOldestExpiredLock(fs, locksDir, LOCK_EXPIRY_SEC);
+ Assert.assertNotNull(expired);
+
+ // heartbeat, ensure its no longer stale and read back the heartbeat data
+ lock1.heartbeat("1");
+ expired = FileLock.locateOldestExpiredLock(fs, locksDir, 1);
+ Assert.assertNull(expired);
+
+ FileLock.LogEntry lastEntry = lock1.getLastLogEntry();
+ Assert.assertNotNull(lastEntry);
+ Assert.assertEquals("1", lastEntry.fileOffset);
+
+ // wait and check for expiry again
+ Thread.sleep(WAIT_MSEC);
+ expired = FileLock.locateOldestExpiredLock(fs, locksDir, LOCK_EXPIRY_SEC);
+ Assert.assertNotNull(expired);
+ } finally {
+ lock1.release();
+ fs.delete(file1, false);
+ }
}
- }
-
- public static void closeUnderlyingLockFile(FileLock lock) throws ReflectiveOperationException {
- Method m = FileLock.class.getDeclaredMethod("forceCloseLockFile");
- m.setAccessible(true);
- m.invoke(lock);
- }
-
- /** return null if file not found */
- private ArrayList<String> readTextFile(Path file) throws IOException {
- FSDataInputStream os = null;
- try {
- os = fs.open(file);
- if (os == null) {
- return null;
- }
- BufferedReader reader = new BufferedReader(new InputStreamReader(os));
- ArrayList<String> lines = new ArrayList<>();
- for (String line = reader.readLine(); line != null; line = reader.readLine()) {
- lines.add(line);
- }
- return lines;
- } catch( FileNotFoundException e) {
- return null;
- } finally {
- if(os!=null) {
- os.close();
- }
+
+ @Test
+ public void testStaleLockDetection_MultipleLocks() throws Exception {
+ final int LOCK_EXPIRY_SEC = 1;
+ final int WAIT_MSEC = 1500;
+ Path file1 = new Path(filesDir + Path.SEPARATOR + "file1");
+ Path file2 = new Path(filesDir + Path.SEPARATOR + "file2");
+ Path file3 = new Path(filesDir + Path.SEPARATOR + "file3");
+
+ fs.create(file1).close();
+ fs.create(file2).close();
+ fs.create(file3).close();
+
+ // 1) acquire locks on file1,file2,file3
+ FileLock lock1 = FileLock.tryLock(fs, file1, locksDir, "spout1");
+ FileLock lock2 = FileLock.tryLock(fs, file2, locksDir, "spout2");
+ FileLock lock3 = FileLock.tryLock(fs, file3, locksDir, "spout3");
+ Assert.assertNotNull(lock1);
+ Assert.assertNotNull(lock2);
+ Assert.assertNotNull(lock3);
+
+ try {
+ HdfsUtils.Pair<Path, FileLock.LogEntry> expired = FileLock.locateOldestExpiredLock(fs, locksDir, LOCK_EXPIRY_SEC);
+ Assert.assertNull(expired);
+
+ // 2) wait for all 3 locks to expire then heart beat on 2 locks and verify stale lock
+ Thread.sleep(WAIT_MSEC);
+ lock1.heartbeat("1");
+ lock2.heartbeat("1");
+
+ expired = FileLock.locateOldestExpiredLock(fs, locksDir, LOCK_EXPIRY_SEC);
+ Assert.assertNotNull(expired);
+ Assert.assertEquals("spout3", expired.getValue().componentID);
+ } finally {
+ lock1.release();
+ lock2.release();
+ lock3.release();
+ fs.delete(file1, false);
+ fs.delete(file2, false);
+ fs.delete(file3, false);
+ }
}
- }
- class FileLockingThread extends Thread {
+ @Test
+ public void testLockRecovery() throws Exception {
+ final int LOCK_EXPIRY_SEC = 1;
+ final int WAIT_MSEC = LOCK_EXPIRY_SEC * 1000 + 500;
+ Path file1 = new Path(filesDir + Path.SEPARATOR + "file1");
+ Path file2 = new Path(filesDir + Path.SEPARATOR + "file2");
+ Path file3 = new Path(filesDir + Path.SEPARATOR + "file3");
+
+ fs.create(file1).close();
+ fs.create(file2).close();
+ fs.create(file3).close();
+
+ // 1) acquire locks on file1,file2,file3
+ FileLock lock1 = FileLock.tryLock(fs, file1, locksDir, "spout1");
+ FileLock lock2 = FileLock.tryLock(fs, file2, locksDir, "spout2");
+ FileLock lock3 = FileLock.tryLock(fs, file3, locksDir, "spout3");
+ Assert.assertNotNull(lock1);
+ Assert.assertNotNull(lock2);
+ Assert.assertNotNull(lock3);
- private int thdNum;
- private final FileSystem fs;
- public boolean cleanExit = false;
- private Path fileToLock;
- private Path locksDir;
- private String spoutId;
+ try {
+ HdfsUtils.Pair<Path, FileLock.LogEntry> expired = FileLock.locateOldestExpiredLock(fs, locksDir, LOCK_EXPIRY_SEC);
+ Assert.assertNull(expired);
+
+ // 1) Simulate lock file lease expiring and getting closed by HDFS
+ closeUnderlyingLockFile(lock3);
+
+ // 2) wait for all 3 locks to expire then heart beat on 2 locks
+ Thread.sleep(WAIT_MSEC * 2); // wait for locks to expire
+ lock1.heartbeat("1");
+ lock2.heartbeat("1");
+
+ // 3) Take ownership of stale lock
+ FileLock lock3b = FileLock.acquireOldestExpiredLock(fs, locksDir, LOCK_EXPIRY_SEC, "spout1");
+ Assert.assertNotNull(lock3b);
+ Assert.assertEquals("Expected lock3 file", Path.getPathWithoutSchemeAndAuthority(lock3b.getLockFile()), lock3.getLockFile());
+ } finally {
+ lock1.release();
+ lock2.release();
+ lock3.release();
+ fs.delete(file1, false);
+ fs.delete(file2, false);
+ try {
+ fs.delete(file3, false);
+ } catch (Exception e) {
+ e.printStackTrace();
+ }
+ }
+ }
- public FileLockingThread(int thdNum, FileSystem fs, Path fileToLock, Path locksDir, String spoutId)
- throws IOException {
- this.thdNum = thdNum;
- this.fs = fs;
- this.fileToLock = fileToLock;
- this.locksDir = locksDir;
- this.spoutId = spoutId;
+ public static void closeUnderlyingLockFile(FileLock lock) throws ReflectiveOperationException {
+ Method m = FileLock.class.getDeclaredMethod("forceCloseLockFile");
+ m.setAccessible(true);
+ m.invoke(lock);
}
- @Override
- public void run() {
- Thread.currentThread().setName("FileLockingThread-" + thdNum);
- FileLock lock = null;
- try {
- do {
- System.err.println("Trying lock - " + getName());
- lock = FileLock.tryLock(fs, this.fileToLock, this.locksDir, spoutId);
- System.err.println("Acquired lock - " + getName());
- if(lock==null) {
- System.out.println("Retrying lock - " + getName());
- }
- } while (lock==null);
- cleanExit= true;
- } catch (Exception e) {
- e.printStackTrace();
- }
- finally {
+ /**
+ * return null if file not found
+ */
+ private ArrayList<String> readTextFile(Path file) throws IOException {
+ FSDataInputStream os = null;
try {
- if(lock!=null) {
- lock.release();
- System.err.println("Released lock - " + getName());
- }
- } catch (IOException e) {
- e.printStackTrace(System.err);
+ os = fs.open(file);
+ if (os == null) {
+ return null;
+ }
+ BufferedReader reader = new BufferedReader(new InputStreamReader(os));
+ ArrayList<String> lines = new ArrayList<>();
+ for (String line = reader.readLine(); line != null; line = reader.readLine()) {
+ lines.add(line);
+ }
+ return lines;
+ } catch (FileNotFoundException e) {
+ return null;
+ } finally {
+ if (os != null) {
+ os.close();
+ }
+ }
+ }
+
+ class FileLockingThread extends Thread {
+
+ private int thdNum;
+ private final FileSystem fs;
+ public boolean cleanExit = false;
+ private Path fileToLock;
+ private Path locksDir;
+ private String spoutId;
+
+ public FileLockingThread(int thdNum, FileSystem fs, Path fileToLock, Path locksDir, String spoutId)
+ throws IOException {
+ this.thdNum = thdNum;
+ this.fs = fs;
+ this.fileToLock = fileToLock;
+ this.locksDir = locksDir;
+ this.spoutId = spoutId;
}
- }
- System.err.println("Thread exiting - " + getName());
- } // run()
- } // class FileLockingThread
+ @Override
+ public void run() {
+ Thread.currentThread().setName("FileLockingThread-" + thdNum);
+ FileLock lock = null;
+ try {
+ do {
+ System.err.println("Trying lock - " + getName());
+ lock = FileLock.tryLock(fs, this.fileToLock, this.locksDir, spoutId);
+ System.err.println("Acquired lock - " + getName());
+ if (lock == null) {
+ System.out.println("Retrying lock - " + getName());
+ }
+ } while (lock == null && !Thread.currentThread().isInterrupted());
+ cleanExit = true;
+ } catch (Exception e) {
+ e.printStackTrace();
+ } finally {
+ try {
+ if (lock != null) {
+ lock.release();
+ System.err.println("Released lock - " + getName());
+ }
+ } catch (IOException e) {
+ e.printStackTrace(System.err);
+ }
+ }
+ System.err.println("Thread exiting - " + getName());
+ } // run()
+
+ } // class FileLockingThread
}
http://git-wip-us.apache.org/repos/asf/storm/blob/83adfd9d/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/spout/TestHdfsSemantics.java
----------------------------------------------------------------------
diff --git a/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/spout/TestHdfsSemantics.java b/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/spout/TestHdfsSemantics.java
index 6628cc9..33432d9 100644
--- a/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/spout/TestHdfsSemantics.java
+++ b/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/spout/TestHdfsSemantics.java
@@ -1,3 +1,4 @@
+
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
@@ -9,196 +10,192 @@
* <p/>
* http://www.apache.org/licenses/LICENSE-2.0
* <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
-
package org.apache.storm.hdfs.spout;
-import org.apache.hadoop.conf.Configuration;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.fail;
+
import org.apache.hadoop.fs.CommonConfigurationKeys;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileAlreadyExistsException;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.HdfsConfiguration;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.protocol.AlreadyBeingCreatedException;
import org.apache.hadoop.ipc.RemoteException;
import org.junit.After;
-import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.Before;
-import org.junit.BeforeClass;
import org.junit.Test;
import java.io.IOException;
+import org.apache.storm.hdfs.testing.MiniDFSClusterRule;
+import org.junit.Rule;
public class TestHdfsSemantics {
- static MiniDFSCluster.Builder builder;
- static MiniDFSCluster hdfsCluster;
- static FileSystem fs;
- static String hdfsURI;
- static HdfsConfiguration conf = new HdfsConfiguration();
-
- private Path dir = new Path("/tmp/filesdir");
-
- @BeforeClass
- public static void setupClass() throws IOException {
- conf.set(CommonConfigurationKeys.IPC_PING_INTERVAL_KEY,"5000");
- builder = new MiniDFSCluster.Builder(new Configuration());
- hdfsCluster = builder.build();
- fs = hdfsCluster.getFileSystem();
- hdfsURI = "hdfs://localhost:" + hdfsCluster.getNameNodePort() + "/";
- }
-
- @AfterClass
- public static void teardownClass() throws IOException {
- fs.close();
- hdfsCluster.shutdown();
- }
-
- @Before
- public void setUp() throws Exception {
- assert fs.mkdirs(dir) ;
- }
-
- @After
- public void tearDown() throws Exception {
- fs.delete(dir, true);
- }
-
-
- @Test
- public void testDeleteSemantics() throws Exception {
- Path file = new Path(dir.toString() + Path.SEPARATOR_CHAR + "file1");
-// try {
- // 1) Delete absent file - should return false
- Assert.assertFalse(fs.exists(file));
- try {
- Assert.assertFalse(fs.delete(file, false));
- } catch (IOException e) {
- e.printStackTrace();
- }
+ @Rule
+ public MiniDFSClusterRule dfsClusterRule = new MiniDFSClusterRule();
- // 2) deleting open file - should return true
- fs.create(file, false);
- Assert.assertTrue(fs.delete(file, false));
-
- // 3) deleting closed file - should return true
- FSDataOutputStream os = fs.create(file, false);
- os.close();
- Assert.assertTrue(fs.exists(file));
- Assert.assertTrue(fs.delete(file, false));
- Assert.assertFalse(fs.exists(file));
- }
-
- @Test
- public void testConcurrentDeletion() throws Exception {
- Path file = new Path(dir.toString() + Path.SEPARATOR_CHAR + "file1");
- fs.create(file).close();
- // 1 concurrent deletion - only one thread should succeed
- FileDeletionThread[] thds = startThreads(10, file);
- int successCount=0;
- for (FileDeletionThread thd : thds) {
- thd.join();
- if( thd.succeeded)
- successCount++;
- if(thd.exception!=null)
- Assert.assertNotNull(thd.exception);
- }
- System.err.println(successCount);
- Assert.assertEquals(1, successCount);
-
- }
-
- @Test
- public void testAppendSemantics() throws Exception {
- //1 try to append to an open file
- Path file1 = new Path(dir.toString() + Path.SEPARATOR_CHAR + "file1");
- FSDataOutputStream os1 = fs.create(file1, false);
- try {
- fs.append(file1); // should fail
- Assert.assertTrue("Append did not throw an exception", false);
- } catch (RemoteException e) {
- // expecting AlreadyBeingCreatedException inside RemoteException
- Assert.assertEquals(AlreadyBeingCreatedException.class, e.unwrapRemoteException().getClass());
+ private FileSystem fs;
+ private final HdfsConfiguration conf = new HdfsConfiguration();
+
+ private final Path dir = new Path("/tmp/filesdir");
+
+ @Before
+ public void setup() throws IOException {
+ conf.set(CommonConfigurationKeys.IPC_PING_INTERVAL_KEY, "5000");
+ fs = dfsClusterRule.getDfscluster().getFileSystem();
+ assert fs.mkdirs(dir);
}
- //2 try to append to a closed file
- os1.close();
- FSDataOutputStream os2 = fs.append(file1); // should pass
- os2.close();
- }
-
- @Test
- public void testDoubleCreateSemantics() throws Exception {
- //1 create an already existing open file w/o override flag
- Path file1 = new Path(dir.toString() + Path.SEPARATOR_CHAR + "file1");
- FSDataOutputStream os1 = fs.create(file1, false);
- try {
- fs.create(file1, false); // should fail
- Assert.assertTrue("Create did not throw an exception", false);
- } catch (RemoteException e) {
- Assert.assertEquals(AlreadyBeingCreatedException.class, e.unwrapRemoteException().getClass());
+ @After
+ public void teardown() throws IOException {
+ fs.delete(dir, true);
+ fs.close();
}
- //2 close file and retry creation
- os1.close();
- try {
- fs.create(file1, false); // should still fail
- } catch (FileAlreadyExistsException e) {
- // expecting this exception
+
+ @Test
+ public void testDeleteSemantics() throws Exception {
+ Path file = new Path(dir.toString() + Path.SEPARATOR_CHAR + "file1");
+// try {
+ // 1) Delete absent file - should return false
+ Assert.assertFalse(fs.exists(file));
+ try {
+ Assert.assertFalse(fs.delete(file, false));
+ } catch (IOException e) {
+ e.printStackTrace();
+ }
+
+ // 2) deleting open file - should return true
+ fs.create(file, false);
+ Assert.assertTrue(fs.delete(file, false));
+
+ // 3) deleting closed file - should return true
+ FSDataOutputStream os = fs.create(file, false);
+ os.close();
+ Assert.assertTrue(fs.exists(file));
+ Assert.assertTrue(fs.delete(file, false));
+ Assert.assertFalse(fs.exists(file));
}
- //3 delete file and retry creation
- fs.delete(file1, false);
- FSDataOutputStream os2 = fs.create(file1, false); // should pass
- Assert.assertNotNull(os2);
- os2.close();
- }
+ @Test
+ public void testConcurrentDeletion() throws Exception {
+ Path file = new Path(dir.toString() + Path.SEPARATOR_CHAR + "file1");
+ fs.create(file).close();
+ // 1 concurrent deletion - only one thread should succeed
+ FileDeletionThread[] threads = null;
+ try {
+ threads = startThreads(10, file);
+ int successCount = 0;
+ for (FileDeletionThread thd : threads) {
+ thd.join(30_000);
+ if (thd.succeeded) {
+ successCount++;
+ }
+ if (thd.exception != null) {
+ Assert.assertNotNull(thd.exception);
+ }
+ }
+ System.err.println(successCount);
+ Assert.assertEquals(1, successCount);
+ } finally {
+ if (threads != null) {
+ for (FileDeletionThread thread : threads) {
+ thread.interrupt();
+ thread.join(30_000);
+ if (thread.isAlive()) {
+ throw new RuntimeException("Failed to stop threads within 30 seconds, threads may leak into other tests");
+ }
+ }
+ }
+ }
+ }
+ @Test
+ public void testAppendSemantics() throws Exception {
+ //1 try to append to an open file
+ Path file1 = new Path(dir.toString() + Path.SEPARATOR_CHAR + "file1");
+ try (FSDataOutputStream os1 = fs.create(file1, false)) {
+ fs.append(file1); // should fail
+ fail("Append did not throw an exception");
+ } catch (RemoteException e) {
+ // expecting AlreadyBeingCreatedException inside RemoteException
+ Assert.assertEquals(AlreadyBeingCreatedException.class, e.unwrapRemoteException().getClass());
+ }
+
+ //2 try to append to a closed file
+ try (FSDataOutputStream os2 = fs.append(file1)) {
+ assertNotNull(os2);
+ }
+ }
- private FileDeletionThread[] startThreads(int thdCount, Path file)
- throws IOException {
- FileDeletionThread[] result = new FileDeletionThread[thdCount];
- for (int i = 0; i < thdCount; i++) {
- result[i] = new FileDeletionThread(i, fs, file);
+ @Test
+ public void testDoubleCreateSemantics() throws Exception {
+ //1 create an already existing open file w/o override flag
+ Path file1 = new Path(dir.toString() + Path.SEPARATOR_CHAR + "file1");
+ try (FSDataOutputStream os1 = fs.create(file1, false)) {
+ fs.create(file1, false); // should fail
+ fail("Create did not throw an exception");
+ } catch (RemoteException e) {
+ Assert.assertEquals(AlreadyBeingCreatedException.class, e.unwrapRemoteException().getClass());
+ }
+ //2 close file and retry creation
+ try {
+ fs.create(file1, false); // should still fail
+ fail("Create did not throw an exception");
+ } catch (FileAlreadyExistsException e) {
+ // expecting this exception
+ }
+
+ //3 delete file and retry creation
+ fs.delete(file1, false);
+ try (FSDataOutputStream os2 = fs.create(file1, false)) {
+ Assert.assertNotNull(os2);
+ }
}
- for (FileDeletionThread thd : result) {
- thd.start();
+ private FileDeletionThread[] startThreads(int thdCount, Path file)
+ throws IOException {
+ FileDeletionThread[] result = new FileDeletionThread[thdCount];
+ for (int i = 0; i < thdCount; i++) {
+ result[i] = new FileDeletionThread(i, fs, file);
+ }
+
+ for (FileDeletionThread thd : result) {
+ thd.start();
+ }
+ return result;
}
- return result;
- }
- private static class FileDeletionThread extends Thread {
+ private static class FileDeletionThread extends Thread {
- private final int thdNum;
- private final FileSystem fs;
- private final Path file;
- public boolean succeeded;
- public Exception exception = null;
+ private final int thdNum;
+ private final FileSystem fs;
+ private final Path file;
+ public boolean succeeded;
+ public Exception exception = null;
- public FileDeletionThread(int thdNum, FileSystem fs, Path file)
+ public FileDeletionThread(int thdNum, FileSystem fs, Path file)
throws IOException {
- this.thdNum = thdNum;
- this.fs = fs;
- this.file = file;
- }
-
- @Override
- public void run() {
- Thread.currentThread().setName("FileDeletionThread-" + thdNum);
- try {
- succeeded = fs.delete(file, false);
- } catch (Exception e) {
- exception = e;
- }
- } // run()
-
- } // class FileLockingThread
+ this.thdNum = thdNum;
+ this.fs = fs;
+ this.file = file;
+ }
+
+ @Override
+ public void run() {
+ Thread.currentThread().setName("FileDeletionThread-" + thdNum);
+ try {
+ succeeded = fs.delete(file, false);
+ } catch (Exception e) {
+ exception = e;
+ }
+ } // run()
+
+ } // class FileLockingThread
}
[45/50] [abbrv] storm git commit: Merge branch 'STORM-2814-1.x' of
https://github.com/Ethanlm/storm into STORM-2814-1.x-merge
Posted by pt...@apache.org.
Merge branch 'STORM-2814-1.x' of https://github.com/Ethanlm/storm into STORM-2814-1.x-merge
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/3badc161
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/3badc161
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/3badc161
Branch: refs/heads/metrics_v2
Commit: 3badc161ea4e9d630207eddaf7eddd077e4674f4
Parents: 6d45875 6799c5a
Author: Jungtaek Lim <ka...@gmail.com>
Authored: Thu Nov 16 08:23:04 2017 +0900
Committer: Jungtaek Lim <ka...@gmail.com>
Committed: Thu Nov 16 08:23:04 2017 +0900
----------------------------------------------------------------------
storm-core/src/clj/org/apache/storm/daemon/logviewer.clj | 2 +-
storm-core/src/clj/org/apache/storm/ui/helpers.clj | 6 ++++--
2 files changed, 5 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
[46/50] [abbrv] storm git commit: STORM-2546: Fix storm-kafka-client
spout getting stuck when retriable offsets were deleted from the Kafka log
due to topic compaction
Posted by pt...@apache.org.
STORM-2546: Fix storm-kafka-client spout getting stuck when retriable offsets were deleted from the Kafka log due to topic compaction
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/d9fe768a
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/d9fe768a
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/d9fe768a
Branch: refs/heads/metrics_v2
Commit: d9fe768a9f7cb47bb2b75a26e81676fc69b1fde0
Parents: 6d45875
Author: Stig Rohde Døssing <sr...@apache.org>
Authored: Sat Sep 2 23:50:33 2017 +0200
Committer: Stig Rohde Døssing <sr...@apache.org>
Committed: Thu Nov 16 07:09:20 2017 +0100
----------------------------------------------------------------------
.../apache/storm/kafka/spout/KafkaSpout.java | 48 +++-
.../storm/kafka/spout/KafkaSpoutConfig.java | 21 ++
.../storm/kafka/spout/KafkaSpoutCommitTest.java | 113 ----------
.../storm/kafka/spout/KafkaSpoutConfigTest.java | 24 ++
.../storm/kafka/spout/KafkaSpoutEmitTest.java | 9 +-
.../KafkaSpoutLogCompactionSupportTest.java | 223 +++++++++++++++++++
.../spout/KafkaSpoutMessagingGuaranteeTest.java | 8 +-
.../kafka/spout/KafkaSpoutRetryLimitTest.java | 3 +-
.../SpoutWithMockedConsumerSetupHelper.java | 66 +++++-
.../kafka/spout/internal/OffsetManagerTest.java | 1 -
10 files changed, 378 insertions(+), 138 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/d9fe768a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpout.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpout.java b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpout.java
index 51d81c7..4cda53c 100644
--- a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpout.java
+++ b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpout.java
@@ -91,7 +91,7 @@ public class KafkaSpout<K, V> extends BaseRichSpout {
// Always empty if processing guarantee is none or at-most-once
private transient Set<KafkaSpoutMessageId> emitted;
// Records that have been polled and are queued to be emitted in the nextTuple() call. One record is emitted per nextTuple()
- private transient Iterator<ConsumerRecord<K, V>> waitingToEmit;
+ private transient Iterator<ConsumerRecord<K, V>> waitingToEmit;
// Triggers when a subscription should be refreshed
private transient Timer refreshSubscriptionTimer;
private transient TopologyContext context;
@@ -152,7 +152,7 @@ public class KafkaSpout<K, V> extends BaseRichSpout {
previousAssignment = partitions;
LOG.info("Partitions revoked. [consumer-group={}, consumer={}, topic-partitions={}]",
- kafkaSpoutConfig.getConsumerGroupId(), kafkaConsumer, partitions);
+ kafkaSpoutConfig.getConsumerGroupId(), kafkaConsumer, partitions);
if (isAtLeastOnceProcessing()) {
commitOffsetsForAckedTuples();
@@ -271,7 +271,7 @@ public class KafkaSpout<K, V> extends BaseRichSpout {
private Set<TopicPartition> poll() {
final int maxUncommittedOffsets = kafkaSpoutConfig.getMaxUncommittedOffsets();
-
+
if (waitingToEmit()) {
LOG.debug("Not polling. Tuples waiting to be emitted.");
return Collections.emptySet();
@@ -317,7 +317,7 @@ public class KafkaSpout<K, V> extends BaseRichSpout {
// ======== poll =========
private ConsumerRecords<K, V> pollKafkaBroker(Set<TopicPartition> pollablePartitions) {
- doSeekRetriableTopicPartitions(pollablePartitions);
+ final Map<TopicPartition, Long> retriableOffsets = doSeekRetriableTopicPartitions(pollablePartitions);
Set<TopicPartition> pausedPartitions = new HashSet<>(kafkaConsumer.assignment());
Iterator<TopicPartition> pausedIter = pausedPartitions.iterator();
while (pausedIter.hasNext()) {
@@ -328,6 +328,7 @@ public class KafkaSpout<K, V> extends BaseRichSpout {
try {
kafkaConsumer.pause(pausedPartitions);
final ConsumerRecords<K, V> consumerRecords = kafkaConsumer.poll(kafkaSpoutConfig.getPollTimeoutMs());
+ ackRetriableOffsetsIfCompactedAway(retriableOffsets, consumerRecords);
final int numPolledRecords = consumerRecords.count();
LOG.debug("Polled [{}] records from Kafka.",
numPolledRecords);
@@ -341,13 +342,42 @@ public class KafkaSpout<K, V> extends BaseRichSpout {
}
}
- private void doSeekRetriableTopicPartitions(Set<TopicPartition> pollablePartitions) {
+ private Map<TopicPartition, Long> doSeekRetriableTopicPartitions(Set<TopicPartition> pollablePartitions) {
final Map<TopicPartition, Long> retriableTopicPartitions = retryService.earliestRetriableOffsets();
-
+ for (TopicPartition tp : retriableTopicPartitions.keySet()) {
+ if (!pollablePartitions.contains(tp)) {
+ retriableTopicPartitions.remove(tp);
+ }
+ }
for (Entry<TopicPartition, Long> retriableTopicPartitionAndOffset : retriableTopicPartitions.entrySet()) {
- if (pollablePartitions.contains(retriableTopicPartitionAndOffset.getKey())) {
- //Seek directly to the earliest retriable message for each retriable topic partition
- kafkaConsumer.seek(retriableTopicPartitionAndOffset.getKey(), retriableTopicPartitionAndOffset.getValue());
+ //Seek directly to the earliest retriable message for each retriable topic partition
+ kafkaConsumer.seek(retriableTopicPartitionAndOffset.getKey(), retriableTopicPartitionAndOffset.getValue());
+ }
+ return retriableTopicPartitions;
+ }
+
+ private void ackRetriableOffsetsIfCompactedAway(Map<TopicPartition, Long> earliestRetriableOffsets,
+ ConsumerRecords<K, V> consumerRecords) {
+ for (Entry<TopicPartition, Long> entry : earliestRetriableOffsets.entrySet()) {
+ TopicPartition tp = entry.getKey();
+ List<ConsumerRecord<K, V>> records = consumerRecords.records(tp);
+ if (!records.isEmpty()) {
+ ConsumerRecord<K, V> record = records.get(0);
+ long seekOffset = entry.getValue();
+ long earliestReceivedOffset = record.offset();
+ if (seekOffset < earliestReceivedOffset) {
+ //Since we asked for tuples starting at seekOffset, some retriable records must have been compacted away.
+ //Ack up to the first offset received if the record is not already acked or currently in the topology
+ for (long i = seekOffset; i < earliestReceivedOffset; i++) {
+ KafkaSpoutMessageId msgId = retryService.getMessageId(new ConsumerRecord<>(tp.topic(), tp.partition(), i, null, null));
+ if (!offsetManagers.get(tp).contains(msgId) && !emitted.contains(msgId)) {
+ LOG.debug("Record at offset [{}] appears to have been compacted away from topic [{}], marking as acked", i, tp);
+ retryService.remove(msgId);
+ emitted.add(msgId);
+ ack(msgId);
+ }
+ }
+ }
}
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/d9fe768a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpoutConfig.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpoutConfig.java b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpoutConfig.java
index 8ff21b6..58d4753 100644
--- a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpoutConfig.java
+++ b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpoutConfig.java
@@ -728,6 +728,27 @@ public class KafkaSpoutConfig<K, V> implements Serializable {
if (builder.processingGuarantee == ProcessingGuarantee.NONE) {
builder.kafkaProps.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "true");
} else {
+ String autoOffsetResetPolicy = (String)builder.kafkaProps.get(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG);
+ if (builder.processingGuarantee == ProcessingGuarantee.AT_LEAST_ONCE) {
+ if (autoOffsetResetPolicy == null) {
+ /*
+ If the user wants to explicitly set an auto offset reset policy, we should respect it, but when the spout is configured
+ for at-least-once processing we should default to seeking to the earliest offset in case there's an offset out of range
+ error, rather than seeking to the latest (Kafka's default). This type of error will typically happen when the consumer
+ requests an offset that was deleted.
+ */
+ builder.kafkaProps.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
+ } else if (!autoOffsetResetPolicy.equals("earliest") && !autoOffsetResetPolicy.equals("none")) {
+ LOG.warn("Cannot guarantee at-least-once processing with auto.offset.reset.policy other than 'earliest' or 'none'."
+ + " Some messages may be skipped.");
+ }
+ } else if (builder.processingGuarantee == ProcessingGuarantee.AT_MOST_ONCE) {
+ if (autoOffsetResetPolicy != null
+ && (!autoOffsetResetPolicy.equals("latest") && !autoOffsetResetPolicy.equals("none"))) {
+ LOG.warn("Cannot guarantee at-most-once processing with auto.offset.reset.policy other than 'latest' or 'none'."
+ + " Some messages may be processed more than once.");
+ }
+ }
builder.kafkaProps.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "false");
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/d9fe768a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutCommitTest.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutCommitTest.java b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutCommitTest.java
deleted file mode 100644
index 95e3b60..0000000
--- a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutCommitTest.java
+++ /dev/null
@@ -1,113 +0,0 @@
-/*
- * Copyright 2017 The Apache Software Foundation.
- *
- * Licensed under the Apache License, Version 2.0 (the "License");
- * you may not use this file except in compliance with the License.
- * You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.storm.kafka.spout;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-import static org.mockito.Matchers.*;
-import static org.mockito.Mockito.*;
-
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.kafka.clients.consumer.*;
-import org.apache.kafka.common.TopicPartition;
-import org.apache.storm.kafka.spout.builders.SingleTopicKafkaSpoutConfiguration;
-import org.apache.storm.spout.SpoutOutputCollector;
-import org.apache.storm.task.TopologyContext;
-import org.apache.storm.utils.Time;
-import org.apache.storm.utils.Time.SimulatedTime;
-import org.junit.Before;
-import org.junit.Test;
-import org.mockito.ArgumentCaptor;
-import org.mockito.Captor;
-import org.mockito.InOrder;
-import org.mockito.MockitoAnnotations;
-
-import static org.apache.storm.kafka.spout.builders.SingleTopicKafkaSpoutConfiguration.createKafkaSpoutConfigBuilder;
-
-public class KafkaSpoutCommitTest {
-
- private final long offsetCommitPeriodMs = 2_000;
- private final TopologyContext contextMock = mock(TopologyContext.class);
- private final SpoutOutputCollector collectorMock = mock(SpoutOutputCollector.class);
- private final Map<String, Object> conf = new HashMap<>();
- private final TopicPartition partition = new TopicPartition(SingleTopicKafkaSpoutConfiguration.TOPIC, 1);
- private KafkaConsumer<String, String> consumerMock;
- private KafkaSpoutConfig<String, String> spoutConfig;
-
- @Captor
- private ArgumentCaptor<Map<TopicPartition, OffsetAndMetadata>> commitCapture;
-
- @Before
- public void setUp() {
- MockitoAnnotations.initMocks(this);
- spoutConfig = createKafkaSpoutConfigBuilder(-1)
- .setOffsetCommitPeriodMs(offsetCommitPeriodMs)
- .build();
- consumerMock = mock(KafkaConsumer.class);
- }
-
- @Test
- public void testCommitSuccessWithOffsetVoids() {
- //Verify that the commit logic can handle offset voids
- try (SimulatedTime simulatedTime = new SimulatedTime()) {
- Set<TopicPartition> assignedPartitions = Collections.singleton(partition);
- KafkaSpout<String, String> spout = SpoutWithMockedConsumerSetupHelper.setupSpout(spoutConfig, conf, contextMock, collectorMock, consumerMock, assignedPartitions);
- Map<TopicPartition, List<ConsumerRecord<String, String>>> records = new HashMap<>();
- List<ConsumerRecord<String, String>> recordsForPartition = new ArrayList<>();
- // Offsets emitted are 0,1,2,3,4,<void>,8,9
- recordsForPartition.addAll(SpoutWithMockedConsumerSetupHelper.<String, String>createRecords(partition, 0, 5));
- recordsForPartition.addAll(SpoutWithMockedConsumerSetupHelper.<String, String>createRecords(partition, 8, 2));
- records.put(partition, recordsForPartition);
-
- when(consumerMock.poll(anyLong()))
- .thenReturn(new ConsumerRecords<>(records));
-
- for (int i = 0; i < recordsForPartition.size(); i++) {
- spout.nextTuple();
- }
-
- ArgumentCaptor<KafkaSpoutMessageId> messageIds = ArgumentCaptor.forClass(KafkaSpoutMessageId.class);
- verify(collectorMock, times(recordsForPartition.size())).emit(anyString(), anyList(), messageIds.capture());
-
- for (KafkaSpoutMessageId messageId : messageIds.getAllValues()) {
- spout.ack(messageId);
- }
-
- // Advance time and then trigger first call to kafka consumer commit; the commit must progress to offset 9
- Time.advanceTime(KafkaSpout.TIMER_DELAY_MS + offsetCommitPeriodMs);
- Map<TopicPartition, List<ConsumerRecord<String, String>>> emptyConsumerRecords = Collections.emptyMap();
- when(consumerMock.poll(anyLong()))
- .thenReturn(new ConsumerRecords<>(emptyConsumerRecords));
- spout.nextTuple();
-
- InOrder inOrder = inOrder(consumerMock);
- inOrder.verify(consumerMock).commitSync(commitCapture.capture());
- inOrder.verify(consumerMock).poll(anyLong());
-
- //verify that Offset 10 was last committed offset, since this is the offset the spout should resume at
- Map<TopicPartition, OffsetAndMetadata> commits = commitCapture.getValue();
- assertTrue(commits.containsKey(partition));
- assertEquals(10, commits.get(partition).offset());
- }
- }
-
-}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/storm/blob/d9fe768a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutConfigTest.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutConfigTest.java b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutConfigTest.java
index 051d212..59135e4 100644
--- a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutConfigTest.java
+++ b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutConfigTest.java
@@ -17,8 +17,11 @@
*/
package org.apache.storm.kafka.spout;
+import static org.hamcrest.CoreMatchers.is;
+import static org.hamcrest.CoreMatchers.nullValue;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertNull;
+import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue;
import java.util.HashMap;
@@ -38,6 +41,7 @@ public class KafkaSpoutConfigTest {
HashMap<String, Object> expected = new HashMap<>();
expected.put(ConsumerConfig.BOOTSTRAP_SERVERS_CONFIG, "localhost:1234");
expected.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "false");
+ expected.put(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "earliest");
expected.put(ConsumerConfig.KEY_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
expected.put(ConsumerConfig.VALUE_DESERIALIZER_CLASS_CONFIG, StringDeserializer.class);
assertEquals(expected, conf.getKafkaProps());
@@ -51,4 +55,24 @@ public class KafkaSpoutConfigTest {
assertTrue("Failed to set emit null tuples to true", conf.isEmitNullTuples());
}
+
+ @Test
+ public void testShouldNotChangeAutoOffsetResetPolicyWhenNotUsingAtLeastOnce() {
+ KafkaSpoutConfig<String, String> conf = KafkaSpoutConfig.builder("localhost:1234", "topic")
+ .setProcessingGuarantee(KafkaSpoutConfig.ProcessingGuarantee.AT_MOST_ONCE)
+ .build();
+
+ assertThat("When at-least-once is not specified, the spout should use the Kafka default auto offset reset policy",
+ conf.getKafkaProps().get(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG), nullValue());
+ }
+
+ @Test
+ public void testWillRespectExplicitAutoOffsetResetPolicy() {
+ KafkaSpoutConfig<String, String> conf = KafkaSpoutConfig.builder("localhost:1234", "topic")
+ .setProp(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG, "none")
+ .build();
+
+ assertThat("Should allow users to pick a different auto offset reset policy than the one recommended for the at-least-once processing guarantee",
+ (String)conf.getKafkaProps().get(ConsumerConfig.AUTO_OFFSET_RESET_CONFIG), is("none"));
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/d9fe768a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutEmitTest.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutEmitTest.java b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutEmitTest.java
index 5d4252e..677daea 100755
--- a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutEmitTest.java
+++ b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutEmitTest.java
@@ -74,7 +74,7 @@ public class KafkaSpoutEmitTest {
public void testNextTupleEmitsAtMostOneTuple() {
//The spout should emit at most one message per call to nextTuple
//This is necessary for Storm to be able to throttle the spout according to maxSpoutPending
- KafkaSpout<String, String> spout = SpoutWithMockedConsumerSetupHelper.setupSpout(spoutConfig, conf, contextMock, collectorMock, consumerMock, Collections.singleton(partition));
+ KafkaSpout<String, String> spout = SpoutWithMockedConsumerSetupHelper.setupSpout(spoutConfig, conf, contextMock, collectorMock, consumerMock, partition);
Map<TopicPartition, List<ConsumerRecord<String, String>>> records = new HashMap<>();
records.put(partition, SpoutWithMockedConsumerSetupHelper.<String, String>createRecords(partition, 0, 10));
@@ -92,7 +92,7 @@ public class KafkaSpoutEmitTest {
//Emit maxUncommittedOffsets messages, and fail all of them. Then ensure that the spout will retry them when the retry backoff has passed
try (SimulatedTime simulatedTime = new SimulatedTime()) {
- KafkaSpout<String, String> spout = SpoutWithMockedConsumerSetupHelper.setupSpout(spoutConfig, conf, contextMock, collectorMock, consumerMock, Collections.singleton(partition));
+ KafkaSpout<String, String> spout = SpoutWithMockedConsumerSetupHelper.setupSpout(spoutConfig, conf, contextMock, collectorMock, consumerMock, partition);
Map<TopicPartition, List<ConsumerRecord<String, String>>> records = new HashMap<>();
int numRecords = spoutConfig.getMaxUncommittedOffsets();
//This is cheating a bit since maxPollRecords would normally spread this across multiple polls
@@ -139,10 +139,7 @@ public class KafkaSpoutEmitTest {
//This verifies that partitions can't prevent each other from retrying tuples due to the maxUncommittedOffsets limit.
try (SimulatedTime simulatedTime = new SimulatedTime()) {
TopicPartition partitionTwo = new TopicPartition(SingleTopicKafkaSpoutConfiguration.TOPIC, 2);
- Set<TopicPartition> partitions = new HashSet<>();
- partitions.add(partition);
- partitions.add(partitionTwo);
- KafkaSpout<String, String> spout = SpoutWithMockedConsumerSetupHelper.setupSpout(spoutConfig, conf, contextMock, collectorMock, consumerMock, partitions);
+ KafkaSpout<String, String> spout = SpoutWithMockedConsumerSetupHelper.setupSpout(spoutConfig, conf, contextMock, collectorMock, consumerMock, partition, partitionTwo);
Map<TopicPartition, List<ConsumerRecord<String, String>>> records = new HashMap<>();
//This is cheating a bit since maxPollRecords would normally spread this across multiple polls
records.put(partition, SpoutWithMockedConsumerSetupHelper.<String, String>createRecords(partition, 0, spoutConfig.getMaxUncommittedOffsets()));
http://git-wip-us.apache.org/repos/asf/storm/blob/d9fe768a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutLogCompactionSupportTest.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutLogCompactionSupportTest.java b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutLogCompactionSupportTest.java
new file mode 100644
index 0000000..855d253
--- /dev/null
+++ b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutLogCompactionSupportTest.java
@@ -0,0 +1,223 @@
+/*
+ * Copyright 2017 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.storm.kafka.spout;
+
+import static org.apache.storm.kafka.spout.builders.SingleTopicKafkaSpoutConfiguration.createKafkaSpoutConfigBuilder;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+
+import org.apache.kafka.clients.consumer.*;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.storm.spout.SpoutOutputCollector;
+import org.apache.storm.task.TopologyContext;
+import org.apache.storm.utils.Time;
+import org.apache.storm.utils.Time.SimulatedTime;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.ArgumentCaptor;
+import org.mockito.Captor;
+import org.mockito.InOrder;
+import org.mockito.MockitoAnnotations;
+
+import static org.hamcrest.CoreMatchers.is;
+
+import static org.hamcrest.Matchers.hasKey;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThat;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Matchers.anyList;
+import static org.mockito.Matchers.anyLong;
+import static org.mockito.Matchers.anyString;
+import static org.mockito.Mockito.inOrder;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.reset;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import org.apache.storm.kafka.spout.builders.SingleTopicKafkaSpoutConfiguration;
+
+public class KafkaSpoutLogCompactionSupportTest {
+
+ private final long offsetCommitPeriodMs = 2_000;
+ private final TopologyContext contextMock = mock(TopologyContext.class);
+ private final SpoutOutputCollector collectorMock = mock(SpoutOutputCollector.class);
+ private final Map<String, Object> conf = new HashMap<>();
+ private final TopicPartition partition = new TopicPartition(SingleTopicKafkaSpoutConfiguration.TOPIC, 1);
+ private KafkaConsumer<String, String> consumerMock;
+ private KafkaSpoutConfig<String, String> spoutConfig;
+
+ @Captor
+ private ArgumentCaptor<Map<TopicPartition, OffsetAndMetadata>> commitCapture;
+
+ @Before
+ public void setUp() {
+ MockitoAnnotations.initMocks(this);
+ spoutConfig = createKafkaSpoutConfigBuilder(-1)
+ .setOffsetCommitPeriodMs(offsetCommitPeriodMs)
+ .build();
+ consumerMock = mock(KafkaConsumer.class);
+ }
+
+ @Test
+ public void testCommitSuccessWithOffsetVoids() {
+ //Verify that the commit logic can handle offset voids due to log compaction
+ try (SimulatedTime simulatedTime = new SimulatedTime()) {
+ KafkaSpout<String, String> spout = SpoutWithMockedConsumerSetupHelper.setupSpout(spoutConfig, conf, contextMock, collectorMock, consumerMock, partition);
+ Map<TopicPartition, List<ConsumerRecord<String, String>>> records = new HashMap<>();
+ List<ConsumerRecord<String, String>> recordsForPartition = new ArrayList<>();
+ // Offsets emitted are 0,1,2,3,4,<void>,8,9
+ recordsForPartition.addAll(SpoutWithMockedConsumerSetupHelper.<String, String>createRecords(partition, 0, 5));
+ recordsForPartition.addAll(SpoutWithMockedConsumerSetupHelper.<String, String>createRecords(partition, 8, 2));
+ records.put(partition, recordsForPartition);
+
+ when(consumerMock.poll(anyLong()))
+ .thenReturn(new ConsumerRecords<>(records));
+
+ for (int i = 0; i < recordsForPartition.size(); i++) {
+ spout.nextTuple();
+ }
+
+ ArgumentCaptor<KafkaSpoutMessageId> messageIds = ArgumentCaptor.forClass(KafkaSpoutMessageId.class);
+ verify(collectorMock, times(recordsForPartition.size())).emit(anyString(), anyList(), messageIds.capture());
+
+ for (KafkaSpoutMessageId messageId : messageIds.getAllValues()) {
+ spout.ack(messageId);
+ }
+
+ // Advance time and then trigger first call to kafka consumer commit; the commit must progress to offset 9
+ Time.advanceTime(KafkaSpout.TIMER_DELAY_MS + offsetCommitPeriodMs);
+ when(consumerMock.poll(anyLong()))
+ .thenReturn(new ConsumerRecords<String, String>(Collections.<TopicPartition, List<ConsumerRecord<String, String>>>emptyMap()));
+ spout.nextTuple();
+
+ InOrder inOrder = inOrder(consumerMock);
+ inOrder.verify(consumerMock).commitSync(commitCapture.capture());
+ inOrder.verify(consumerMock).poll(anyLong());
+
+ //verify that Offset 10 was last committed offset, since this is the offset the spout should resume at
+ Map<TopicPartition, OffsetAndMetadata> commits = commitCapture.getValue();
+ assertTrue(commits.containsKey(partition));
+ assertEquals(10, commits.get(partition).offset());
+ }
+ }
+
+ @Test
+ public void testWillSkipRetriableTuplesIfOffsetsAreCompactedAway() {
+ /*
+ Verify that failed offsets will only retry if the corresponding message exists.
+ When log compaction is enabled in Kafka it is possible that a tuple can fail,
+ and then be impossible to retry because the message in Kafka has been deleted.
+ The spout needs to quietly ack such tuples to allow commits to progress past the deleted offset.
+ */
+ try (SimulatedTime simulatedTime = new SimulatedTime()) {
+ TopicPartition partitionTwo = new TopicPartition(SingleTopicKafkaSpoutConfiguration.TOPIC, 2);
+ KafkaSpout<String, String> spout = SpoutWithMockedConsumerSetupHelper.setupSpout(spoutConfig, conf, contextMock, collectorMock, consumerMock, partition, partitionTwo);
+
+ List<KafkaSpoutMessageId> firstPartitionMsgIds = SpoutWithMockedConsumerSetupHelper
+ .pollAndEmit(spout, consumerMock, 3, collectorMock, partition, 0, 1, 2);
+ reset(collectorMock);
+ List<KafkaSpoutMessageId> secondPartitionMsgIds = SpoutWithMockedConsumerSetupHelper
+ .pollAndEmit(spout, consumerMock, 3, collectorMock, partitionTwo, 0, 1, 2);
+ reset(collectorMock);
+
+ for(int i = 0; i < 3; i++) {
+ spout.fail(firstPartitionMsgIds.get(i));
+ spout.fail(secondPartitionMsgIds.get(i));
+ }
+
+ Time.advanceTime(50);
+
+ //The failed tuples are ready for retry. Make it appear like 0 and 1 on the first partition were compacted away.
+ //In this case the second partition acts as control to verify that we only skip past offsets that are no longer present.
+ Map<TopicPartition, int[]> retryOffsets = new HashMap<>();
+ retryOffsets.put(partition, new int[] {2});
+ retryOffsets.put(partitionTwo, new int[] {0, 1, 2});
+ int expectedEmits = 4; //2 on first partition, 0-2 on second partition
+ List<KafkaSpoutMessageId> retryMessageIds = SpoutWithMockedConsumerSetupHelper.pollAndEmit(spout, consumerMock, expectedEmits, collectorMock, retryOffsets);
+
+ Time.advanceTime(KafkaSpout.TIMER_DELAY_MS + offsetCommitPeriodMs);
+ spout.nextTuple();
+
+ verify(consumerMock).commitSync(commitCapture.capture());
+ Map<TopicPartition, OffsetAndMetadata> committed = commitCapture.getValue();
+ assertThat(committed.keySet(), is(Collections.singleton(partition)));
+ assertThat("The first partition should have committed up to the first retriable tuple that is not missing", committed.get(partition).offset(), is(2L));
+
+ for(KafkaSpoutMessageId msgId : retryMessageIds) {
+ spout.ack(msgId);
+ }
+
+ //The spout should now commit all the offsets, since all offsets are either acked or were missing when retrying
+ Time.advanceTime(KafkaSpout.TIMER_DELAY_MS + offsetCommitPeriodMs);
+ spout.nextTuple();
+
+ verify(consumerMock, times(2)).commitSync(commitCapture.capture());
+ committed = commitCapture.getValue();
+ assertThat(committed, hasKey(partition));
+ assertThat(committed, hasKey(partitionTwo));
+ assertThat(committed.get(partition).offset(), is(3L));
+ assertThat(committed.get(partitionTwo).offset(), is(3L));
+ }
+ }
+
+ @Test
+ public void testWillSkipRetriableTuplesIfOffsetsAreCompactedAwayWithoutAckingPendingTuples() {
+ //Demonstrate that the spout doesn't ack pending tuples when skipping compacted tuples. The pending tuples should be allowed to finish normally.
+ try (SimulatedTime simulatedTime = new SimulatedTime()) {
+ KafkaSpout<String, String> spout = SpoutWithMockedConsumerSetupHelper.setupSpout(spoutConfig, conf, contextMock, collectorMock, consumerMock, partition);
+
+ List<KafkaSpoutMessageId> firstPartitionMsgIds = SpoutWithMockedConsumerSetupHelper
+ .pollAndEmit(spout, consumerMock, 3, collectorMock, partition, 0, 1, 2);
+ reset(collectorMock);
+
+ spout.fail(firstPartitionMsgIds.get(0));
+ spout.fail(firstPartitionMsgIds.get(2));
+
+ Time.advanceTime(50);
+
+ //The failed tuples are ready for retry. Make it appear like 0 and 1 were compacted away.
+ List<KafkaSpoutMessageId> retryMessageIds = SpoutWithMockedConsumerSetupHelper.pollAndEmit(spout, consumerMock, 1, collectorMock, partition, 2);
+ for(KafkaSpoutMessageId msgId : retryMessageIds) {
+ spout.ack(msgId);
+ }
+
+ Time.advanceTime(KafkaSpout.TIMER_DELAY_MS + offsetCommitPeriodMs);
+ spout.nextTuple();
+
+ verify(consumerMock).commitSync(commitCapture.capture());
+ Map<TopicPartition, OffsetAndMetadata> committed = commitCapture.getValue();
+ assertThat(committed.keySet(), is(Collections.singleton(partition)));
+ assertThat("The first partition should have committed the missing offset, but no further since the next tuple is pending",
+ committed.get(partition).offset(), is(1L));
+
+ spout.ack(firstPartitionMsgIds.get(1));
+
+ Time.advanceTime(KafkaSpout.TIMER_DELAY_MS + offsetCommitPeriodMs);
+ spout.nextTuple();
+
+ verify(consumerMock, times(2)).commitSync(commitCapture.capture());
+ committed = commitCapture.getValue();
+ assertThat(committed.keySet(), is(Collections.singleton(partition)));
+ assertThat("The first partition should have committed all offsets", committed.get(partition).offset(), is(3L));
+ }
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/storm/blob/d9fe768a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutMessagingGuaranteeTest.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutMessagingGuaranteeTest.java b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutMessagingGuaranteeTest.java
index 8613ce7..95bc728 100644
--- a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutMessagingGuaranteeTest.java
+++ b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutMessagingGuaranteeTest.java
@@ -67,7 +67,7 @@ public class KafkaSpoutMessagingGuaranteeTest {
KafkaSpoutConfig<String, String> spoutConfig = createKafkaSpoutConfigBuilder(-1)
.setProcessingGuarantee(KafkaSpoutConfig.ProcessingGuarantee.AT_MOST_ONCE)
.build();
- KafkaSpout<String, String> spout = SpoutWithMockedConsumerSetupHelper.setupSpout(spoutConfig, conf, contextMock, collectorMock, consumerMock, Collections.singleton(partition));
+ KafkaSpout<String, String> spout = SpoutWithMockedConsumerSetupHelper.setupSpout(spoutConfig, conf, contextMock, collectorMock, consumerMock, partition);
when(consumerMock.poll(anyLong())).thenReturn(new ConsumerRecords<>(Collections.singletonMap(partition,
SpoutWithMockedConsumerSetupHelper.<String, String>createRecords(partition, 0, 1))));
@@ -82,7 +82,7 @@ public class KafkaSpoutMessagingGuaranteeTest {
}
private void doTestModeDisregardsMaxUncommittedOffsets(KafkaSpoutConfig<String, String> spoutConfig) {
- KafkaSpout<String, String> spout = SpoutWithMockedConsumerSetupHelper.setupSpout(spoutConfig, conf, contextMock, collectorMock, consumerMock, Collections.singleton(partition));
+ KafkaSpout<String, String> spout = SpoutWithMockedConsumerSetupHelper.setupSpout(spoutConfig, conf, contextMock, collectorMock, consumerMock, partition);
when(consumerMock.poll(anyLong()))
.thenReturn(new ConsumerRecords<>(Collections.singletonMap(partition,
@@ -117,7 +117,7 @@ public class KafkaSpoutMessagingGuaranteeTest {
}
private void doTestModeCannotReplayTuples(KafkaSpoutConfig<String, String> spoutConfig) {
- KafkaSpout<String, String> spout = SpoutWithMockedConsumerSetupHelper.setupSpout(spoutConfig, conf, contextMock, collectorMock, consumerMock, Collections.singleton(partition));
+ KafkaSpout<String, String> spout = SpoutWithMockedConsumerSetupHelper.setupSpout(spoutConfig, conf, contextMock, collectorMock, consumerMock, partition);
when(consumerMock.poll(anyLong())).thenReturn(new ConsumerRecords<>(Collections.singletonMap(partition,
SpoutWithMockedConsumerSetupHelper.<String, String>createRecords(partition, 0, 1))));
@@ -163,7 +163,7 @@ public class KafkaSpoutMessagingGuaranteeTest {
private void doTestModeDoesNotCommitAckedTuples(KafkaSpoutConfig<String, String> spoutConfig) {
try (SimulatedTime time = new SimulatedTime()) {
- KafkaSpout<String, String> spout = SpoutWithMockedConsumerSetupHelper.setupSpout(spoutConfig, conf, contextMock, collectorMock, consumerMock, Collections.singleton(partition));
+ KafkaSpout<String, String> spout = SpoutWithMockedConsumerSetupHelper.setupSpout(spoutConfig, conf, contextMock, collectorMock, consumerMock,partition);
when(consumerMock.poll(anyLong())).thenReturn(new ConsumerRecords<>(Collections.singletonMap(partition,
SpoutWithMockedConsumerSetupHelper.<String, String>createRecords(partition, 0, 1))));
http://git-wip-us.apache.org/repos/asf/storm/blob/d9fe768a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutRetryLimitTest.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutRetryLimitTest.java b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutRetryLimitTest.java
index 24b1373..58cfd51 100644
--- a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutRetryLimitTest.java
+++ b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutRetryLimitTest.java
@@ -23,7 +23,6 @@ import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
-import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
import java.util.List;
@@ -79,7 +78,7 @@ public class KafkaSpoutRetryLimitTest {
public void testFailingTupleCompletesAckAfterRetryLimitIsMet() {
//Spout should ack failed messages after they hit the retry limit
try (SimulatedTime simulatedTime = new SimulatedTime()) {
- KafkaSpout<String, String> spout = SpoutWithMockedConsumerSetupHelper.setupSpout(spoutConfig, conf, contextMock, collectorMock, consumerMock, Collections.singleton(partition));
+ KafkaSpout<String, String> spout = SpoutWithMockedConsumerSetupHelper.setupSpout(spoutConfig, conf, contextMock, collectorMock, consumerMock, partition);
Map<TopicPartition, List<ConsumerRecord<String, String>>> records = new HashMap<>();
int lastOffset = 3;
int numRecords = lastOffset + 1;
http://git-wip-us.apache.org/repos/asf/storm/blob/d9fe768a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/SpoutWithMockedConsumerSetupHelper.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/SpoutWithMockedConsumerSetupHelper.java b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/SpoutWithMockedConsumerSetupHelper.java
index 6f6196a..e808335 100644
--- a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/SpoutWithMockedConsumerSetupHelper.java
+++ b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/SpoutWithMockedConsumerSetupHelper.java
@@ -17,22 +17,30 @@
package org.apache.storm.kafka.spout;
import static org.mockito.Matchers.anyString;
+import static org.mockito.Matchers.anyList;
+import static org.mockito.Matchers.anyLong;
+import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
import static org.mockito.Mockito.when;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
+import java.util.Map.Entry;
import java.util.Set;
import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import org.apache.kafka.common.PartitionInfo;
import org.apache.kafka.common.TopicPartition;
import org.apache.storm.kafka.spout.internal.KafkaConsumerFactory;
import org.apache.storm.spout.SpoutOutputCollector;
import org.apache.storm.task.TopologyContext;
+import org.mockito.ArgumentCaptor;
public class SpoutWithMockedConsumerSetupHelper {
@@ -50,9 +58,10 @@ public class SpoutWithMockedConsumerSetupHelper {
* @return The spout
*/
public static <K, V> KafkaSpout<K, V> setupSpout(KafkaSpoutConfig<K, V> spoutConfig, Map<String, Object> topoConf,
- TopologyContext contextMock, SpoutOutputCollector collectorMock, final KafkaConsumer<K, V> consumerMock, Set<TopicPartition> assignedPartitions) {
+ TopologyContext contextMock, SpoutOutputCollector collectorMock, final KafkaConsumer<K, V> consumerMock, TopicPartition... assignedPartitions) {
+ Set<TopicPartition> assignedPartitionsSet = new HashSet<>(Arrays.asList(assignedPartitions));
- stubAssignment(contextMock, consumerMock, assignedPartitions);
+ stubAssignment(contextMock, consumerMock, assignedPartitionsSet);
KafkaConsumerFactory<K, V> consumerFactory = new KafkaConsumerFactory<K, V>() {
@Override
public KafkaConsumer<K, V> createConsumer(KafkaSpoutConfig<K, V> kafkaSpoutConfig) {
@@ -64,7 +73,7 @@ public class SpoutWithMockedConsumerSetupHelper {
spout.open(topoConf, contextMock, collectorMock);
spout.activate();
- verify(consumerMock).assign(assignedPartitions);
+ verify(consumerMock).assign(assignedPartitionsSet);
return spout;
}
@@ -116,4 +125,55 @@ public class SpoutWithMockedConsumerSetupHelper {
return recordsForPartition;
}
+ /**
+ * Creates messages for the input offsets, emits the messages by calling nextTuple once per offset and returns the captured message ids
+ * @param <K> The Kafka key type
+ * @param <V> The Kafka value type
+ * @param spout The spout
+ * @param consumerMock The consumer used by the spout
+ * @param expectedEmits The number of expected emits
+ * @param collectorMock The collector used by the spout
+ * @param partition The partition to emit messages on
+ * @param offsetsToEmit The offsets to emit
+ * @return The message ids emitted by the spout during the nextTuple calls
+ */
+ public static <K, V> List<KafkaSpoutMessageId> pollAndEmit(KafkaSpout<K, V> spout, KafkaConsumer<K, V> consumerMock, int expectedEmits, SpoutOutputCollector collectorMock, TopicPartition partition, int... offsetsToEmit) {
+ return pollAndEmit(spout, consumerMock, expectedEmits, collectorMock, Collections.singletonMap(partition, offsetsToEmit));
+ }
+
+ /**
+ * Creates messages for the input offsets, emits the messages by calling nextTuple once per offset and returns the captured message ids
+ * @param <K> The Kafka key type
+ * @param <V> The Kafka value type
+ * @param spout The spout
+ * @param consumerMock The consumer used by the spout
+ * @param collectorMock The collector used by the spout
+ * @param offsetsToEmit The offsets to emit per partition
+ * @return The message ids emitted by the spout during the nextTuple calls
+ */
+ public static <K, V> List<KafkaSpoutMessageId> pollAndEmit(KafkaSpout<K, V> spout, KafkaConsumer<K, V> consumerMock, int expectedEmits, SpoutOutputCollector collectorMock, Map<TopicPartition, int[]> offsetsToEmit) {
+ int totalOffsets = 0;
+ Map<TopicPartition, List<ConsumerRecord<K, V>>> records = new HashMap<>();
+ for (Entry<TopicPartition, int[]> entry : offsetsToEmit.entrySet()) {
+ TopicPartition tp = entry.getKey();
+ List<ConsumerRecord<K, V>> tpRecords = new ArrayList<>();
+ for (Integer offset : entry.getValue()) {
+ tpRecords.add(new ConsumerRecord<K, V>(tp.topic(), tp.partition(), offset, null, null));
+ totalOffsets++;
+ }
+ records.put(tp, tpRecords);
+ }
+
+ when(consumerMock.poll(anyLong()))
+ .thenReturn(new ConsumerRecords<>(records));
+
+ for (int i = 0; i < totalOffsets; i++) {
+ spout.nextTuple();
+ }
+
+ ArgumentCaptor<KafkaSpoutMessageId> messageIds = ArgumentCaptor.forClass(KafkaSpoutMessageId.class);
+ verify(collectorMock, times(expectedEmits)).emit(anyString(), anyList(), messageIds.capture());
+ return messageIds.getAllValues();
+ }
+
}
http://git-wip-us.apache.org/repos/asf/storm/blob/d9fe768a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/internal/OffsetManagerTest.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/internal/OffsetManagerTest.java b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/internal/OffsetManagerTest.java
index 3acc252..46b3c9b 100644
--- a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/internal/OffsetManagerTest.java
+++ b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/internal/OffsetManagerTest.java
@@ -21,7 +21,6 @@ import static org.hamcrest.CoreMatchers.nullValue;
import static org.junit.Assert.assertThat;
import java.util.NoSuchElementException;
-import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.OffsetAndMetadata;
import org.apache.kafka.common.TopicPartition;
import org.apache.storm.kafka.spout.KafkaSpoutMessageId;
[12/50] [abbrv] storm git commit: STORM-2775 Update kafkaPartition
metrics to match the same format as kafkaOffset
Posted by pt...@apache.org.
STORM-2775 Update kafkaPartition metrics to match the same format as kafkaOffset
STORM-2775 Send kafkaPartition metrics in both the "old" and "new" format for backwards compatibility
STORM-2775 Update kafkaPartition metrics to match the same format as kafkaOffset
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/39e12aa2
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/39e12aa2
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/39e12aa2
Branch: refs/heads/metrics_v2
Commit: 39e12aa221edc3e815c52622258f8d3ceb3a2891
Parents: 90e1385
Author: Kevin Conaway <ke...@walmart.com>
Authored: Thu Oct 12 14:48:30 2017 -0400
Committer: Stig Rohde Døssing <sr...@apache.org>
Committed: Sat Oct 14 16:22:35 2017 +0200
----------------------------------------------------------------------
.../apache/storm/kafka/PartitionManager.java | 21 ++++++++++++++------
1 file changed, 15 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/39e12aa2/external/storm-kafka/src/jvm/org/apache/storm/kafka/PartitionManager.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka/src/jvm/org/apache/storm/kafka/PartitionManager.java b/external/storm-kafka/src/jvm/org/apache/storm/kafka/PartitionManager.java
index 5420887..c025e3b 100644
--- a/external/storm-kafka/src/jvm/org/apache/storm/kafka/PartitionManager.java
+++ b/external/storm-kafka/src/jvm/org/apache/storm/kafka/PartitionManager.java
@@ -166,13 +166,22 @@ public class PartitionManager {
}
public Map getMetricsDataMap() {
+ String[] metricPrefixes = new String[] {
+ _partition.getId(), // Correct metric prefix, see STORM-2775
+ _partition.toString() // Old prefix, kept for backwards compatibility
+ };
+
Map<String, Object> ret = new HashMap<>();
- ret.put(_partition + "/fetchAPILatencyMax", _fetchAPILatencyMax.getValueAndReset());
- ret.put(_partition + "/fetchAPILatencyMean", _fetchAPILatencyMean.getValueAndReset());
- ret.put(_partition + "/fetchAPICallCount", _fetchAPICallCount.getValueAndReset());
- ret.put(_partition + "/fetchAPIMessageCount", _fetchAPIMessageCount.getValueAndReset());
- ret.put(_partition + "/lostMessageCount", _lostMessageCount.getValueAndReset());
- ret.put(_partition + "/messageIneligibleForRetryCount", _messageIneligibleForRetryCount.getValueAndReset());
+
+ for (String metricPrefix : metricPrefixes) {
+ ret.put(metricPrefix + "/fetchAPILatencyMax", _fetchAPILatencyMax.getValueAndReset());
+ ret.put(metricPrefix + "/fetchAPILatencyMean", _fetchAPILatencyMean.getValueAndReset());
+ ret.put(metricPrefix + "/fetchAPICallCount", _fetchAPICallCount.getValueAndReset());
+ ret.put(metricPrefix + "/fetchAPIMessageCount", _fetchAPIMessageCount.getValueAndReset());
+ ret.put(metricPrefix + "/lostMessageCount", _lostMessageCount.getValueAndReset());
+ ret.put(metricPrefix + "/messageIneligibleForRetryCount", _messageIneligibleForRetryCount.getValueAndReset());
+ }
+
return ret;
}
[15/50] [abbrv] storm git commit: STORM-2607: Switch OffsetManager to
track earliest uncommitted offset instead of last committed offset for
compatibility with commitSync consumer API
Posted by pt...@apache.org.
STORM-2607: Switch OffsetManager to track earliest uncommitted offset instead of last committed offset for compatibility with commitSync consumer API
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/148ee660
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/148ee660
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/148ee660
Branch: refs/heads/metrics_v2
Commit: 148ee6609557d7436ce826da29ff3ec301458586
Parents: 4fc47b2
Author: Stig Rohde Døssing <st...@gmail.com>
Authored: Sat Jul 1 17:29:56 2017 +0200
Committer: Stig Rohde Døssing <sr...@apache.org>
Committed: Tue Oct 17 07:20:39 2017 +0200
----------------------------------------------------------------------
.../apache/storm/kafka/spout/KafkaSpout.java | 6 +-
.../kafka/spout/internal/OffsetManager.java | 86 +++++++-----
.../storm/kafka/spout/KafkaSpoutCommitTest.java | 6 +-
.../kafka/spout/KafkaSpoutRebalanceTest.java | 4 +-
.../kafka/spout/KafkaSpoutRetryLimitTest.java | 8 +-
.../kafka/spout/SingleTopicKafkaSpoutTest.java | 2 +-
.../kafka/spout/internal/OffsetManagerTest.java | 135 +++++++++++++++----
7 files changed, 174 insertions(+), 73 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/148ee660/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpout.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpout.java b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpout.java
index 3582bdb..68bce11 100644
--- a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpout.java
+++ b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpout.java
@@ -202,7 +202,7 @@ public class KafkaSpout<K, V> extends BaseRichSpout {
}
/**
- * sets the cursor to the location dictated by the first poll strategy and returns the fetch offset
+ * Sets the cursor to the location dictated by the first poll strategy and returns the fetch offset.
*/
private long doSeek(TopicPartition tp, OffsetAndMetadata committedOffset) {
if (committedOffset != null) { // offset was committed for this TopicPartition
@@ -211,8 +211,8 @@ public class KafkaSpout<K, V> extends BaseRichSpout {
} else if (firstPollOffsetStrategy.equals(LATEST)) {
kafkaConsumer.seekToEnd(Collections.singleton(tp));
} else {
- // By default polling starts at the last committed offset. +1 to point fetch to the first uncommitted offset.
- kafkaConsumer.seek(tp, committedOffset.offset() + 1);
+ // By default polling starts at the last committed offset, i.e. the first offset that was not marked as processed.
+ kafkaConsumer.seek(tp, committedOffset.offset());
}
} else { // no commits have ever been done, so start at the beginning or end depending on the strategy
if (firstPollOffsetStrategy.equals(EARLIEST) || firstPollOffsetStrategy.equals(UNCOMMITTED_EARLIEST)) {
http://git-wip-us.apache.org/repos/asf/storm/blob/148ee660/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/internal/OffsetManager.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/internal/OffsetManager.java b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/internal/OffsetManager.java
index 1c474e3..7dfe7f6 100755
--- a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/internal/OffsetManager.java
+++ b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/internal/OffsetManager.java
@@ -37,7 +37,7 @@ public class OffsetManager {
/* First offset to be fetched. It is either set to the beginning, end, or to the first uncommitted offset.
* Initial value depends on offset strategy. See KafkaSpoutConsumerRebalanceListener */
private final long initialFetchOffset;
- // Last offset committed to Kafka. Initially it is set to fetchOffset - 1
+ // Committed offset, i.e. the offset where processing will resume if the spout restarts. Initially it is set to fetchOffset.
private long committedOffset;
// Emitted Offsets List
private final NavigableSet<Long> emittedOffsets = new TreeSet<>();
@@ -47,8 +47,8 @@ public class OffsetManager {
public OffsetManager(TopicPartition tp, long initialFetchOffset) {
this.tp = tp;
this.initialFetchOffset = initialFetchOffset;
- this.committedOffset = initialFetchOffset - 1;
- LOG.debug("Instantiated {}", this);
+ this.committedOffset = initialFetchOffset;
+ LOG.debug("Instantiated {}", this.toString());
}
public void addToAckMsgs(KafkaSpoutMessageId msgId) { // O(Log N)
@@ -60,9 +60,11 @@ public class OffsetManager {
}
/**
- * An offset is only committed when all records with lower offset have been
+ * An offset can only be committed when all emitted records with lower offset have been
* acked. This guarantees that all offsets smaller than the committedOffset
- * have been delivered.
+ * have been delivered, or that those offsets no longer exist in Kafka.
+ * <p/>
+ * The returned offset points to the earliest uncommitted offset, and matches the semantics of the KafkaConsumer.commitSync API.
*
* @return the next OffsetAndMetadata to commit, or null if no offset is
* ready to commit.
@@ -71,74 +73,78 @@ public class OffsetManager {
boolean found = false;
long currOffset;
long nextCommitOffset = committedOffset;
- long lastOffMessageOffset = committedOffset;
KafkaSpoutMessageId nextCommitMsg = null; // this is a convenience variable to make it faster to create OffsetAndMetadata
for (KafkaSpoutMessageId currAckedMsg : ackedMsgs) { // complexity is that of a linear scan on a TreeMap
currOffset = currAckedMsg.offset();
- if (currOffset == lastOffMessageOffset + 1) { // found the next offset to commit
+ if (currOffset == nextCommitOffset) { // found the next offset to commit
found = true;
nextCommitMsg = currAckedMsg;
- lastOffMessageOffset = currOffset;
- nextCommitOffset = lastOffMessageOffset + 1;
- } else if (currOffset > lastOffMessageOffset + 1) {
- if (emittedOffsets.contains(lastOffMessageOffset + 1)) {
- LOG.debug("topic-partition [{}] has non-continuous offset [{}]. It will be processed in a subsequent batch.", tp, currOffset);
+ nextCommitOffset = currOffset + 1;
+ } else if (currOffset > nextCommitOffset) {
+ if (emittedOffsets.contains(nextCommitOffset)) {
+ LOG.debug("topic-partition [{}] has non-sequential offset [{}]."
+ + " It will be processed in a subsequent batch.", tp, currOffset);
break;
} else {
/*
- This case will arise in case of non contiguous offset being processed.
- So, if the topic doesn't contain offset = committedOffset + 1 (possible
+ This case will arise in case of non-sequential offset being processed.
+ So, if the topic doesn't contain offset = nextCommitOffset (possible
if the topic is compacted or deleted), the consumer should jump to
the next logical point in the topic. Next logical offset should be the
- first element after committedOffset in the ascending ordered emitted set.
+ first element after nextCommitOffset in the ascending ordered emitted set.
*/
- LOG.debug("Processed non contiguous offset. (committedOffset+1) is no longer part of the topic. Committed: [{}], Processed: [{}]", committedOffset, currOffset);
- final Long nextEmittedOffset = emittedOffsets.ceiling(lastOffMessageOffset);
+ LOG.debug("Processed non-sequential offset."
+ + " The earliest uncommitted offset is no longer part of the topic."
+ + " Missing offset: [{}], Processed: [{}]", nextCommitOffset, currOffset);
+ final Long nextEmittedOffset = emittedOffsets.ceiling(nextCommitOffset);
if (nextEmittedOffset != null && currOffset == nextEmittedOffset) {
- found = true;
+ LOG.debug("Found committable offset: [{}] after missing offset: [{}], skipping to the committable offset",
+ currOffset, nextCommitOffset);
nextCommitMsg = currAckedMsg;
- lastOffMessageOffset = currOffset;
- nextCommitOffset = lastOffMessageOffset + 1;
+ nextCommitOffset = currOffset + 1;
} else {
- LOG.debug("topic-partition [{}] has non-continuous offset [{}]. Next Offset to commit should be [{}]", tp, currOffset, nextEmittedOffset);
+ LOG.debug("Topic-partition [{}] has non-sequential offset [{}]."
+ + " Next offset to commit should be [{}]", tp, currOffset, nextCommitOffset);
break;
}
}
} else {
- throw new IllegalStateException("The offset [" + currOffset + "] is below the current committed "
- + "offset [" + committedOffset + "] for [" + tp + "]."
+ throw new IllegalStateException("The offset [" + currOffset + "] is below the current nextCommitOffset "
+ + "[" + nextCommitOffset + "] for [" + tp + "]."
+ " This should not be possible, and likely indicates a bug in the spout's acking or emit logic.");
}
}
OffsetAndMetadata nextCommitOffsetAndMetadata = null;
if (found) {
- nextCommitOffsetAndMetadata = new OffsetAndMetadata(nextCommitOffset, nextCommitMsg.getMetadata(Thread.currentThread()));
- LOG.debug("topic-partition [{}] has offsets [{}-{}] ready to be committed",
- tp, earliestUncommittedOffset, nextCommitOffsetAndMetadata.offset() - 1);
+ nextCommitOffsetAndMetadata = new OffsetAndMetadata(nextCommitOffset,
+ nextCommitMsg.getMetadata(Thread.currentThread()));
+ LOG.debug("Topic-partition [{}] has offsets [{}-{}] ready to be committed."
+ + " Processing will resume at offset [{}] if the spout restarts",
+ tp, committedOffset, nextCommitOffsetAndMetadata.offset() - 1, nextCommitOffsetAndMetadata.offset());
} else {
- LOG.debug("topic-partition [{}] has NO offsets ready to be committed", tp);
+ LOG.debug("Topic-partition [{}] has no offsets ready to be committed", tp);
}
LOG.trace("{}", this);
return nextCommitOffsetAndMetadata;
}
/**
- * Marks an offset has committed. This method has side effects - it sets the
+ * Marks an offset as committed. This method has side effects - it sets the
* internal state in such a way that future calls to
- * {@link #findNextCommitOffset()} will return offsets greater than the
+ * {@link #findNextCommitOffset()} will return offsets greater than or equal to the
* offset specified, if any.
*
- * @param committedOffset offset to be marked as committed
+ * @param committedOffset The committed offset. All lower offsets are expected to have been committed.
* @return Number of offsets committed in this commit
*/
public long commit(OffsetAndMetadata committedOffset) {
- final long preCommitCommittedOffsets = this.committedOffset;
- final long numCommittedOffsets = committedOffset.offset() - this.committedOffset - 1;
+ final long preCommitCommittedOffset = this.committedOffset;
+ long numCommittedOffsets = 0;
this.committedOffset = committedOffset.offset();
for (Iterator<KafkaSpoutMessageId> iterator = ackedMsgs.iterator(); iterator.hasNext();) {
- if (iterator.next().offset() <= committedOffset.offset()) {
+ if (iterator.next().offset() < committedOffset.offset()) {
iterator.remove();
numCommittedOffsets++;
} else {
@@ -147,7 +153,7 @@ public class OffsetManager {
}
for (Iterator<Long> iterator = emittedOffsets.iterator(); iterator.hasNext();) {
- if (iterator.next() <= committedOffset.offset()) {
+ if (iterator.next() < committedOffset.offset()) {
iterator.remove();
} else {
break;
@@ -156,8 +162,9 @@ public class OffsetManager {
LOG.trace("{}", this);
- LOG.debug("Committed [{}] offsets in the range [{}-{}] for topic-partition [{}].",
- numCommittedOffsets, preCommitCommittedOffsets + 1, this.committedOffset, tp);
+ LOG.debug("Committed [{}] offsets in the range [{}-{}] for topic-partition [{}]."
+ + " Processing will resume at [{}] if the spout restarts.",
+ numCommittedOffsets, preCommitCommittedOffset, this.committedOffset - 1, tp, this.committedOffset);
return numCommittedOffsets;
}
@@ -177,9 +184,14 @@ public class OffsetManager {
public boolean contains(KafkaSpoutMessageId msgId) {
return ackedMsgs.contains(msgId);
}
+
+ //VisibleForTesting
+ boolean containsEmitted(long offset) {
+ return emittedOffsets.contains(offset);
+ }
@Override
- public String toString() {
+ public final String toString() {
return "OffsetManager{"
+ "topic-partition=" + tp
+ ", fetchOffset=" + initialFetchOffset
http://git-wip-us.apache.org/repos/asf/storm/blob/148ee660/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutCommitTest.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutCommitTest.java b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutCommitTest.java
index c9c684f..eb498d4 100644
--- a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutCommitTest.java
+++ b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutCommitTest.java
@@ -42,8 +42,6 @@ import org.mockito.MockitoAnnotations;
import static org.apache.storm.kafka.spout.builders.SingleTopicKafkaSpoutConfiguration.createKafkaSpoutConfigBuilder;
-import org.mockito.stubbing.OngoingStubbing;
-
public class KafkaSpoutCommitTest {
private final long offsetCommitPeriodMs = 2_000;
@@ -107,10 +105,10 @@ public class KafkaSpoutCommitTest {
inOrder.verify(consumerMock).commitSync(commitCapture.capture());
inOrder.verify(consumerMock).poll(anyLong());
- //verify that Offset 9 was last committed offset
+ //verify that Offset 10 was last committed offset, since this is the offset the spout should resume at
Map<TopicPartition, OffsetAndMetadata> commits = commitCapture.getValue();
assertTrue(commits.containsKey(partition));
- assertEquals(9, commits.get(partition).offset());
+ assertEquals(10, commits.get(partition).offset());
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/148ee660/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutRebalanceTest.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutRebalanceTest.java b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutRebalanceTest.java
index 3e50506..010433a 100644
--- a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutRebalanceTest.java
+++ b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutRebalanceTest.java
@@ -53,7 +53,7 @@ import org.mockito.Mockito;
import org.mockito.MockitoAnnotations;
import static org.apache.storm.kafka.spout.builders.SingleTopicKafkaSpoutConfiguration.createKafkaSpoutConfigBuilder;
-import static org.apache.storm.kafka.spout.builders.SingleTopicKafkaSpoutConfiguration.createKafkaSpoutConfigBuilder;
+import static org.mockito.Matchers.anyObject;
import static org.mockito.Matchers.eq;
import java.util.HashSet;
@@ -247,6 +247,6 @@ public class KafkaSpoutRebalanceTest {
//This partition was previously assigned, so the consumer position shouldn't change
verify(consumerMock, never()).seek(eq(assignedPartition), anyLong());
//This partition is new, and should start at the committed offset
- verify(consumerMock).seek(newPartition, committedOffset + 1);
+ verify(consumerMock).seek(newPartition, committedOffset);
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/148ee660/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutRetryLimitTest.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutRetryLimitTest.java b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutRetryLimitTest.java
index 39fa42c..21997f8 100644
--- a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutRetryLimitTest.java
+++ b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutRetryLimitTest.java
@@ -17,7 +17,6 @@ package org.apache.storm.kafka.spout;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
-import static org.mockito.Matchers.*;
import static org.mockito.Mockito.inOrder;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.times;
@@ -45,6 +44,9 @@ import org.mockito.InOrder;
import org.mockito.MockitoAnnotations;
import static org.apache.storm.kafka.spout.builders.SingleTopicKafkaSpoutConfiguration.createKafkaSpoutConfigBuilder;
+import static org.mockito.Matchers.anyList;
+import static org.mockito.Matchers.anyLong;
+import static org.mockito.Matchers.anyString;
public class KafkaSpoutRetryLimitTest {
@@ -108,9 +110,9 @@ public class KafkaSpoutRetryLimitTest {
inOrder.verify(consumerMock).commitSync(commitCapture.capture());
inOrder.verify(consumerMock).poll(anyLong());
- //verify that Offset 3 was committed for the given TopicPartition
+ //verify that offset 4 was committed for the given TopicPartition, since processing should resume at 4.
assertTrue(commitCapture.getValue().containsKey(partition));
- assertEquals(lastOffset, ((OffsetAndMetadata) (commitCapture.getValue().get(partition))).offset());
+ assertEquals(lastOffset + 1, ((OffsetAndMetadata) (commitCapture.getValue().get(partition))).offset());
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/148ee660/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/SingleTopicKafkaSpoutTest.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/SingleTopicKafkaSpoutTest.java b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/SingleTopicKafkaSpoutTest.java
index 59921eb..211ae97 100644
--- a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/SingleTopicKafkaSpoutTest.java
+++ b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/SingleTopicKafkaSpoutTest.java
@@ -159,7 +159,7 @@ public class SingleTopicKafkaSpoutTest {
Map<TopicPartition, OffsetAndMetadata> capturedCommit = commitCapture.getValue();
TopicPartition expectedTp = new TopicPartition(SingleTopicKafkaSpoutConfiguration.TOPIC, 0);
assertThat("Should have committed to the right topic", capturedCommit, Matchers.hasKey(expectedTp));
- assertThat("Should have committed all the acked messages", capturedCommit.get(expectedTp).offset(), is((long)messageCount - 1));
+ assertThat("Should have committed all the acked messages", capturedCommit.get(expectedTp).offset(), is((long)messageCount));
/* Verify that the following acked (now committed) tuples are not emitted again
* Since the consumer position was somewhere in the middle of the acked tuples when the commit happened,
http://git-wip-us.apache.org/repos/asf/storm/blob/148ee660/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/internal/OffsetManagerTest.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/internal/OffsetManagerTest.java b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/internal/OffsetManagerTest.java
index e8896c9..abbacf9 100644
--- a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/internal/OffsetManagerTest.java
+++ b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/internal/OffsetManagerTest.java
@@ -21,6 +21,8 @@ import static org.hamcrest.CoreMatchers.nullValue;
import static org.junit.Assert.assertThat;
import java.util.NoSuchElementException;
+import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.clients.consumer.OffsetAndMetadata;
import org.apache.kafka.common.TopicPartition;
import org.apache.storm.kafka.spout.KafkaSpoutMessageId;
import org.junit.Rule;
@@ -31,54 +33,141 @@ public class OffsetManagerTest {
@Rule
public ExpectedException expect = ExpectedException.none();
+
+ private final long initialFetchOffset = 0;
+ private final TopicPartition testTp = new TopicPartition("testTopic", 0);
+ private final OffsetManager manager = new OffsetManager(testTp, initialFetchOffset);
@Test
public void testSkipMissingOffsetsWhenFindingNextCommitOffsetWithGapInMiddleOfAcked() {
- /*If topic compaction is enabled in Kafka, we sometimes need to commit past a gap of deleted offsets
+ /* If topic compaction is enabled in Kafka, we sometimes need to commit past a gap of deleted offsets
* Since the Kafka consumer should return offsets in order, we can assume that if a message is acked
* then any prior message will have been emitted at least once.
* If we see an acked message and some of the offsets preceding it were not emitted, they must have been compacted away and should be skipped.
*/
-
- TopicPartition tp = new TopicPartition("test", 0);
- OffsetManager manager = new OffsetManager(tp, 0);
-
manager.addToEmitMsgs(0);
manager.addToEmitMsgs(1);
manager.addToEmitMsgs(2);
//3, 4 compacted away
- manager.addToEmitMsgs(5);
- manager.addToEmitMsgs(6);
- manager.addToAckMsgs(new KafkaSpoutMessageId(tp, 0));
- manager.addToAckMsgs(new KafkaSpoutMessageId(tp, 1));
- manager.addToAckMsgs(new KafkaSpoutMessageId(tp, 2));
- manager.addToAckMsgs(new KafkaSpoutMessageId(tp, 6));
+ manager.addToEmitMsgs(initialFetchOffset + 5);
+ manager.addToEmitMsgs(initialFetchOffset + 6);
+ manager.addToAckMsgs(getMessageId(initialFetchOffset));
+ manager.addToAckMsgs(getMessageId(initialFetchOffset + 1));
+ manager.addToAckMsgs(getMessageId(initialFetchOffset + 2));
+ manager.addToAckMsgs(getMessageId(initialFetchOffset + 6));
- assertThat("The offset manager should not skip past offset 5 which is still pending", manager.findNextCommitOffset().offset(), is(2L));
+ assertThat("The offset manager should not skip past offset 5 which is still pending", manager.findNextCommitOffset().offset(), is(initialFetchOffset + 3));
- manager.addToAckMsgs(new KafkaSpoutMessageId(tp, 5));
+ manager.addToAckMsgs(getMessageId(initialFetchOffset + 5));
assertThat("The offset manager should skip past the gap in acked messages, since the messages were not emitted",
- manager.findNextCommitOffset().offset(), is(6L));
+ manager.findNextCommitOffset().offset(), is(initialFetchOffset + 7));
}
@Test
public void testSkipMissingOffsetsWhenFindingNextCommitOffsetWithGapBeforeAcked() {
-
- TopicPartition tp = new TopicPartition("test", 0);
- OffsetManager manager = new OffsetManager(tp, 0);
-
//0-4 compacted away
- manager.addToEmitMsgs(5);
- manager.addToEmitMsgs(6);
- manager.addToAckMsgs(new KafkaSpoutMessageId(tp, 6));
+ manager.addToEmitMsgs(initialFetchOffset + 5);
+ manager.addToEmitMsgs(initialFetchOffset + 6);
+ manager.addToAckMsgs(getMessageId(initialFetchOffset + 6));
assertThat("The offset manager should not skip past offset 5 which is still pending", manager.findNextCommitOffset(), is(nullValue()));
- manager.addToAckMsgs(new KafkaSpoutMessageId(tp, 5));
+ manager.addToAckMsgs(getMessageId(initialFetchOffset + 5));
assertThat("The offset manager should skip past the gap in acked messages, since the messages were not emitted",
- manager.findNextCommitOffset().offset(), is(6L));
+ manager.findNextCommitOffset().offset(), is(initialFetchOffset + 7));
+ }
+
+ @Test
+ public void testFindNextCommittedOffsetWithNoAcks() {
+ OffsetAndMetadata nextCommitOffset = manager.findNextCommitOffset();
+ assertThat("There shouldn't be a next commit offset when nothing has been acked", nextCommitOffset, is(nullValue()));
+ }
+
+ @Test
+ public void testFindNextCommitOffsetWithOneAck() {
+ /*
+ * The KafkaConsumer commitSync API docs: "The committed offset should be the next message your application will consume, i.e.
+ * lastProcessedMessageOffset + 1. "
+ */
+ emitAndAckMessage(getMessageId(initialFetchOffset));
+ OffsetAndMetadata nextCommitOffset = manager.findNextCommitOffset();
+ assertThat("The next commit offset should be one past the processed message offset", nextCommitOffset.offset(), is(initialFetchOffset + 1));
+ }
+
+ @Test
+ public void testFindNextCommitOffsetWithMultipleOutOfOrderAcks() {
+ emitAndAckMessage(getMessageId(initialFetchOffset + 1));
+ emitAndAckMessage(getMessageId(initialFetchOffset));
+ OffsetAndMetadata nextCommitOffset = manager.findNextCommitOffset();
+ assertThat("The next commit offset should be one past the processed message offset", nextCommitOffset.offset(), is(initialFetchOffset + 2));
+ }
+
+ @Test
+ public void testFindNextCommitOffsetWithAckedOffsetGap() {
+ emitAndAckMessage(getMessageId(initialFetchOffset + 2));
+ manager.addToEmitMsgs(initialFetchOffset + 1);
+ emitAndAckMessage(getMessageId(initialFetchOffset));
+ OffsetAndMetadata nextCommitOffset = manager.findNextCommitOffset();
+ assertThat("The next commit offset should cover the sequential acked offsets", nextCommitOffset.offset(), is(initialFetchOffset + 1));
+ }
+
+ @Test
+ public void testFindNextOffsetWithAckedButNotEmittedOffsetGap() {
+ /**
+ * If topic compaction is enabled in Kafka some offsets may be deleted.
+ * We distinguish this case from regular gaps in the acked offset sequence caused by out of order acking
+ * by checking that offsets in the gap have been emitted at some point previously.
+ * If they haven't then they can't exist in Kafka, since the spout emits tuples in order.
+ */
+ emitAndAckMessage(getMessageId(initialFetchOffset + 2));
+ emitAndAckMessage(getMessageId(initialFetchOffset));
+ OffsetAndMetadata nextCommitOffset = manager.findNextCommitOffset();
+ assertThat("The next commit offset should cover all the acked offsets, since the offset in the gap hasn't been emitted and doesn't exist",
+ nextCommitOffset.offset(), is(initialFetchOffset + 3));
+ }
+
+ @Test
+ public void testFindNextCommitOffsetWithUnackedOffsetGap() {
+ manager.addToEmitMsgs(initialFetchOffset + 1);
+ emitAndAckMessage(getMessageId(initialFetchOffset));
+ OffsetAndMetadata nextCommitOffset = manager.findNextCommitOffset();
+ assertThat("The next commit offset should cover the contiguously acked offsets", nextCommitOffset.offset(), is(initialFetchOffset + 1));
+ }
+
+ @Test
+ public void testFindNextCommitOffsetWhenTooLowOffsetIsAcked() {
+ OffsetManager startAtHighOffsetManager = new OffsetManager(testTp, 10);
+ emitAndAckMessage(getMessageId(0));
+ OffsetAndMetadata nextCommitOffset = startAtHighOffsetManager.findNextCommitOffset();
+ assertThat("Acking an offset earlier than the committed offset should have no effect", nextCommitOffset, is(nullValue()));
+ }
+
+ @Test
+ public void testCommit() {
+ emitAndAckMessage(getMessageId(initialFetchOffset));
+ emitAndAckMessage(getMessageId(initialFetchOffset + 1));
+ emitAndAckMessage(getMessageId(initialFetchOffset + 2));
+
+ long committedMessages = manager.commit(new OffsetAndMetadata(initialFetchOffset + 2));
+
+ assertThat("Should have committed all messages to the left of the earliest uncommitted offset", committedMessages, is(2L));
+ assertThat("The committed messages should not be in the acked list anymore", manager.contains(getMessageId(initialFetchOffset)), is(false));
+ assertThat("The committed messages should not be in the emitted list anymore", manager.containsEmitted(initialFetchOffset), is(false));
+ assertThat("The committed messages should not be in the acked list anymore", manager.contains(getMessageId(initialFetchOffset + 1)), is(false));
+ assertThat("The committed messages should not be in the emitted list anymore", manager.containsEmitted(initialFetchOffset + 1), is(false));
+ assertThat("The uncommitted message should still be in the acked list", manager.contains(getMessageId(initialFetchOffset + 2)), is(true));
+ assertThat("The uncommitted message should still be in the emitted list", manager.containsEmitted(initialFetchOffset + 2), is(true));
+ }
+
+ private KafkaSpoutMessageId getMessageId(long offset) {
+ return new KafkaSpoutMessageId(testTp, offset);
+ }
+
+ private void emitAndAckMessage(KafkaSpoutMessageId msgId) {
+ manager.addToEmitMsgs(msgId.offset());
+ manager.addToAckMsgs(msgId);
}
}
[08/50] [abbrv] storm git commit: [STORM-2769] Fast-fail if output
streamId is null
Posted by pt...@apache.org.
[STORM-2769] Fast-fail if output streamId is null
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/d029e1c0
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/d029e1c0
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/d029e1c0
Branch: refs/heads/metrics_v2
Commit: d029e1c044ce81273c2711f295c811f680f23385
Parents: cb2d7e8
Author: Ethan Li <et...@gmail.com>
Authored: Tue Oct 3 13:54:18 2017 -0500
Committer: Stig Rohde Døssing <sr...@apache.org>
Committed: Wed Oct 4 21:26:57 2017 +0200
----------------------------------------------------------------------
.../src/jvm/org/apache/storm/topology/OutputFieldsGetter.java | 3 +++
1 file changed, 3 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/d029e1c0/storm-core/src/jvm/org/apache/storm/topology/OutputFieldsGetter.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/topology/OutputFieldsGetter.java b/storm-core/src/jvm/org/apache/storm/topology/OutputFieldsGetter.java
index abc939c..fa4ca84 100644
--- a/storm-core/src/jvm/org/apache/storm/topology/OutputFieldsGetter.java
+++ b/storm-core/src/jvm/org/apache/storm/topology/OutputFieldsGetter.java
@@ -39,6 +39,9 @@ public class OutputFieldsGetter implements OutputFieldsDeclarer {
}
public void declareStream(String streamId, boolean direct, Fields fields) {
+ if (null == streamId) {
+ throw new IllegalArgumentException("streamId can't be null");
+ }
if(_fields.containsKey(streamId)) {
throw new IllegalArgumentException("Fields for " + streamId + " already set");
}
[34/50] [abbrv] storm git commit: Merge branch 'STORM-2535-1.x' of
https://github.com/srdo/storm into STORM-2535-1.x
Posted by pt...@apache.org.
Merge branch 'STORM-2535-1.x' of https://github.com/srdo/storm into STORM-2535-1.x
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/c6324246
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/c6324246
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/c6324246
Branch: refs/heads/metrics_v2
Commit: c6324246f926ea622b2c53fca3b5354d334522fe
Parents: ef73328 6a41b22
Author: Jungtaek Lim <ka...@gmail.com>
Authored: Mon Nov 13 08:04:05 2017 +0900
Committer: Jungtaek Lim <ka...@gmail.com>
Committed: Mon Nov 13 08:04:05 2017 +0900
----------------------------------------------------------------------
.../org/apache/storm/integration_test.clj | 53 ++++++++++++++------
1 file changed, 37 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
[49/50] [abbrv] storm git commit: Merge branch 'STORM-2815-1.x' of
https://github.com/Ethanlm/storm into STORM-2815-1.x-merge
Posted by pt...@apache.org.
Merge branch 'STORM-2815-1.x' of https://github.com/Ethanlm/storm into STORM-2815-1.x-merge
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/79626398
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/79626398
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/79626398
Branch: refs/heads/metrics_v2
Commit: 79626398f0e6db269bf6632c3e6e29a76ea1297b
Parents: 818ed16 ad13f45
Author: Jungtaek Lim <ka...@gmail.com>
Authored: Fri Nov 17 00:32:54 2017 +0900
Committer: Jungtaek Lim <ka...@gmail.com>
Committed: Fri Nov 17 00:32:54 2017 +0900
----------------------------------------------------------------------
.../src/clj/org/apache/storm/daemon/logviewer.clj | 10 +++++-----
storm-core/src/clj/org/apache/storm/ui/core.clj | 3 ++-
storm-core/src/clj/org/apache/storm/ui/helpers.clj | 15 ++++++++-------
3 files changed, 15 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
[22/50] [abbrv] storm git commit: Merge branch 'STORM-2784-1.x-merge'
into 1.x-branch
Posted by pt...@apache.org.
Merge branch 'STORM-2784-1.x-merge' into 1.x-branch
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/3414a8ca
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/3414a8ca
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/3414a8ca
Branch: refs/heads/metrics_v2
Commit: 3414a8cad8e811faa7bcebcd0026ed9ebb3dab06
Parents: 502ed7e 667ffee
Author: Jungtaek Lim <ka...@gmail.com>
Authored: Fri Oct 27 15:24:47 2017 +0900
Committer: Jungtaek Lim <ka...@gmail.com>
Committed: Fri Oct 27 15:24:47 2017 +0900
----------------------------------------------------------------------
.../src/main/java/org/apache/storm/kafka/spout/KafkaSpout.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
[44/50] [abbrv] storm git commit: [STORM-2814] Logviewer HTTP server
should return 403 if the user is unauthorized
Posted by pt...@apache.org.
[STORM-2814] Logviewer HTTP server should return 403 if the user is unauthorized
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/6799c5a3
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/6799c5a3
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/6799c5a3
Branch: refs/heads/metrics_v2
Commit: 6799c5a30dedc39cda9a641ecbf8bb81aa4eec17
Parents: 6d45875
Author: Ethan Li <et...@gmail.com>
Authored: Wed Nov 15 08:33:00 2017 -0600
Committer: Ethan Li <et...@gmail.com>
Committed: Wed Nov 15 08:33:20 2017 -0600
----------------------------------------------------------------------
storm-core/src/clj/org/apache/storm/daemon/logviewer.clj | 2 +-
storm-core/src/clj/org/apache/storm/ui/helpers.clj | 6 ++++--
2 files changed, 5 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/6799c5a3/storm-core/src/clj/org/apache/storm/daemon/logviewer.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/org/apache/storm/daemon/logviewer.clj b/storm-core/src/clj/org/apache/storm/daemon/logviewer.clj
index d7b21de..a59cbb9 100644
--- a/storm-core/src/clj/org/apache/storm/daemon/logviewer.clj
+++ b/storm-core/src/clj/org/apache/storm/daemon/logviewer.clj
@@ -849,7 +849,7 @@
"bytes in size (inclusive)"))))
(catch Exception ex
(json-response (exception->json ex) callback :status 500))))
- (json-response (unauthorized-user-json user) callback :status 401))
+ (json-response (unauthorized-user-json user) callback :status 403))
(json-response {"error" "Not Found"
"errorMessage" "The file was not found on this node."}
callback
http://git-wip-us.apache.org/repos/asf/storm/blob/6799c5a3/storm-core/src/clj/org/apache/storm/ui/helpers.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/org/apache/storm/ui/helpers.clj b/storm-core/src/clj/org/apache/storm/ui/helpers.clj
index 94b426b..78c78be 100644
--- a/storm-core/src/clj/org/apache/storm/ui/helpers.clj
+++ b/storm-core/src/clj/org/apache/storm/ui/helpers.clj
@@ -36,7 +36,8 @@
[ring.util.response :as response])
(:require [compojure.route :as route]
[compojure.handler :as handler])
- (:require [metrics.meters :refer [defmeter mark!]]))
+ (:require [metrics.meters :refer [defmeter mark!]]
+ [ring.util.response :as resp]))
(defmeter num-web-requests)
@@ -133,7 +134,8 @@
"errorMessage" (str "User " user " is not authorized.")})
(defn unauthorized-user-html [user]
- [[:h2 "User '" (escape-html user) "' is not authorized."]])
+ (-> (resp/response (str "User '" (escape-html user) "' is not authorized."))
+ (resp/status 403)))
(defn- mk-ssl-connector [port ks-path ks-password ks-type key-password
ts-path ts-password ts-type need-client-auth want-client-auth]
[18/50] [abbrv] storm git commit: STORM-2786: Enable tick tuples for
ackers
Posted by pt...@apache.org.
STORM-2786: Enable tick tuples for ackers
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/da041895
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/da041895
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/da041895
Branch: refs/heads/metrics_v2
Commit: da041895a80d1691d1af95ddc47b9a35999ad917
Parents: 9aff9f9
Author: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Authored: Mon Oct 23 14:03:37 2017 -0500
Committer: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Committed: Mon Oct 23 14:03:37 2017 -0500
----------------------------------------------------------------------
.../src/clj/org/apache/storm/daemon/executor.clj | 19 ++++++++++---------
1 file changed, 10 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/da041895/storm-core/src/clj/org/apache/storm/daemon/executor.clj
----------------------------------------------------------------------
diff --git a/storm-core/src/clj/org/apache/storm/daemon/executor.clj b/storm-core/src/clj/org/apache/storm/daemon/executor.clj
index 52063fc..66d1851 100644
--- a/storm-core/src/clj/org/apache/storm/daemon/executor.clj
+++ b/storm-core/src/clj/org/apache/storm/daemon/executor.clj
@@ -347,21 +347,22 @@
(defn setup-ticks! [worker executor-data]
(let [storm-conf (:storm-conf executor-data)
+ comp-id (:component-id executor-data)
tick-time-secs (storm-conf TOPOLOGY-TICK-TUPLE-FREQ-SECS)
receive-queue (:receive-queue executor-data)
context (:worker-context executor-data)]
(when tick-time-secs
- (if (or (Utils/isSystemId (:component-id executor-data))
+ (if (or (and (not= "__acker" comp-id) (Utils/isSystemId comp-id))
(and (= false (storm-conf TOPOLOGY-ENABLE-MESSAGE-TIMEOUTS))
(= :spout (:type executor-data))))
- (log-message "Timeouts disabled for executor " (:component-id executor-data) ":" (:executor-id executor-data))
- (schedule-recurring
- (:user-timer worker)
- tick-time-secs
- tick-time-secs
- (fn []
- (let [val [(AddressedTuple. AddressedTuple/BROADCAST_DEST (TupleImpl. context [tick-time-secs] Constants/SYSTEM_TASK_ID Constants/SYSTEM_TICK_STREAM_ID))]]
- (disruptor/publish receive-queue val))))))))
+ (log-message "Timeouts disabled for executor " comp-id ":" (:executor-id executor-data))
+ (let [val [(AddressedTuple. AddressedTuple/BROADCAST_DEST (TupleImpl. context [tick-time-secs] Constants/SYSTEM_TASK_ID Constants/SYSTEM_TICK_STREAM_ID))]]
+ (schedule-recurring
+ (:user-timer worker)
+ tick-time-secs
+ tick-time-secs
+ (fn []
+ (disruptor/publish receive-queue val))))))))
(defn mk-executor [worker executor-id initial-credentials]
(let [executor-data (mk-executor-data worker executor-id)
[09/50] [abbrv] storm git commit: Improve mention of aggregate() in
the partitionAggregate() docs
Posted by pt...@apache.org.
Improve mention of aggregate() in the partitionAggregate() docs
It was not obvious (to me) that aggregate() is covered in detail at
another place in the document. That specific section describes the
details of the optimization much more exhaustively.
Also, it was unclear what the benefits should be compared to, because
other aggregator types are only introduced afterwards.
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/e83878a7
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/e83878a7
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/e83878a7
Branch: refs/heads/metrics_v2
Commit: e83878a7b3e533427fda34a45ac7e120b33f73c0
Parents: d029e1c
Author: Felix Dreissig <f3...@f30.me>
Authored: Sat Jan 16 00:48:12 2016 +0100
Committer: Stig Rohde Døssing <sr...@apache.org>
Committed: Mon Oct 9 20:37:36 2017 +0200
----------------------------------------------------------------------
docs/Trident-API-Overview.md | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/e83878a7/docs/Trident-API-Overview.md
----------------------------------------------------------------------
diff --git a/docs/Trident-API-Overview.md b/docs/Trident-API-Overview.md
index ef743db..b602aa1 100644
--- a/docs/Trident-API-Overview.md
+++ b/docs/Trident-API-Overview.md
@@ -484,7 +484,7 @@ public class Count implements CombinerAggregator<Long> {
}
```
-The benefits of CombinerAggregators are seen when you use them with the aggregate method instead of partitionAggregate. In that case, Trident automatically optimizes the computation by doing partial aggregations before transferring tuples over the network.
+CombinerAggregators offer high efficiency when used with the aggregate method instead of partitionAggregate ([see below](#aggregation-operations)).
A ReducerAggregator has the following interface:
[31/50] [abbrv] storm git commit: STORM-2722: close the JMSSpout in
the tests when done
Posted by pt...@apache.org.
STORM-2722: close the JMSSpout in the tests when done
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/c70d7d49
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/c70d7d49
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/c70d7d49
Branch: refs/heads/metrics_v2
Commit: c70d7d49fbd36a7ab4c95c75205e1311f8fc41e2
Parents: c460369
Author: Robert (Bobby) Evans <ev...@yahoo-inc.com>
Authored: Mon Sep 18 14:50:07 2017 -0500
Committer: Stig Rohde Døssing <sr...@apache.org>
Committed: Sat Nov 11 12:45:15 2017 +0100
----------------------------------------------------------------------
.../apache/storm/jms/spout/JmsSpoutTest.java | 72 +++++++++++---------
1 file changed, 41 insertions(+), 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/c70d7d49/external/storm-jms/src/test/java/org/apache/storm/jms/spout/JmsSpoutTest.java
----------------------------------------------------------------------
diff --git a/external/storm-jms/src/test/java/org/apache/storm/jms/spout/JmsSpoutTest.java b/external/storm-jms/src/test/java/org/apache/storm/jms/spout/JmsSpoutTest.java
index da312da..b6406c8 100644
--- a/external/storm-jms/src/test/java/org/apache/storm/jms/spout/JmsSpoutTest.java
+++ b/external/storm-jms/src/test/java/org/apache/storm/jms/spout/JmsSpoutTest.java
@@ -45,28 +45,33 @@ public class JmsSpoutTest {
@Test
public void testFailure() throws JMSException, Exception {
JmsSpout spout = new JmsSpout();
- JmsProvider mockProvider = new MockJmsProvider();
- MockSpoutOutputCollector mockCollector = new MockSpoutOutputCollector();
- SpoutOutputCollector collector =
+ try {
+ JmsProvider mockProvider = new MockJmsProvider();
+ MockSpoutOutputCollector mockCollector = new MockSpoutOutputCollector();
+ SpoutOutputCollector collector =
new SpoutOutputCollector(mockCollector);
- spout.setJmsProvider(new MockJmsProvider());
- spout.setJmsTupleProducer(new MockTupleProducer());
- spout.setJmsAcknowledgeMode(Session.CLIENT_ACKNOWLEDGE);
- spout.setRecoveryPeriodMs(10); // Rapid recovery for testing.
- spout.open(new HashMap<>(), null, collector);
- ConnectionFactory connectionFactory = mockProvider.connectionFactory();
- Destination destination = mockProvider.destination();
- Message msg = this.sendMessage(connectionFactory, destination);
- Thread.sleep(100);
- spout.nextTuple(); // Pretend to be storm.
- Assert.assertTrue(mockCollector.emitted);
+ spout.setJmsProvider(new MockJmsProvider());
+ spout.setJmsTupleProducer(new MockTupleProducer());
+ spout.setJmsAcknowledgeMode(Session.CLIENT_ACKNOWLEDGE);
+ spout.setRecoveryPeriodMs(10); // Rapid recovery for testing.
+ spout.open(new HashMap<>(), null, collector);
+ ConnectionFactory connectionFactory = mockProvider.connectionFactory();
+ Destination destination = mockProvider.destination();
+ Message msg = this.sendMessage(connectionFactory, destination);
+ Thread.sleep(100);
+ LOG.info("Calling nextTuple on the spout...");
+ spout.nextTuple(); // Pretend to be storm.
+ Assert.assertTrue(mockCollector.emitted);
- mockCollector.reset();
- spout.fail(msg.getJMSMessageID()); // Mock failure
- Thread.sleep(5000);
- spout.nextTuple(); // Pretend to be storm.
- Thread.sleep(5000);
- Assert.assertTrue(mockCollector.emitted); // Should have been re-emitted
+ mockCollector.reset();
+ spout.fail(msg.getJMSMessageID()); // Mock failure
+ Thread.sleep(5000);
+ spout.nextTuple(); // Pretend to be storm.
+ Thread.sleep(5000);
+ Assert.assertTrue(mockCollector.emitted); // Should have been re-emitted
+ } finally {
+ spout.close();
+ }
}
@Test
@@ -87,21 +92,26 @@ public class JmsSpoutTest {
@Test
public void testOpenWorksMultipleTypesOfNumberObjects() throws Exception {
JmsSpout spout = new JmsSpout();
- spout.setJmsProvider(new MockJmsProvider());
- spout.setJmsTupleProducer(new MockTupleProducer());
- Map<String, Object> configuration = new HashMap<String, Object>();
- MockSpoutOutputCollector delegateCollector =
+ try {
+ spout.setJmsProvider(new MockJmsProvider());
+ spout.setJmsTupleProducer(new MockTupleProducer());
+ Map<String, Object> configuration = new HashMap<String, Object>();
+ MockSpoutOutputCollector delegateCollector =
new MockSpoutOutputCollector();
- SpoutOutputCollector collector =
+ SpoutOutputCollector collector =
new SpoutOutputCollector(delegateCollector);
- // Test with long value
- configuration.put(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS, 1000L);
- spout.open(configuration, null, collector);
+ // Test with long value
+ configuration.put(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS, 1000L);
+ spout.open(configuration, null, collector);
+ spout.close();
- // Test with integer value
- configuration.put(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS, 1000);
- spout.open(configuration, null, collector);
+ // Test with integer value
+ configuration.put(Config.TOPOLOGY_MESSAGE_TIMEOUT_SECS, 1000);
+ spout.open(configuration, null, collector);
+ } finally {
+ spout.close();
+ }
}
public Message sendMessage(ConnectionFactory connectionFactory,
[06/50] [abbrv] storm git commit: STORM-2666: Fix storm-kafka-client
spout sometimes emitting messages that were already committed. Expand tests,
add some runtime validation,
minor refactoring to increase code readability. Ensure OffsetManager commits
as
Posted by pt...@apache.org.
STORM-2666: Fix storm-kafka-client spout sometimes emitting messages that were already committed. Expand tests, add some runtime validation, minor refactoring to increase code readability. Ensure OffsetManager commits as many offsets as possible when an offset void (deleted offsets) occurs, rather than just up to the gap.
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/9b95f3da
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/9b95f3da
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/9b95f3da
Branch: refs/heads/metrics_v2
Commit: 9b95f3da608760b334bb8a9a272e64e0fb9fb3ac
Parents: cb2d7e8
Author: Stig Rohde Døssing <sr...@apache.org>
Authored: Sat Aug 12 16:56:45 2017 +0200
Committer: Stig Rohde Døssing <sr...@apache.org>
Committed: Wed Oct 4 20:36:40 2017 +0200
----------------------------------------------------------------------
.../apache/storm/kafka/spout/KafkaSpout.java | 58 ++--
.../KafkaSpoutRetryExponentialBackoff.java | 37 +--
.../kafka/spout/KafkaSpoutRetryService.java | 9 +-
.../kafka/spout/internal/OffsetManager.java | 8 +-
.../storm/kafka/spout/KafkaSpoutCommitTest.java | 23 +-
.../kafka/spout/KafkaSpoutRebalanceTest.java | 56 +++-
.../KafkaSpoutRetryExponentialBackoffTest.java | 287 +++++++++++++++++++
.../kafka/spout/SingleTopicKafkaSpoutTest.java | 62 +++-
.../kafka/spout/internal/OffsetManagerTest.java | 84 ++++++
9 files changed, 555 insertions(+), 69 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/9b95f3da/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpout.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpout.java b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpout.java
index fbd869c..4f9dacb 100644
--- a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpout.java
+++ b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpout.java
@@ -34,7 +34,7 @@ import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.concurrent.TimeUnit;
-
+import org.apache.commons.lang.Validate;
import org.apache.kafka.clients.consumer.ConsumerRebalanceListener;
import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
@@ -147,10 +147,13 @@ public class KafkaSpout<K, V> extends BaseRichSpout {
// =========== Consumer Rebalance Listener - On the same thread as the caller ===========
private class KafkaSpoutConsumerRebalanceListener implements ConsumerRebalanceListener {
+ private Collection<TopicPartition> previousAssignment = new HashSet<>();
+
@Override
public void onPartitionsRevoked(Collection<TopicPartition> partitions) {
LOG.info("Partitions revoked. [consumer-group={}, consumer={}, topic-partitions={}]",
- kafkaSpoutConfig.getConsumerGroupId(), kafkaConsumer, partitions);
+ kafkaSpoutConfig.getConsumerGroupId(), kafkaConsumer, partitions);
+ previousAssignment = partitions;
if (isAtLeastOnce() && initialized) {
initialized = false;
commitOffsetsForAckedTuples();
@@ -175,20 +178,14 @@ public class KafkaSpout<K, V> extends BaseRichSpout {
* Emitted messages for partitions that are no longer assigned to this spout can't
* be acked and should not be retried, hence remove them from emitted collection.
*/
- Set<TopicPartition> partitionsSet = new HashSet<>(partitions);
- Iterator<KafkaSpoutMessageId> msgIdIterator = emitted.iterator();
- while (msgIdIterator.hasNext()) {
- KafkaSpoutMessageId msgId = msgIdIterator.next();
- if (!partitionsSet.contains(msgId.getTopicPartition())) {
- msgIdIterator.remove();
- }
- }
+ emitted.removeIf(msgId -> !partitions.contains(msgId.getTopicPartition()));
}
- for (TopicPartition tp : partitions) {
+ Set<TopicPartition> newPartitions = new HashSet<>(partitions);
+ newPartitions.removeAll(previousAssignment);
+ for (TopicPartition tp : newPartitions) {
final OffsetAndMetadata committedOffset = kafkaConsumer.committed(tp);
final long fetchOffset = doSeek(tp, committedOffset);
- // Add offset managers for the new partitions.
// If this partition was previously assigned to this spout, leave the acked offsets as they were to resume where it left off
if (isAtLeastOnce() && !offsetManagers.containsKey(tp)) {
offsetManagers.put(tp, new OffsetManager(tp, fetchOffset));
@@ -202,18 +199,14 @@ public class KafkaSpout<K, V> extends BaseRichSpout {
* sets the cursor to the location dictated by the first poll strategy and returns the fetch offset
*/
private long doSeek(TopicPartition tp, OffsetAndMetadata committedOffset) {
- long fetchOffset;
if (committedOffset != null) { // offset was committed for this TopicPartition
if (firstPollOffsetStrategy.equals(EARLIEST)) {
kafkaConsumer.seekToBeginning(Collections.singleton(tp));
- fetchOffset = kafkaConsumer.position(tp);
} else if (firstPollOffsetStrategy.equals(LATEST)) {
kafkaConsumer.seekToEnd(Collections.singleton(tp));
- fetchOffset = kafkaConsumer.position(tp);
} else {
// By default polling starts at the last committed offset. +1 to point fetch to the first uncommitted offset.
- fetchOffset = committedOffset.offset() + 1;
- kafkaConsumer.seek(tp, fetchOffset);
+ kafkaConsumer.seek(tp, committedOffset.offset() + 1);
}
} else { // no commits have ever been done, so start at the beginning or end depending on the strategy
if (firstPollOffsetStrategy.equals(EARLIEST) || firstPollOffsetStrategy.equals(UNCOMMITTED_EARLIEST)) {
@@ -221,9 +214,8 @@ public class KafkaSpout<K, V> extends BaseRichSpout {
} else if (firstPollOffsetStrategy.equals(LATEST) || firstPollOffsetStrategy.equals(UNCOMMITTED_LATEST)) {
kafkaConsumer.seekToEnd(Collections.singleton(tp));
}
- fetchOffset = kafkaConsumer.position(tp);
}
- return fetchOffset;
+ return kafkaConsumer.position(tp);
}
}
@@ -231,7 +223,7 @@ public class KafkaSpout<K, V> extends BaseRichSpout {
@Override
public void nextTuple() {
try {
- if (initialized) {
+ if (initialized) {
if (commit()) {
commitOffsetsForAckedTuples();
}
@@ -339,12 +331,15 @@ public class KafkaSpout<K, V> extends BaseRichSpout {
*/
private boolean emitTupleIfNotEmitted(ConsumerRecord<K, V> record) {
final TopicPartition tp = new TopicPartition(record.topic(), record.partition());
- final KafkaSpoutMessageId msgId = retryService.getMessageId(record);
+ final KafkaSpoutMessageId msgId = retryService.getMessageId(tp, record.offset());
if (offsetManagers.containsKey(tp) && offsetManagers.get(tp).contains(msgId)) { // has been acked
LOG.trace("Tuple for record [{}] has already been acked. Skipping", record);
} else if (emitted.contains(msgId)) { // has been emitted and it's pending ack or fail
LOG.trace("Tuple for record [{}] has already been emitted. Skipping", record);
} else {
+ Validate.isTrue(kafkaConsumer.committed(tp) == null || kafkaConsumer.committed(tp).offset() < kafkaConsumer.position(tp),
+ "The spout is about to emit a message that has already been committed."
+ + " This should never occur, and indicates a bug in the spout");
final List<Object> tuple = kafkaSpoutConfig.getTranslator().apply(record);
if (isEmitTuple(tuple)) {
final boolean isScheduled = retryService.isScheduled(msgId);
@@ -408,6 +403,23 @@ public class KafkaSpout<K, V> extends BaseRichSpout {
for (Map.Entry<TopicPartition, OffsetAndMetadata> tpOffset : nextCommitOffsets.entrySet()) {
//Update the OffsetManager for each committed partition, and update numUncommittedOffsets
final TopicPartition tp = tpOffset.getKey();
+ long position = kafkaConsumer.position(tp);
+ long committedOffset = tpOffset.getValue().offset();
+ if (position < committedOffset) {
+ /*
+ * The position is behind the committed offset. This can happen in some cases, e.g. if a message failed,
+ * lots of (more than max.poll.records) later messages were acked, and the failed message then gets acked.
+ * The consumer may only be part way through "catching up" to where it was when it went back to retry the failed tuple.
+ * Skip the consumer forward to the committed offset drop the current waiting to emit list,
+ * since it'll likely contain committed offsets.
+ */
+ LOG.debug("Consumer fell behind committed offset. Catching up. Position was [{}], skipping to [{}]",
+ position, committedOffset);
+ kafkaConsumer.seek(tp, committedOffset);
+ waitingToEmit = null;
+ }
+
+
final OffsetManager offsetManager = offsetManagers.get(tp);
long numCommittedOffsets = offsetManager.commit(tpOffset.getValue());
numUncommittedOffsets -= numCommittedOffsets;
@@ -437,6 +449,8 @@ public class KafkaSpout<K, V> extends BaseRichSpout {
LOG.debug("Received direct ack for message [{}], associated with null tuple", msgId);
}
} else {
+ Validate.isTrue(!retryService.isScheduled(msgId), "The message id " + msgId + " is queued for retry while being acked."
+ + " This should never occur barring errors in the RetryService implementation or the spout code.");
offsetManagers.get(msgId.getTopicPartition()).addToAckMsgs(msgId);
emitted.remove(msgId);
}
@@ -456,6 +470,8 @@ public class KafkaSpout<K, V> extends BaseRichSpout {
LOG.debug("Received fail for tuple this spout is no longer tracking. Partitions may have been reassigned. Ignoring message [{}]", msgId);
return;
}
+ Validate.isTrue(!retryService.isScheduled(msgId), "The message id " + msgId + " is queued for retry while being failed."
+ + " This should never occur barring errors in the RetryService implementation or the spout code.");
msgId.incrementNumFails();
if (!retryService.schedule(msgId)) {
LOG.debug("Reached maximum number of retries. Message [{}] being marked as acked.", msgId);
http://git-wip-us.apache.org/repos/asf/storm/blob/9b95f3da/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpoutRetryExponentialBackoff.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpoutRetryExponentialBackoff.java b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpoutRetryExponentialBackoff.java
index 6b53779..81ef986 100644
--- a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpoutRetryExponentialBackoff.java
+++ b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpoutRetryExponentialBackoff.java
@@ -32,7 +32,7 @@ import java.util.Map;
import java.util.Set;
import java.util.TreeSet;
import java.util.concurrent.TimeUnit;
-import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.commons.lang.Validate;
import org.apache.storm.utils.Time;
/**
@@ -49,6 +49,7 @@ public class KafkaSpoutRetryExponentialBackoff implements KafkaSpoutRetryService
private final TimeInterval maxDelay;
private final int maxRetries;
+ //This class assumes that there is at most one retry schedule per message id in this set at a time.
private final Set<RetrySchedule> retrySchedules = new TreeSet<>(RETRY_ENTRY_TIME_STAMP_COMPARATOR);
private final Set<KafkaSpoutMessageId> toRetryMsgs = new HashSet<>(); // Convenience data structure to speedup lookups
@@ -168,7 +169,7 @@ public class KafkaSpoutRetryExponentialBackoff implements KafkaSpoutRetryService
this.delayPeriod = delayPeriod;
this.maxRetries = maxRetries;
this.maxDelay = maxDelay;
- LOG.debug("Instantiated {}", this);
+ LOG.debug("Instantiated {}", this.toStringImpl());
}
@Override
@@ -196,13 +197,14 @@ public class KafkaSpoutRetryExponentialBackoff implements KafkaSpoutRetryService
@Override
public boolean isReady(KafkaSpoutMessageId msgId) {
boolean retry = false;
- if (toRetryMsgs.contains(msgId)) {
+ if (isScheduled(msgId)) {
final long currentTimeNanos = Time.nanoTime();
for (RetrySchedule retrySchedule : retrySchedules) {
if (retrySchedule.retry(currentTimeNanos)) {
if (retrySchedule.msgId.equals(msgId)) {
retry = true;
LOG.debug("Found entry to retry {}", retrySchedule);
+ break; //Stop searching if the message is known to be ready for retry
}
} else {
LOG.debug("Entry to retry not found {}", retrySchedule);
@@ -221,14 +223,14 @@ public class KafkaSpoutRetryExponentialBackoff implements KafkaSpoutRetryService
@Override
public boolean remove(KafkaSpoutMessageId msgId) {
boolean removed = false;
- if (toRetryMsgs.contains(msgId)) {
+ if (isScheduled(msgId)) {
+ toRetryMsgs.remove(msgId);
for (Iterator<RetrySchedule> iterator = retrySchedules.iterator(); iterator.hasNext(); ) {
final RetrySchedule retrySchedule = iterator.next();
if (retrySchedule.msgId().equals(msgId)) {
iterator.remove();
- toRetryMsgs.remove(msgId);
removed = true;
- break;
+ break; //There is at most one schedule per message id
}
}
}
@@ -261,15 +263,8 @@ public class KafkaSpoutRetryExponentialBackoff implements KafkaSpoutRetryService
LOG.debug("Not scheduling [{}] because reached maximum number of retries [{}].", msgId, maxRetries);
return false;
} else {
- if (toRetryMsgs.contains(msgId)) {
- for (Iterator<RetrySchedule> iterator = retrySchedules.iterator(); iterator.hasNext(); ) {
- final RetrySchedule retrySchedule = iterator.next();
- if (retrySchedule.msgId().equals(msgId)) {
- iterator.remove();
- toRetryMsgs.remove(msgId);
- }
- }
- }
+ //Remove existing schedule for the message id
+ remove(msgId);
final RetrySchedule retrySchedule = new RetrySchedule(msgId, nextTime(msgId));
retrySchedules.add(retrySchedule);
toRetryMsgs.add(msgId);
@@ -294,9 +289,9 @@ public class KafkaSpoutRetryExponentialBackoff implements KafkaSpoutRetryService
}
@Override
- public KafkaSpoutMessageId getMessageId(ConsumerRecord<?, ?> record) {
- KafkaSpoutMessageId msgId = new KafkaSpoutMessageId(record);
- if (toRetryMsgs.contains(msgId)) {
+ public KafkaSpoutMessageId getMessageId(TopicPartition tp, long offset) {
+ KafkaSpoutMessageId msgId = new KafkaSpoutMessageId(tp, offset);
+ if (isScheduled(msgId)) {
for (KafkaSpoutMessageId originalMsgId : toRetryMsgs) {
if (originalMsgId.equals(msgId)) {
return originalMsgId;
@@ -308,6 +303,7 @@ public class KafkaSpoutRetryExponentialBackoff implements KafkaSpoutRetryService
// if value is greater than Long.MAX_VALUE it truncates to Long.MAX_VALUE
private long nextTime(KafkaSpoutMessageId msgId) {
+ Validate.isTrue(msgId.numFails() > 0, "nextTime assumes the message has failed at least once");
final long currentTimeNanos = Time.nanoTime();
final long nextTimeNanos = msgId.numFails() == 1 // numFails = 1, 2, 3, ...
? currentTimeNanos + initialDelay.lengthNanos
@@ -317,6 +313,11 @@ public class KafkaSpoutRetryExponentialBackoff implements KafkaSpoutRetryService
@Override
public String toString() {
+ return toStringImpl();
+ }
+
+ private String toStringImpl() {
+ //This is here to avoid an overridable call in the constructor
return "KafkaSpoutRetryExponentialBackoff{" +
"delay=" + initialDelay +
", ratio=" + delayPeriod +
http://git-wip-us.apache.org/repos/asf/storm/blob/9b95f3da/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpoutRetryService.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpoutRetryService.java b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpoutRetryService.java
index 5147752..b70acd7 100644
--- a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpoutRetryService.java
+++ b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpoutRetryService.java
@@ -23,7 +23,7 @@ import org.apache.kafka.common.TopicPartition;
import java.io.Serializable;
import java.util.Collection;
import java.util.Map;
-import org.apache.kafka.clients.consumer.ConsumerRecord;
+import org.apache.kafka.common.TopicPartition;
/**
* Represents the logic that manages the retrial of failed tuples.
@@ -84,9 +84,10 @@ public interface KafkaSpoutRetryService extends Serializable {
int readyMessageCount();
/**
- * Gets the {@link KafkaSpoutMessageId} for the given record.
- * @param record The record to fetch the id for
+ * Gets the {@link KafkaSpoutMessageId} for the record on the given topic partition and offset.
+ * @param topicPartition The topic partition of the record
+ * @param offset The offset of the record
* @return The id the record was scheduled for retry with, or a new {@link KafkaSpoutMessageId} if the record was not scheduled for retry.
*/
- KafkaSpoutMessageId getMessageId(ConsumerRecord<?, ?> record);
+ KafkaSpoutMessageId getMessageId(TopicPartition topicPartition, long offset);
}
http://git-wip-us.apache.org/repos/asf/storm/blob/9b95f3da/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/internal/OffsetManager.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/internal/OffsetManager.java b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/internal/OffsetManager.java
index 5139072..b6d36d8 100755
--- a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/internal/OffsetManager.java
+++ b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/internal/OffsetManager.java
@@ -92,7 +92,7 @@ public class OffsetManager {
first element after committedOffset in the ascending ordered emitted set.
*/
LOG.debug("Processed non contiguous offset. (committedOffset+1) is no longer part of the topic. Committed: [{}], Processed: [{}]", committedOffset, currOffset);
- final Long nextEmittedOffset = emittedOffsets.ceiling(nextCommitOffset);
+ final Long nextEmittedOffset = emittedOffsets.ceiling(nextCommitOffset + 1);
if (nextEmittedOffset != null && currOffset == nextEmittedOffset) {
found = true;
nextCommitMsg = currAckedMsg;
@@ -103,9 +103,9 @@ public class OffsetManager {
}
}
} else {
- //Received a redundant ack. Ignore and continue processing.
- LOG.warn("topic-partition [{}] has unexpected offset [{}]. Current committed Offset [{}]",
- tp, currOffset, committedOffset);
+ throw new IllegalStateException("The offset [" + currOffset + "] is below the current committed "
+ + "offset [" + committedOffset + "] for [" + tp + "]."
+ + " This should not be possible, and likely indicates a bug in the spout's acking or emit logic.");
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/9b95f3da/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutCommitTest.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutCommitTest.java b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutCommitTest.java
index afc9b82..c9c684f 100644
--- a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutCommitTest.java
+++ b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutCommitTest.java
@@ -42,6 +42,8 @@ import org.mockito.MockitoAnnotations;
import static org.apache.storm.kafka.spout.builders.SingleTopicKafkaSpoutConfiguration.createKafkaSpoutConfigBuilder;
+import org.mockito.stubbing.OngoingStubbing;
+
public class KafkaSpoutCommitTest {
private final long offsetCommitPeriodMs = 2_000;
@@ -94,7 +96,7 @@ public class KafkaSpoutCommitTest {
spout.ack(messageId);
}
- // Advance time and then trigger first call to kafka consumer commit; the commit will progress till offset 4
+ // Advance time and then trigger first call to kafka consumer commit; the commit must progress to offset 9
Time.advanceTime(KafkaSpout.TIMER_DELAY_MS + offsetCommitPeriodMs);
Map<TopicPartition, List<ConsumerRecord<String, String>>> emptyConsumerRecords = Collections.emptyMap();
when(consumerMock.poll(anyLong()))
@@ -105,25 +107,8 @@ public class KafkaSpoutCommitTest {
inOrder.verify(consumerMock).commitSync(commitCapture.capture());
inOrder.verify(consumerMock).poll(anyLong());
- //verify that Offset 4 was last committed offset
- //the offset void should be bridged in the next commit
- Map<TopicPartition, OffsetAndMetadata> commits = commitCapture.getValue();
- assertTrue(commits.containsKey(partition));
- assertEquals(4, commits.get(partition).offset());
-
- //Trigger second kafka consumer commit
- reset(consumerMock);
- when(consumerMock.poll(anyLong()))
- .thenReturn(new ConsumerRecords<String, String>(emptyConsumerRecords));
- Time.advanceTime(KafkaSpout.TIMER_DELAY_MS + offsetCommitPeriodMs);
- spout.nextTuple();
-
- inOrder = inOrder(consumerMock);
- inOrder.verify(consumerMock).commitSync(commitCapture.capture());
- inOrder.verify(consumerMock).poll(anyLong());
-
//verify that Offset 9 was last committed offset
- commits = commitCapture.getValue();
+ Map<TopicPartition, OffsetAndMetadata> commits = commitCapture.getValue();
assertTrue(commits.containsKey(partition));
assertEquals(9, commits.get(partition).offset());
}
http://git-wip-us.apache.org/repos/asf/storm/blob/9b95f3da/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutRebalanceTest.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutRebalanceTest.java b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutRebalanceTest.java
index ab57052..b16ba5d 100644
--- a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutRebalanceTest.java
+++ b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutRebalanceTest.java
@@ -35,7 +35,6 @@ import java.util.List;
import java.util.Map;
import org.apache.kafka.clients.consumer.ConsumerRebalanceListener;
-import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.ConsumerRecords;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import org.apache.kafka.clients.consumer.OffsetAndMetadata;
@@ -55,6 +54,11 @@ import org.mockito.MockitoAnnotations;
import static org.apache.storm.kafka.spout.builders.SingleTopicKafkaSpoutConfiguration.createKafkaSpoutConfigBuilder;
import static org.apache.storm.kafka.spout.builders.SingleTopicKafkaSpoutConfiguration.createKafkaSpoutConfigBuilder;
+import static org.mockito.Matchers.eq;
+
+import java.util.HashSet;
+import java.util.Set;
+import org.apache.kafka.clients.consumer.ConsumerRecord;
public class KafkaSpoutRebalanceTest {
@@ -181,7 +185,7 @@ public class KafkaSpoutRebalanceTest {
when(retryServiceMock.getMessageId(Mockito.any(ConsumerRecord.class)))
.thenReturn(new KafkaSpoutMessageId(partitionThatWillBeRevoked, 0))
.thenReturn(new KafkaSpoutMessageId(assignedPartition, 0));
-
+
//Emit a message on each partition and revoke the first partition
List<KafkaSpoutMessageId> emittedMessageIds = emitOneMessagePerPartitionThenRevokeOnePartition(
spout, partitionThatWillBeRevoked, assignedPartition, rebalanceListenerCapture);
@@ -197,4 +201,52 @@ public class KafkaSpoutRebalanceTest {
verify(retryServiceMock, never()).schedule(emittedMessageIds.get(0));
verify(retryServiceMock).schedule(emittedMessageIds.get(1));
}
+
+ @Test
+ public void testReassignPartitionSeeksForOnlyNewPartitions() {
+ /*
+ * When partitions are reassigned, the spout should seek with the first poll offset strategy for new partitions.
+ * Previously assigned partitions should be left alone, since the spout keeps the emitted and acked state for those.
+ */
+
+ ArgumentCaptor<ConsumerRebalanceListener> rebalanceListenerCapture = ArgumentCaptor.forClass(ConsumerRebalanceListener.class);
+ Subscription subscriptionMock = mock(Subscription.class);
+ doNothing()
+ .when(subscriptionMock)
+ .subscribe(any(), rebalanceListenerCapture.capture(), any());
+ KafkaSpout<String, String> spout = new KafkaSpout<>(createKafkaSpoutConfigBuilder(subscriptionMock, -1)
+ .setFirstPollOffsetStrategy(KafkaSpoutConfig.FirstPollOffsetStrategy.UNCOMMITTED_EARLIEST)
+ .build(), consumerFactory);
+ String topic = SingleTopicKafkaSpoutConfiguration.TOPIC;
+ TopicPartition assignedPartition = new TopicPartition(topic, 1);
+ TopicPartition newPartition = new TopicPartition(topic, 2);
+
+ //Setup spout with mock consumer so we can get at the rebalance listener
+ spout.open(conf, contextMock, collectorMock);
+ spout.activate();
+
+ //Assign partitions to the spout
+ ConsumerRebalanceListener consumerRebalanceListener = rebalanceListenerCapture.getValue();
+ Set<TopicPartition> assignedPartitions = new HashSet<>();
+ assignedPartitions.add(assignedPartition);
+ consumerRebalanceListener.onPartitionsAssigned(assignedPartitions);
+ reset(consumerMock);
+
+ //Set up committed so it looks like some messages have been committed on each partition
+ long committedOffset = 500;
+ when(consumerMock.committed(assignedPartition)).thenReturn(new OffsetAndMetadata(committedOffset));
+ when(consumerMock.committed(newPartition)).thenReturn(new OffsetAndMetadata(committedOffset));
+
+ //Now rebalance and add a new partition
+ consumerRebalanceListener.onPartitionsRevoked(assignedPartitions);
+ Set<TopicPartition> newAssignedPartitions = new HashSet<>();
+ newAssignedPartitions.add(assignedPartition);
+ newAssignedPartitions.add(newPartition);
+ consumerRebalanceListener.onPartitionsAssigned(newAssignedPartitions);
+
+ //This partition was previously assigned, so the consumer position shouldn't change
+ verify(consumerMock, never()).seek(eq(assignedPartition), anyLong());
+ //This partition is new, and should start at the committed offset
+ verify(consumerMock).seek(newPartition, committedOffset + 1);
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/9b95f3da/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutRetryExponentialBackoffTest.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutRetryExponentialBackoffTest.java b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutRetryExponentialBackoffTest.java
new file mode 100644
index 0000000..c543f8b
--- /dev/null
+++ b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutRetryExponentialBackoffTest.java
@@ -0,0 +1,287 @@
+/*
+ * Copyright 2017 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.storm.kafka.spout;
+
+import static org.hamcrest.CoreMatchers.is;
+import static org.hamcrest.CoreMatchers.sameInstance;
+import static org.junit.Assert.assertThat;
+
+import java.util.Arrays;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.storm.kafka.spout.KafkaSpoutRetryExponentialBackoff.TimeInterval;
+import org.apache.storm.utils.Time;
+import org.apache.storm.utils.Time.SimulatedTime;
+import org.junit.Test;
+
+public class KafkaSpoutRetryExponentialBackoffTest {
+
+ private final TopicPartition testTopic = new TopicPartition("topic", 0);
+ private final TopicPartition testTopic2 = new TopicPartition("other-topic", 0);
+
+ private KafkaSpoutRetryExponentialBackoff createNoWaitRetryService() {
+ return new KafkaSpoutRetryExponentialBackoff(TimeInterval.seconds(0), TimeInterval.seconds(0), 1, TimeInterval.seconds(0));
+ }
+
+ private KafkaSpoutRetryExponentialBackoff createOneSecondWaitRetryService() {
+ return new KafkaSpoutRetryExponentialBackoff(TimeInterval.seconds(1), TimeInterval.seconds(0), 1, TimeInterval.seconds(1));
+ }
+
+ @Test
+ public void testCanScheduleRetry() {
+ KafkaSpoutRetryExponentialBackoff retryService = createNoWaitRetryService();
+ long offset = 0;
+ KafkaSpoutMessageId msgId = retryService.getMessageId(testTopic, offset);
+ msgId.incrementNumFails();
+
+ boolean scheduled = retryService.schedule(msgId);
+
+ assertThat("The service must schedule the message for retry", scheduled, is(true));
+ KafkaSpoutMessageId retrievedMessageId = retryService.getMessageId(testTopic, offset);
+ assertThat("The service should return the original message id when asked for the same tp/offset twice", retrievedMessageId, sameInstance(msgId));
+ assertThat(retryService.isScheduled(msgId), is(true));
+ assertThat(retryService.isReady(msgId), is(true));
+ assertThat(retryService.readyMessageCount(), is(1));
+ assertThat(retryService.earliestRetriableOffsets(), is(Collections.singletonMap(testTopic, msgId.offset())));
+ }
+
+ @Test
+ public void testCanRescheduleRetry() {
+ try (SimulatedTime time = new SimulatedTime()) {
+
+ KafkaSpoutRetryExponentialBackoff retryService = createOneSecondWaitRetryService();
+ long offset = 0;
+ KafkaSpoutMessageId msgId = retryService.getMessageId(testTopic, offset);
+ msgId.incrementNumFails();
+
+ retryService.schedule(msgId);
+ Time.advanceTime(500);
+ boolean scheduled = retryService.schedule(msgId);
+
+ assertThat("The service must be able to reschedule an already scheduled id", scheduled, is(true));
+ Time.advanceTime(500);
+ assertThat("The message should not be ready for retry yet since it was rescheduled", retryService.isReady(msgId), is(false));
+ assertThat(retryService.isScheduled(msgId), is(true));
+ assertThat(retryService.earliestRetriableOffsets(), is(Collections.emptyMap()));
+ assertThat(retryService.readyMessageCount(), is(0));
+ Time.advanceTime(500);
+ assertThat("The message should be ready for retry once the full delay has passed", retryService.isReady(msgId), is(true));
+ assertThat(retryService.isScheduled(msgId), is(true));
+ assertThat(retryService.earliestRetriableOffsets(), is(Collections.singletonMap(testTopic, msgId.offset())));
+ assertThat(retryService.readyMessageCount(), is(1));
+ }
+ }
+
+ @Test
+ public void testCannotContainMultipleSchedulesForId() {
+ try (SimulatedTime time = new SimulatedTime()) {
+
+ KafkaSpoutRetryExponentialBackoff retryService = createOneSecondWaitRetryService();
+ long offset = 0;
+ KafkaSpoutMessageId msgId = retryService.getMessageId(testTopic, offset);
+ msgId.incrementNumFails();
+
+ retryService.schedule(msgId);
+ Time.advanceTime(500);
+ boolean scheduled = retryService.schedule(msgId);
+
+ retryService.remove(msgId);
+ assertThat("The message should no longer be scheduled", retryService.isScheduled(msgId), is(false));
+ Time.advanceTime(500);
+ assertThat("The message should not be ready for retry because it isn't scheduled", retryService.isReady(msgId), is(false));
+ }
+ }
+
+ @Test
+ public void testCanRemoveRetry() {
+ KafkaSpoutRetryExponentialBackoff retryService = createNoWaitRetryService();
+ long offset = 0;
+ KafkaSpoutMessageId msgId = retryService.getMessageId(testTopic, offset);
+ msgId.incrementNumFails();
+
+ retryService.schedule(msgId);
+ boolean removed = retryService.remove(msgId);
+
+ assertThat(removed, is(true));
+ assertThat(retryService.isScheduled(msgId), is(false));
+ assertThat(retryService.isReady(msgId), is(false));
+ assertThat(retryService.earliestRetriableOffsets(), is(Collections.emptyMap()));
+ assertThat(retryService.readyMessageCount(), is(0));
+ }
+
+ @Test
+ public void testCanHandleMultipleTopics() {
+ try (SimulatedTime time = new SimulatedTime()) {
+ //Tests that isScheduled, isReady and earliestRetriableOffsets are mutually consistent when there are messages from multiple partitions scheduled
+ KafkaSpoutRetryExponentialBackoff retryService = createOneSecondWaitRetryService();
+ long offset = 0;
+
+ KafkaSpoutMessageId msgIdTp1 = retryService.getMessageId(testTopic, offset);
+ KafkaSpoutMessageId msgIdTp2 = retryService.getMessageId(testTopic2, offset);
+ msgIdTp1.incrementNumFails();
+ msgIdTp2.incrementNumFails();
+
+ boolean scheduledOne = retryService.schedule(msgIdTp1);
+ Time.advanceTime(500);
+ boolean scheduledTwo = retryService.schedule(msgIdTp2);
+
+ //The retry schedules for two messages should be unrelated
+ assertThat(scheduledOne, is(true));
+ assertThat(retryService.isScheduled(msgIdTp1), is(true));
+ assertThat(scheduledTwo, is(true));
+ assertThat(retryService.isScheduled(msgIdTp2), is(true));
+ assertThat(retryService.isReady(msgIdTp1), is(false));
+ assertThat(retryService.isReady(msgIdTp2), is(false));
+
+ Time.advanceTime(500);
+ assertThat(retryService.isReady(msgIdTp1), is(true));
+ assertThat(retryService.isReady(msgIdTp2), is(false));
+ assertThat(retryService.earliestRetriableOffsets(), is(Collections.singletonMap(testTopic, offset)));
+
+ Time.advanceTime(500);
+ assertThat(retryService.isReady(msgIdTp2), is(true));
+ Map<TopicPartition, Long> earliestOffsets = new HashMap<>();
+ earliestOffsets.put(testTopic, offset);
+ earliestOffsets.put(testTopic2, offset);
+ assertThat(retryService.earliestRetriableOffsets(), is(earliestOffsets));
+
+ //The service must be able to remove retry schedules for unnecessary partitions
+ retryService.retainAll(Collections.singleton(testTopic2));
+ assertThat(retryService.isScheduled(msgIdTp1), is(false));
+ assertThat(retryService.isScheduled(msgIdTp2), is(true));
+ assertThat(retryService.isReady(msgIdTp1), is(false));
+ assertThat(retryService.isReady(msgIdTp2), is(true));
+ assertThat(retryService.earliestRetriableOffsets(), is(Collections.singletonMap(testTopic2, offset)));
+ }
+ }
+
+ @Test
+ public void testCanHandleMultipleMessagesOnPartition() {
+ try (SimulatedTime time = new SimulatedTime()) {
+ //Tests that isScheduled, isReady and earliestRetriableOffsets are mutually consistent when there are multiple messages scheduled on a partition
+ KafkaSpoutRetryExponentialBackoff retryService = createOneSecondWaitRetryService();
+ long offset = 0;
+
+ KafkaSpoutMessageId msgIdEarliest = retryService.getMessageId(testTopic, offset);
+ KafkaSpoutMessageId msgIdLatest = retryService.getMessageId(testTopic, offset + 1);
+ msgIdEarliest.incrementNumFails();
+ msgIdLatest.incrementNumFails();
+
+ retryService.schedule(msgIdEarliest);
+ Time.advanceTime(500);
+ retryService.schedule(msgIdLatest);
+
+ assertThat(retryService.isScheduled(msgIdEarliest), is(true));
+ assertThat(retryService.isScheduled(msgIdLatest), is(true));
+
+ Time.advanceTime(500);
+ assertThat(retryService.isReady(msgIdEarliest), is(true));
+ assertThat(retryService.isReady(msgIdLatest), is(false));
+ assertThat(retryService.earliestRetriableOffsets(), is(Collections.singletonMap(testTopic, msgIdEarliest.offset())));
+
+ Time.advanceTime(500);
+ assertThat(retryService.isReady(msgIdEarliest), is(true));
+ assertThat(retryService.isReady(msgIdLatest), is(true));
+ assertThat(retryService.earliestRetriableOffsets(), is(Collections.singletonMap(testTopic, msgIdEarliest.offset())));
+
+ retryService.remove(msgIdEarliest);
+ assertThat(retryService.earliestRetriableOffsets(), is(Collections.singletonMap(testTopic, msgIdLatest.offset())));
+ }
+ }
+
+ @Test
+ public void testMaxRetries() {
+ try (SimulatedTime time = new SimulatedTime()) {
+ int maxRetries = 3;
+ KafkaSpoutRetryExponentialBackoff retryService = new KafkaSpoutRetryExponentialBackoff(TimeInterval.seconds(0), TimeInterval.seconds(0), maxRetries, TimeInterval.seconds(0));
+ long offset = 0;
+
+ KafkaSpoutMessageId msgId = retryService.getMessageId(testTopic, offset);
+ for (int i = 0; i < maxRetries; i++) {
+ msgId.incrementNumFails();
+ }
+
+ //Should be allowed to retry 3 times, in addition to original try
+ boolean scheduled = retryService.schedule(msgId);
+
+ assertThat(scheduled, is(true));
+ assertThat(retryService.isScheduled(msgId), is(true));
+
+ retryService.remove(msgId);
+ msgId.incrementNumFails();
+ boolean rescheduled = retryService.schedule(msgId);
+
+ assertThat("The message should not be allowed to retry once the limit is reached", rescheduled, is(false));
+ assertThat(retryService.isScheduled(msgId), is(false));
+ }
+ }
+
+ @Test
+ public void testMaxDelay() {
+ try (SimulatedTime time = new SimulatedTime()) {
+ int maxDelaySecs = 2;
+ KafkaSpoutRetryExponentialBackoff retryService = new KafkaSpoutRetryExponentialBackoff(TimeInterval.seconds(500), TimeInterval.seconds(0), 1, TimeInterval.seconds(maxDelaySecs));
+ long offset = 0;
+
+ KafkaSpoutMessageId msgId = retryService.getMessageId(testTopic, offset);
+ msgId.incrementNumFails();
+
+ retryService.schedule(msgId);
+
+ Time.advanceTimeSecs(maxDelaySecs);
+ assertThat("The message should be ready for retry after the max delay", retryService.isReady(msgId), is(true));
+ }
+ }
+
+ private void validateBackoff(int expectedBackoffSeconds, KafkaSpoutMessageId msgId, KafkaSpoutRetryExponentialBackoff retryService) {
+ Time.advanceTimeSecs(expectedBackoffSeconds - 1);
+ assertThat("The message should not be ready for retry until the backoff has expired", retryService.isReady(msgId), is(false));
+ Time.advanceTimeSecs(1);
+ assertThat(retryService.isReady(msgId), is(true));
+ }
+
+ @Test
+ public void testExponentialBackoff() {
+ try (SimulatedTime time = new SimulatedTime()) {
+ KafkaSpoutRetryExponentialBackoff retryService = new KafkaSpoutRetryExponentialBackoff(TimeInterval.seconds(0), TimeInterval.seconds(4), Integer.MAX_VALUE, TimeInterval.seconds(Integer.MAX_VALUE));
+ long offset = 0;
+
+ KafkaSpoutMessageId msgId = retryService.getMessageId(testTopic, offset);
+ msgId.incrementNumFails();
+ msgId.incrementNumFails(); //First failure is the initial delay, so not interesting
+
+ //Expecting 4*2^(failCount-1)
+ List<Integer> expectedBackoffsSecs = Arrays.asList(new Integer[]{8, 16, 32});
+
+ for (Integer expectedBackoffSecs : expectedBackoffsSecs) {
+ retryService.schedule(msgId);
+
+ Time.advanceTimeSecs(expectedBackoffSecs - 1);
+ assertThat("The message should not be ready for retry until backoff " + expectedBackoffSecs + " has expired", retryService.isReady(msgId), is(false));
+ Time.advanceTimeSecs(1);
+ assertThat("The message should be ready for retry once backoff " + expectedBackoffSecs + " has expired", retryService.isReady(msgId), is(true));
+
+ msgId.incrementNumFails();
+ retryService.remove(msgId);
+ }
+ }
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/9b95f3da/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/SingleTopicKafkaSpoutTest.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/SingleTopicKafkaSpoutTest.java b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/SingleTopicKafkaSpoutTest.java
index cbbb391..8d7da7f 100644
--- a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/SingleTopicKafkaSpoutTest.java
+++ b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/SingleTopicKafkaSpoutTest.java
@@ -59,6 +59,11 @@ import org.mockito.Captor;
import org.mockito.MockitoAnnotations;
import static org.apache.storm.kafka.spout.builders.SingleTopicKafkaSpoutConfiguration.createKafkaSpoutConfigBuilder;
+import static org.mockito.Matchers.anyList;
+import static org.mockito.Matchers.anyString;
+
+import org.apache.kafka.clients.consumer.ConsumerConfig;
+import org.hamcrest.Matchers;
public class SingleTopicKafkaSpoutTest {
@@ -76,6 +81,7 @@ public class SingleTopicKafkaSpoutTest {
private KafkaConsumer<String, String> consumerSpy;
private KafkaConsumerFactory<String, String> consumerFactory;
private KafkaSpout<String, String> spout;
+ private final int maxPollRecords = 10;
@Before
public void setUp() {
@@ -84,6 +90,7 @@ public class SingleTopicKafkaSpoutTest {
.setOffsetCommitPeriodMs(commitOffsetPeriodMs)
.setRetry(new KafkaSpoutRetryExponentialBackoff(KafkaSpoutRetryExponentialBackoff.TimeInterval.seconds(0), KafkaSpoutRetryExponentialBackoff.TimeInterval.seconds(0),
maxRetries, KafkaSpoutRetryExponentialBackoff.TimeInterval.seconds(0)))
+ .setProp(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, maxPollRecords)
.build();
this.consumerSpy = spy(new KafkaConsumerFactoryDefault<String, String>().createConsumer(spoutConfig));
this.consumerFactory = new KafkaConsumerFactory<String, String>() {
@@ -115,6 +122,59 @@ public class SingleTopicKafkaSpoutTest {
}
@Test
+ public void testSeekToCommittedOffsetIfConsumerPositionIsBehindWhenCommitting() throws Exception {
+ try (SimulatedTime simulatedTime = new SimulatedTime()) {
+ int messageCount = maxPollRecords * 2;
+ prepareSpout(messageCount);
+
+ //Emit all messages and fail the first one while acking the rest
+ for (int i = 0; i < messageCount; i++) {
+ spout.nextTuple();
+ }
+ ArgumentCaptor<KafkaSpoutMessageId> messageIdCaptor = ArgumentCaptor.forClass(KafkaSpoutMessageId.class);
+ verify(collector, times(messageCount)).emit(anyObject(), anyObject(), messageIdCaptor.capture());
+ List<KafkaSpoutMessageId> messageIds = messageIdCaptor.getAllValues();
+ for (int i = 1; i < messageIds.size(); i++) {
+ spout.ack(messageIds.get(i));
+ }
+ KafkaSpoutMessageId failedTuple = messageIds.get(0);
+ spout.fail(failedTuple);
+
+ //Advance the time and replay the failed tuple.
+ reset(collector);
+ spout.nextTuple();
+ ArgumentCaptor<KafkaSpoutMessageId> failedIdReplayCaptor = ArgumentCaptor.forClass(KafkaSpoutMessageId.class);
+ verify(collector).emit(anyObject(), anyObject(), failedIdReplayCaptor.capture());
+
+ assertThat("Expected replay of failed tuple", failedIdReplayCaptor.getValue(), is(failedTuple));
+
+ /* Ack the tuple, and commit.
+ * Since the tuple is more than max poll records behind the most recent emitted tuple, the consumer won't catch up in this poll.
+ */
+ Time.advanceTime(KafkaSpout.TIMER_DELAY_MS + commitOffsetPeriodMs);
+ spout.ack(failedIdReplayCaptor.getValue());
+ spout.nextTuple();
+ verify(consumerSpy).commitSync(commitCapture.capture());
+
+ Map<TopicPartition, OffsetAndMetadata> capturedCommit = commitCapture.getValue();
+ TopicPartition expectedTp = new TopicPartition(SingleTopicKafkaSpoutConfiguration.TOPIC, 0);
+ assertThat("Should have committed to the right topic", capturedCommit, Matchers.hasKey(expectedTp));
+ assertThat("Should have committed all the acked messages", capturedCommit.get(expectedTp).offset(), is((long)messageCount - 1));
+
+ /* Verify that the following acked (now committed) tuples are not emitted again
+ * Since the consumer position was somewhere in the middle of the acked tuples when the commit happened,
+ * this verifies that the spout keeps the consumer position ahead of the committed offset when committing
+ */
+ reset(collector);
+ //Just do a few polls to check that nothing more is emitted
+ for(int i = 0; i < 3; i++) {
+ spout.nextTuple();
+ }
+ verify(collector, never()).emit(anyString(), anyList(), anyObject());
+ }
+ }
+
+ @Test
public void shouldContinueWithSlowDoubleAcks() throws Exception {
try (SimulatedTime simulatedTime = new SimulatedTime()) {
int messageCount = 20;
@@ -271,7 +331,7 @@ public class SingleTopicKafkaSpoutTest {
verifyAllMessagesCommitted(messageCount);
}
}
-
+
@Test
public void shouldReplayAllFailedTuplesWhenFailedOutOfOrder() throws Exception {
//The spout must reemit retriable tuples, even if they fail out of order.
http://git-wip-us.apache.org/repos/asf/storm/blob/9b95f3da/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/internal/OffsetManagerTest.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/internal/OffsetManagerTest.java b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/internal/OffsetManagerTest.java
new file mode 100644
index 0000000..e8896c9
--- /dev/null
+++ b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/internal/OffsetManagerTest.java
@@ -0,0 +1,84 @@
+/*
+ * Copyright 2017 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.storm.kafka.spout.internal;
+
+import static org.hamcrest.CoreMatchers.is;
+import static org.hamcrest.CoreMatchers.nullValue;
+import static org.junit.Assert.assertThat;
+
+import java.util.NoSuchElementException;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.storm.kafka.spout.KafkaSpoutMessageId;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.rules.ExpectedException;
+
+public class OffsetManagerTest {
+
+ @Rule
+ public ExpectedException expect = ExpectedException.none();
+
+ @Test
+ public void testSkipMissingOffsetsWhenFindingNextCommitOffsetWithGapInMiddleOfAcked() {
+ /*If topic compaction is enabled in Kafka, we sometimes need to commit past a gap of deleted offsets
+ * Since the Kafka consumer should return offsets in order, we can assume that if a message is acked
+ * then any prior message will have been emitted at least once.
+ * If we see an acked message and some of the offsets preceding it were not emitted, they must have been compacted away and should be skipped.
+ */
+
+ TopicPartition tp = new TopicPartition("test", 0);
+ OffsetManager manager = new OffsetManager(tp, 0);
+
+ manager.addToEmitMsgs(0);
+ manager.addToEmitMsgs(1);
+ manager.addToEmitMsgs(2);
+ //3, 4 compacted away
+ manager.addToEmitMsgs(5);
+ manager.addToEmitMsgs(6);
+ manager.addToAckMsgs(new KafkaSpoutMessageId(tp, 0));
+ manager.addToAckMsgs(new KafkaSpoutMessageId(tp, 1));
+ manager.addToAckMsgs(new KafkaSpoutMessageId(tp, 2));
+ manager.addToAckMsgs(new KafkaSpoutMessageId(tp, 6));
+
+ assertThat("The offset manager should not skip past offset 5 which is still pending", manager.findNextCommitOffset().offset(), is(2L));
+
+ manager.addToAckMsgs(new KafkaSpoutMessageId(tp, 5));
+
+ assertThat("The offset manager should skip past the gap in acked messages, since the messages were not emitted",
+ manager.findNextCommitOffset().offset(), is(6L));
+ }
+
+ @Test
+ public void testSkipMissingOffsetsWhenFindingNextCommitOffsetWithGapBeforeAcked() {
+
+ TopicPartition tp = new TopicPartition("test", 0);
+ OffsetManager manager = new OffsetManager(tp, 0);
+
+ //0-4 compacted away
+ manager.addToEmitMsgs(5);
+ manager.addToEmitMsgs(6);
+ manager.addToAckMsgs(new KafkaSpoutMessageId(tp, 6));
+
+ assertThat("The offset manager should not skip past offset 5 which is still pending", manager.findNextCommitOffset(), is(nullValue()));
+
+ manager.addToAckMsgs(new KafkaSpoutMessageId(tp, 5));
+
+ assertThat("The offset manager should skip past the gap in acked messages, since the messages were not emitted",
+ manager.findNextCommitOffset().offset(), is(6L));
+ }
+
+}
[39/50] [abbrv] storm git commit: STORM-2810: Fix resource leaks in
storm-hdfs tests
Posted by pt...@apache.org.
http://git-wip-us.apache.org/repos/asf/storm/blob/83adfd9d/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/spout/TestHdfsSpout.java
----------------------------------------------------------------------
diff --git a/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/spout/TestHdfsSpout.java b/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/spout/TestHdfsSpout.java
index f60cbf3..057787d 100644
--- a/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/spout/TestHdfsSpout.java
+++ b/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/spout/TestHdfsSpout.java
@@ -1,3 +1,4 @@
+
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
@@ -15,14 +16,12 @@
* See the License for the specific language governing permissions and
* limitations under the License.
*/
-
package org.apache.storm.hdfs.spout;
import org.apache.storm.Config;
import org.apache.storm.spout.SpoutOutputCollector;
import org.apache.storm.task.TopologyContext;
import org.apache.hadoop.hdfs.DistributedFileSystem;
-import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.util.ReflectionUtils;
import org.apache.storm.hdfs.common.HdfsUtils;
@@ -57,703 +56,707 @@ import java.util.List;
import java.util.Map;
import org.apache.storm.hdfs.common.HdfsUtils.Pair;
-
+import org.apache.storm.hdfs.testing.MiniDFSClusterRule;
+import org.junit.ClassRule;
public class TestHdfsSpout {
- @Rule
- public TemporaryFolder tempFolder = new TemporaryFolder();
- public File baseFolder;
- private Path source;
- private Path archive;
- private Path badfiles;
-
-
- public TestHdfsSpout() {
- }
-
- static MiniDFSCluster.Builder builder;
- static MiniDFSCluster hdfsCluster;
- static DistributedFileSystem fs;
- static String hdfsURI;
- static Configuration conf = new Configuration();
-
- @BeforeClass
- public static void setupClass() throws IOException {
- builder = new MiniDFSCluster.Builder(new Configuration());
- hdfsCluster = builder.build();
- fs = hdfsCluster.getFileSystem();
- hdfsURI = "hdfs://localhost:" + hdfsCluster.getNameNodePort() + "/";
- }
-
- @AfterClass
- public static void teardownClass() throws IOException {
- fs.close();
- hdfsCluster.shutdown();
- }
-
-
- @Before
- public void setup() throws Exception {
- baseFolder = tempFolder.newFolder("hdfsspout");
- source = new Path(baseFolder.toString() + "/source");
- fs.mkdirs(source);
- archive = new Path(baseFolder.toString() + "/archive");
- fs.mkdirs(archive);
- badfiles = new Path(baseFolder.toString() + "/bad");
- fs.mkdirs(badfiles);
- }
-
- @After
- public void shutDown() throws IOException {
- fs.delete(new Path(baseFolder.toString()), true);
- }
-
- @Test
- public void testSimpleText_noACK() throws IOException {
- Path file1 = new Path(source.toString() + "/file1.txt");
- createTextFile(file1, 5);
-
- Path file2 = new Path(source.toString() + "/file2.txt");
- createTextFile(file2, 5);
-
- HdfsSpout spout = makeSpout(Configs.TEXT, TextFileReader.defaultFields);
- spout.setCommitFrequencyCount(1);
- spout.setCommitFrequencySec(1);
-
- Map conf = getCommonConfigs();
- openSpout(spout, 0, conf);
-
- runSpout(spout,"r11");
-
- Path arc1 = new Path(archive.toString() + "/file1.txt");
- Path arc2 = new Path(archive.toString() + "/file2.txt");
- checkCollectorOutput_txt((MockCollector) spout.getCollector(), arc1, arc2);
- }
-
- @Test
- public void testSimpleText_ACK() throws IOException {
- Path file1 = new Path(source.toString() + "/file1.txt");
- createTextFile(file1, 5);
-
- Path file2 = new Path(source.toString() + "/file2.txt");
- createTextFile(file2, 5);
-
- HdfsSpout spout = makeSpout(Configs.TEXT, TextFileReader.defaultFields);
- spout.setCommitFrequencyCount(1);
- spout.setCommitFrequencySec(1);
-
- Map conf = getCommonConfigs();
- conf.put(Config.TOPOLOGY_ACKER_EXECUTORS, "1"); // enable ACKing
- openSpout(spout, 0, conf);
-
- // consume file 1
- runSpout(spout, "r6", "a0", "a1", "a2", "a3", "a4");
- Path arc1 = new Path(archive.toString() + "/file1.txt");
- checkCollectorOutput_txt((MockCollector) spout.getCollector(), arc1);
-
- // consume file 2
- runSpout(spout, "r6", "a5", "a6", "a7", "a8", "a9");
- Path arc2 = new Path(archive.toString() + "/file2.txt");
- checkCollectorOutput_txt((MockCollector) spout.getCollector(), arc1, arc2);
- }
-
- @Test
- public void testResumeAbandoned_Text_NoAck() throws Exception {
- Path file1 = new Path(source.toString() + "/file1.txt");
- createTextFile(file1, 6);
-
- final Integer lockExpirySec = 1;
-
- HdfsSpout spout = makeSpout(Configs.TEXT, TextFileReader.defaultFields);
- spout.setCommitFrequencyCount(1);
- spout.setCommitFrequencySec(1000); // effectively disable commits based on time
- spout.setLockTimeoutSec(lockExpirySec);
-
-
- HdfsSpout spout2 = makeSpout(Configs.TEXT, TextFileReader.defaultFields);
- spout2.setCommitFrequencyCount(1);
- spout2.setCommitFrequencySec(1000); // effectively disable commits based on time
- spout2.setLockTimeoutSec(lockExpirySec);
-
- Map conf = getCommonConfigs();
- openSpout(spout, 0, conf);
- openSpout(spout2, 1, conf);
-
- // consume file 1 partially
- List<String> res = runSpout(spout, "r2");
- Assert.assertEquals(2, res.size());
-
- // abandon file
- FileLock lock = getField(spout, "lock");
- TestFileLock.closeUnderlyingLockFile(lock);
- Thread.sleep(lockExpirySec * 2 * 1000);
-
- // check lock file presence
- Assert.assertTrue(fs.exists(lock.getLockFile()));
-
- // create another spout to take over processing and read a few lines
- List<String> res2 = runSpout(spout2, "r3");
- Assert.assertEquals(3, res2.size());
-
- // check lock file presence
- Assert.assertTrue(fs.exists(lock.getLockFile()));
-
- // check lock file contents
- List<String> contents = readTextFile(fs, lock.getLockFile().toString());
- Assert.assertFalse(contents.isEmpty());
-
- // finish up reading the file
- res2 = runSpout(spout2, "r2");
- Assert.assertEquals(4, res2.size());
-
- // check lock file is gone
- Assert.assertFalse(fs.exists(lock.getLockFile()));
- FileReader rdr = getField(spout2, "reader");
- Assert.assertNull(rdr);
- Assert.assertTrue(getBoolField(spout2, "fileReadCompletely"));
+ @ClassRule
+ public static MiniDFSClusterRule DFS_CLUSTER_RULE = new MiniDFSClusterRule();
+ @Rule
+ public TemporaryFolder tempFolder = new TemporaryFolder();
+ public File baseFolder;
- }
-
- @Test
- public void testResumeAbandoned_Seq_NoAck() throws Exception {
- Path file1 = new Path(source.toString() + "/file1.seq");
- createSeqFile(fs, file1, 6);
-
- final Integer lockExpirySec = 1;
-
- HdfsSpout spout = makeSpout(Configs.SEQ, SequenceFileReader.defaultFields);
- spout.setCommitFrequencyCount(1);
- spout.setCommitFrequencySec(1000); // effectively disable commits based on time
- spout.setLockTimeoutSec(lockExpirySec);
-
-
- HdfsSpout spout2 = makeSpout(Configs.SEQ, SequenceFileReader.defaultFields);
- spout2.setCommitFrequencyCount(1);
- spout2.setCommitFrequencySec(1000); // effectively disable commits based on time
- spout2.setLockTimeoutSec(lockExpirySec);
-
- Map conf = getCommonConfigs();
- openSpout(spout, 0, conf);
- openSpout(spout2, 1, conf);
-
- // consume file 1 partially
- List<String> res = runSpout(spout, "r2");
- Assert.assertEquals(2, res.size());
- // abandon file
- FileLock lock = getField(spout, "lock");
- TestFileLock.closeUnderlyingLockFile(lock);
- Thread.sleep(lockExpirySec * 2 * 1000);
-
- // check lock file presence
- Assert.assertTrue(fs.exists(lock.getLockFile()));
-
- // create another spout to take over processing and read a few lines
- List<String> res2 = runSpout(spout2, "r3");
- Assert.assertEquals(3, res2.size());
-
- // check lock file presence
- Assert.assertTrue(fs.exists(lock.getLockFile()));
-
- // check lock file contents
- List<String> contents = getTextFileContents(fs, lock.getLockFile());
- Assert.assertFalse(contents.isEmpty());
-
- // finish up reading the file
- res2 = runSpout(spout2, "r3");
- Assert.assertEquals(4, res2.size());
+ private Path source;
+ private Path archive;
+ private Path badfiles;
- // check lock file is gone
- Assert.assertFalse(fs.exists(lock.getLockFile()));
- FileReader rdr = getField(spout2, "reader");
- Assert.assertNull( rdr );
- Assert.assertTrue(getBoolField(spout2, "fileReadCompletely"));
- }
+ private static DistributedFileSystem fs;
+ private static final Configuration conf = new Configuration();
- private void checkCollectorOutput_txt(MockCollector collector, Path... txtFiles) throws IOException {
- ArrayList<String> expected = new ArrayList<>();
- for (Path txtFile : txtFiles) {
- List<String> lines= getTextFileContents(fs, txtFile);
- expected.addAll(lines);
+ @BeforeClass
+ public static void setupClass() throws IOException {
+ fs = DFS_CLUSTER_RULE.getDfscluster().getFileSystem();
}
- List<String> actual = new ArrayList<>();
- for (Pair<HdfsSpout.MessageId, List<Object>> item : collector.items) {
- actual.add(item.getValue().get(0).toString());
+ @AfterClass
+ public static void teardownClass() throws IOException {
+ fs.close();
}
- Assert.assertEquals(expected, actual);
- }
- private List<String> getTextFileContents(FileSystem fs, Path txtFile) throws IOException {
- ArrayList<String> result = new ArrayList<>();
- FSDataInputStream istream = fs.open(txtFile);
- InputStreamReader isreader = new InputStreamReader(istream,"UTF-8");
- BufferedReader reader = new BufferedReader(isreader);
+ @Before
+ public void setup() throws Exception {
+ baseFolder = tempFolder.newFolder("hdfsspout");
+ source = new Path(baseFolder.toString() + "/source");
+ fs.mkdirs(source);
+ archive = new Path(baseFolder.toString() + "/archive");
+ fs.mkdirs(archive);
+ badfiles = new Path(baseFolder.toString() + "/bad");
+ fs.mkdirs(badfiles);
+ }
- for( String line = reader.readLine(); line!=null; line = reader.readLine() ) {
- result.add(line);
+ @After
+ public void shutDown() throws IOException {
+ fs.delete(new Path(baseFolder.toString()), true);
}
- isreader.close();
- return result;
- }
+ @Test
+ public void testSimpleText_noACK() throws Exception {
+ Path file1 = new Path(source.toString() + "/file1.txt");
+ createTextFile(file1, 5);
- private void checkCollectorOutput_seq(MockCollector collector, Path... seqFiles) throws IOException {
- ArrayList<String> expected = new ArrayList<>();
- for (Path seqFile : seqFiles) {
- List<String> lines= getSeqFileContents(fs, seqFile);
- expected.addAll(lines);
- }
- Assert.assertTrue(expected.equals(collector.lines));
- }
-
- private List<String> getSeqFileContents(FileSystem fs, Path... seqFiles) throws IOException {
- ArrayList<String> result = new ArrayList<>();
-
- for (Path seqFile : seqFiles) {
- Path file = new Path(fs.getUri().toString() + seqFile.toString());
- SequenceFile.Reader reader = new SequenceFile.Reader(conf, SequenceFile.Reader.file(file));
- try {
- Writable key = (Writable) ReflectionUtils.newInstance(reader.getKeyClass(), conf);
- Writable value = (Writable) ReflectionUtils.newInstance(reader.getValueClass(), conf);
- while (reader.next(key, value)) {
- String keyValStr = Arrays.asList(key, value).toString();
- result.add(keyValStr);
+ Path file2 = new Path(source.toString() + "/file2.txt");
+ createTextFile(file2, 5);
+
+ try (AutoCloseableHdfsSpout closeableSpout = makeSpout(Configs.TEXT, TextFileReader.defaultFields)) {
+ HdfsSpout spout = closeableSpout.spout;
+ spout.setCommitFrequencyCount(1);
+ spout.setCommitFrequencySec(1);
+
+ Map conf = getCommonConfigs();
+ openSpout(spout, 0, conf);
+
+ runSpout(spout, "r11");
+
+ Path arc1 = new Path(archive.toString() + "/file1.txt");
+ Path arc2 = new Path(archive.toString() + "/file2.txt");
+ checkCollectorOutput_txt((MockCollector) spout.getCollector(), arc1, arc2);
}
- } finally {
- reader.close();
- }
- }// for
- return result;
- }
-
- private List<String> listDir(Path p) throws IOException {
- ArrayList<String> result = new ArrayList<>();
- RemoteIterator<LocatedFileStatus> fileNames = fs.listFiles(p, false);
- while ( fileNames.hasNext() ) {
- LocatedFileStatus fileStatus = fileNames.next();
- result.add(Path.getPathWithoutSchemeAndAuthority(fileStatus.getPath()).toString());
}
- return result;
- }
+ @Test
+ public void testSimpleText_ACK() throws Exception {
+ Path file1 = new Path(source.toString() + "/file1.txt");
+ createTextFile(file1, 5);
- @Test
- public void testMultipleFileConsumption_Ack() throws Exception {
- Path file1 = new Path(source.toString() + "/file1.txt");
- createTextFile(file1, 5);
+ Path file2 = new Path(source.toString() + "/file2.txt");
+ createTextFile(file2, 5);
+ try (AutoCloseableHdfsSpout closeableSpout = makeSpout(Configs.TEXT, TextFileReader.defaultFields)) {
+ HdfsSpout spout = closeableSpout.spout;
+ spout.setCommitFrequencyCount(1);
+ spout.setCommitFrequencySec(1);
- HdfsSpout spout = makeSpout(Configs.TEXT, TextFileReader.defaultFields);
- spout.setCommitFrequencyCount(1);
- spout.setCommitFrequencySec(1);
+ Map conf = getCommonConfigs();
+ conf.put(Config.TOPOLOGY_ACKER_EXECUTORS, "1"); // enable ACKing
+ openSpout(spout, 0, conf);
- Map conf = getCommonConfigs();
- conf.put(Config.TOPOLOGY_ACKER_EXECUTORS, "1"); // enable ACKing
- openSpout(spout, 0, conf);
+ // consume file 1
+ runSpout(spout, "r6", "a0", "a1", "a2", "a3", "a4");
+ Path arc1 = new Path(archive.toString() + "/file1.txt");
+ checkCollectorOutput_txt((MockCollector) spout.getCollector(), arc1);
- // read few lines from file1 dont ack
- runSpout(spout, "r3");
- FileReader reader = getField(spout, "reader");
- Assert.assertNotNull(reader);
- Assert.assertEquals(false, getBoolField(spout, "fileReadCompletely"));
+ // consume file 2
+ runSpout(spout, "r6", "a5", "a6", "a7", "a8", "a9");
+ Path arc2 = new Path(archive.toString() + "/file2.txt");
+ checkCollectorOutput_txt((MockCollector) spout.getCollector(), arc1, arc2);
+ }
+ }
- // read remaining lines
- runSpout(spout, "r3");
- reader = getField(spout, "reader");
- Assert.assertNotNull(reader);
- Assert.assertEquals(true, getBoolField(spout, "fileReadCompletely") );
+ @Test
+ public void testResumeAbandoned_Text_NoAck() throws Exception {
+ Path file1 = new Path(source.toString() + "/file1.txt");
+ createTextFile(file1, 6);
+
+ final Integer lockExpirySec = 1;
+
+ try (AutoCloseableHdfsSpout closeableSpout = makeSpout(Configs.TEXT, TextFileReader.defaultFields)) {
+ HdfsSpout spout = closeableSpout.spout;
+ spout.setCommitFrequencyCount(1);
+ spout.setCommitFrequencySec(1000); // effectively disable commits based on time
+ spout.setLockTimeoutSec(lockExpirySec);
+
+ try (AutoCloseableHdfsSpout closeableSpout2 = makeSpout(Configs.TEXT, TextFileReader.defaultFields)) {
+ HdfsSpout spout2 = closeableSpout2.spout;
+ spout2.setCommitFrequencyCount(1);
+ spout2.setCommitFrequencySec(1000); // effectively disable commits based on time
+ spout2.setLockTimeoutSec(lockExpirySec);
+
+ Map conf = getCommonConfigs();
+ openSpout(spout, 0, conf);
+ openSpout(spout2, 1, conf);
+
+ // consume file 1 partially
+ List<String> res = runSpout(spout, "r2");
+ Assert.assertEquals(2, res.size());
+
+ // abandon file
+ FileLock lock = getField(spout, "lock");
+ TestFileLock.closeUnderlyingLockFile(lock);
+ Thread.sleep(lockExpirySec * 2 * 1000);
+
+ // check lock file presence
+ Assert.assertTrue(fs.exists(lock.getLockFile()));
+
+ // create another spout to take over processing and read a few lines
+ List<String> res2 = runSpout(spout2, "r3");
+ Assert.assertEquals(3, res2.size());
+
+ // check lock file presence
+ Assert.assertTrue(fs.exists(lock.getLockFile()));
+
+ // check lock file contents
+ List<String> contents = readTextFile(fs, lock.getLockFile().toString());
+ Assert.assertFalse(contents.isEmpty());
+
+ // finish up reading the file
+ res2 = runSpout(spout2, "r2");
+ Assert.assertEquals(4, res2.size());
+
+ // check lock file is gone
+ Assert.assertFalse(fs.exists(lock.getLockFile()));
+ FileReader rdr = getField(spout2, "reader");
+ Assert.assertNull(rdr);
+ Assert.assertTrue(getBoolField(spout2, "fileReadCompletely"));
+ }
+ }
+ }
- // ack few
- runSpout(spout, "a0", "a1", "a2");
- reader = getField(spout, "reader");
- Assert.assertNotNull(reader);
- Assert.assertEquals(true, getBoolField(spout, "fileReadCompletely"));
+ @Test
+ public void testResumeAbandoned_Seq_NoAck() throws Exception {
+ Path file1 = new Path(source.toString() + "/file1.seq");
+ createSeqFile(fs, file1, 6);
+
+ final Integer lockExpirySec = 1;
+
+ try (AutoCloseableHdfsSpout closeableSpout = makeSpout(Configs.SEQ, SequenceFileReader.defaultFields)) {
+ HdfsSpout spout = closeableSpout.spout;
+ spout.setCommitFrequencyCount(1);
+ spout.setCommitFrequencySec(1000); // effectively disable commits based on time
+ spout.setLockTimeoutSec(lockExpirySec);
+
+ try (AutoCloseableHdfsSpout closeableSpout2 = makeSpout(Configs.SEQ, SequenceFileReader.defaultFields)) {
+ HdfsSpout spout2 = closeableSpout2.spout;
+ spout2.setCommitFrequencyCount(1);
+ spout2.setCommitFrequencySec(1000); // effectively disable commits based on time
+ spout2.setLockTimeoutSec(lockExpirySec);
+
+ Map conf = getCommonConfigs();
+ openSpout(spout, 0, conf);
+ openSpout(spout2, 1, conf);
+
+ // consume file 1 partially
+ List<String> res = runSpout(spout, "r2");
+ Assert.assertEquals(2, res.size());
+ // abandon file
+ FileLock lock = getField(spout, "lock");
+ TestFileLock.closeUnderlyingLockFile(lock);
+ Thread.sleep(lockExpirySec * 2 * 1000);
+
+ // check lock file presence
+ Assert.assertTrue(fs.exists(lock.getLockFile()));
+
+ // create another spout to take over processing and read a few lines
+ List<String> res2 = runSpout(spout2, "r3");
+ Assert.assertEquals(3, res2.size());
+
+ // check lock file presence
+ Assert.assertTrue(fs.exists(lock.getLockFile()));
+
+ // check lock file contents
+ List<String> contents = getTextFileContents(fs, lock.getLockFile());
+ Assert.assertFalse(contents.isEmpty());
+
+ // finish up reading the file
+ res2 = runSpout(spout2, "r3");
+ Assert.assertEquals(4, res2.size());
+
+ // check lock file is gone
+ Assert.assertFalse(fs.exists(lock.getLockFile()));
+ FileReader rdr = getField(spout2, "reader");
+ Assert.assertNull(rdr);
+ Assert.assertTrue(getBoolField(spout2, "fileReadCompletely"));
+ }
+ }
+ }
- //ack rest
- runSpout(spout, "a3", "a4");
- reader = getField(spout, "reader");
- Assert.assertNull(reader);
- Assert.assertEquals(true, getBoolField(spout, "fileReadCompletely"));
+ private void checkCollectorOutput_txt(MockCollector collector, Path... txtFiles) throws IOException {
+ ArrayList<String> expected = new ArrayList<>();
+ for (Path txtFile : txtFiles) {
+ List<String> lines = getTextFileContents(fs, txtFile);
+ expected.addAll(lines);
+ }
+ List<String> actual = new ArrayList<>();
+ for (Pair<HdfsSpout.MessageId, List<Object>> item : collector.items) {
+ actual.add(item.getValue().get(0).toString());
+ }
+ Assert.assertEquals(expected, actual);
+ }
- // go to next file
- Path file2 = new Path(source.toString() + "/file2.txt");
- createTextFile(file2, 5);
+ private List<String> getTextFileContents(FileSystem fs, Path txtFile) throws IOException {
+ ArrayList<String> result = new ArrayList<>();
+ FSDataInputStream istream = fs.open(txtFile);
+ InputStreamReader isreader = new InputStreamReader(istream, "UTF-8");
+ BufferedReader reader = new BufferedReader(isreader);
- // Read 1 line
- runSpout(spout, "r1");
- Assert.assertNotNull(getField(spout, "reader"));
- Assert.assertEquals(false, getBoolField(spout, "fileReadCompletely"));
+ for (String line = reader.readLine(); line != null; line = reader.readLine()) {
+ result.add(line);
+ }
+ isreader.close();
+ return result;
+ }
- // ack 1 tuple
- runSpout(spout, "a5");
- Assert.assertNotNull(getField(spout, "reader"));
- Assert.assertEquals(false, getBoolField(spout, "fileReadCompletely"));
+ private void checkCollectorOutput_seq(MockCollector collector, Path... seqFiles) throws IOException {
+ ArrayList<String> expected = new ArrayList<>();
+ for (Path seqFile : seqFiles) {
+ List<String> lines = getSeqFileContents(fs, seqFile);
+ expected.addAll(lines);
+ }
+ Assert.assertTrue(expected.equals(collector.lines));
+ }
+ private List<String> getSeqFileContents(FileSystem fs, Path... seqFiles) throws IOException {
+ ArrayList<String> result = new ArrayList<>();
+
+ for (Path seqFile : seqFiles) {
+ Path file = new Path(fs.getUri().toString() + seqFile.toString());
+ SequenceFile.Reader reader = new SequenceFile.Reader(conf, SequenceFile.Reader.file(file));
+ try {
+ Writable key = (Writable) ReflectionUtils.newInstance(reader.getKeyClass(), conf);
+ Writable value = (Writable) ReflectionUtils.newInstance(reader.getValueClass(), conf);
+ while (reader.next(key, value)) {
+ String keyValStr = Arrays.asList(key, value).toString();
+ result.add(keyValStr);
+ }
+ } finally {
+ reader.close();
+ }
+ }// for
+ return result;
+ }
- // read and ack remaining lines
- runSpout(spout, "r5", "a6", "a7", "a8", "a9");
- Assert.assertNull(getField(spout, "reader"));
- Assert.assertEquals(true, getBoolField(spout, "fileReadCompletely"));
- }
+ private List<String> listDir(Path p) throws IOException {
+ ArrayList<String> result = new ArrayList<>();
+ RemoteIterator<LocatedFileStatus> fileNames = fs.listFiles(p, false);
+ while (fileNames.hasNext()) {
+ LocatedFileStatus fileStatus = fileNames.next();
+ result.add(Path.getPathWithoutSchemeAndAuthority(fileStatus.getPath()).toString());
+ }
+ return result;
+ }
- private static <T> T getField(HdfsSpout spout, String fieldName) throws NoSuchFieldException, IllegalAccessException {
- Field readerFld = HdfsSpout.class.getDeclaredField(fieldName);
- readerFld.setAccessible(true);
- return (T) readerFld.get(spout);
- }
+ @Test
+ public void testMultipleFileConsumption_Ack() throws Exception {
+ Path file1 = new Path(source.toString() + "/file1.txt");
+ createTextFile(file1, 5);
+
+ try (AutoCloseableHdfsSpout closeableSpout = makeSpout(Configs.TEXT, TextFileReader.defaultFields)) {
+ HdfsSpout spout = closeableSpout.spout;
+ spout.setCommitFrequencyCount(1);
+ spout.setCommitFrequencySec(1);
+
+ Map conf = getCommonConfigs();
+ conf.put(Config.TOPOLOGY_ACKER_EXECUTORS, "1"); // enable ACKing
+ openSpout(spout, 0, conf);
+
+ // read few lines from file1 dont ack
+ runSpout(spout, "r3");
+ FileReader reader = getField(spout, "reader");
+ Assert.assertNotNull(reader);
+ Assert.assertEquals(false, getBoolField(spout, "fileReadCompletely"));
+
+ // read remaining lines
+ runSpout(spout, "r3");
+ reader = getField(spout, "reader");
+ Assert.assertNotNull(reader);
+ Assert.assertEquals(true, getBoolField(spout, "fileReadCompletely"));
+
+ // ack few
+ runSpout(spout, "a0", "a1", "a2");
+ reader = getField(spout, "reader");
+ Assert.assertNotNull(reader);
+ Assert.assertEquals(true, getBoolField(spout, "fileReadCompletely"));
+
+ //ack rest
+ runSpout(spout, "a3", "a4");
+ reader = getField(spout, "reader");
+ Assert.assertNull(reader);
+ Assert.assertEquals(true, getBoolField(spout, "fileReadCompletely"));
+
+ // go to next file
+ Path file2 = new Path(source.toString() + "/file2.txt");
+ createTextFile(file2, 5);
+
+ // Read 1 line
+ runSpout(spout, "r1");
+ Assert.assertNotNull(getField(spout, "reader"));
+ Assert.assertEquals(false, getBoolField(spout, "fileReadCompletely"));
+
+ // ack 1 tuple
+ runSpout(spout, "a5");
+ Assert.assertNotNull(getField(spout, "reader"));
+ Assert.assertEquals(false, getBoolField(spout, "fileReadCompletely"));
+
+ // read and ack remaining lines
+ runSpout(spout, "r5", "a6", "a7", "a8", "a9");
+ Assert.assertNull(getField(spout, "reader"));
+ Assert.assertEquals(true, getBoolField(spout, "fileReadCompletely"));
+ }
+ }
- private static boolean getBoolField(HdfsSpout spout, String fieldName) throws NoSuchFieldException, IllegalAccessException {
- Field readerFld = HdfsSpout.class.getDeclaredField(fieldName);
- readerFld.setAccessible(true);
- return readerFld.getBoolean(spout);
- }
+ private static <T> T getField(HdfsSpout spout, String fieldName) throws NoSuchFieldException, IllegalAccessException {
+ Field readerFld = HdfsSpout.class.getDeclaredField(fieldName);
+ readerFld.setAccessible(true);
+ return (T) readerFld.get(spout);
+ }
+ private static boolean getBoolField(HdfsSpout spout, String fieldName) throws NoSuchFieldException, IllegalAccessException {
+ Field readerFld = HdfsSpout.class.getDeclaredField(fieldName);
+ readerFld.setAccessible(true);
+ return readerFld.getBoolean(spout);
+ }
- @Test
- public void testSimpleSequenceFile() throws IOException {
- //1) create a couple files to consume
- source = new Path("/tmp/hdfsspout/source");
- fs.mkdirs(source);
- archive = new Path("/tmp/hdfsspout/archive");
- fs.mkdirs(archive);
+ @Test
+ public void testSimpleSequenceFile() throws Exception {
+ //1) create a couple files to consume
+ source = new Path("/tmp/hdfsspout/source");
+ fs.mkdirs(source);
+ archive = new Path("/tmp/hdfsspout/archive");
+ fs.mkdirs(archive);
- Path file1 = new Path(source + "/file1.seq");
- createSeqFile(fs, file1, 5);
+ Path file1 = new Path(source + "/file1.seq");
+ createSeqFile(fs, file1, 5);
- Path file2 = new Path(source + "/file2.seq");
- createSeqFile(fs, file2, 5);
+ Path file2 = new Path(source + "/file2.seq");
+ createSeqFile(fs, file2, 5);
+ try (AutoCloseableHdfsSpout closeableSpout = makeSpout(Configs.SEQ, SequenceFileReader.defaultFields)) {
+ HdfsSpout spout = closeableSpout.spout;
+ Map conf = getCommonConfigs();
+ openSpout(spout, 0, conf);
- HdfsSpout spout = makeSpout(Configs.SEQ, SequenceFileReader.defaultFields);
- Map conf = getCommonConfigs();
- openSpout(spout, 0, conf);
+ // consume both files
+ List<String> res = runSpout(spout, "r11");
+ Assert.assertEquals(10, res.size());
- // consume both files
- List<String> res = runSpout(spout, "r11");
- Assert.assertEquals(10, res.size());
+ Assert.assertEquals(2, listDir(archive).size());
- Assert.assertEquals(2, listDir(archive).size());
+ Path f1 = new Path(archive + "/file1.seq");
+ Path f2 = new Path(archive + "/file2.seq");
+ checkCollectorOutput_seq((MockCollector) spout.getCollector(), f1, f2);
+ }
+ }
- Path f1 = new Path(archive + "/file1.seq");
- Path f2 = new Path(archive + "/file2.seq");
+ @Test
+ public void testReadFailures() throws Exception {
+ // 1) create couple of input files to read
+ Path file1 = new Path(source.toString() + "/file1.txt");
+ Path file2 = new Path(source.toString() + "/file2.txt");
+
+ createTextFile(file1, 6);
+ createTextFile(file2, 7);
+ Assert.assertEquals(2, listDir(source).size());
+
+ // 2) run spout
+ try (AutoCloseableHdfsSpout closeableSpout = makeSpout(MockTextFailingReader.class.getName(), MockTextFailingReader.defaultFields)) {
+ HdfsSpout spout = closeableSpout.spout;
+ Map conf = getCommonConfigs();
+ openSpout(spout, 0, conf);
+
+ List<String> res = runSpout(spout, "r11");
+ String[] expected = new String[]{"[line 0]", "[line 1]", "[line 2]", "[line 0]", "[line 1]", "[line 2]"};
+ Assert.assertArrayEquals(expected, res.toArray());
+
+ // 3) make sure 6 lines (3 from each file) were read in all
+ Assert.assertEquals(((MockCollector) spout.getCollector()).lines.size(), 6);
+ ArrayList<Path> badFiles = HdfsUtils.listFilesByModificationTime(fs, badfiles, 0);
+ Assert.assertEquals(badFiles.size(), 2);
+ }
+ }
- checkCollectorOutput_seq((MockCollector) spout.getCollector(), f1, f2);
- }
+ // check lock creation/deletion and contents
+ @Test
+ public void testLocking() throws Exception {
+ Path file1 = new Path(source.toString() + "/file1.txt");
+ createTextFile(file1, 10);
+
+ // 0) config spout to log progress in lock file for each tuple
+ try (AutoCloseableHdfsSpout closeableSpout = makeSpout(Configs.TEXT, TextFileReader.defaultFields)) {
+ HdfsSpout spout = closeableSpout.spout;
+ spout.setCommitFrequencyCount(1);
+ spout.setCommitFrequencySec(1000); // effectively disable commits based on time
+
+ Map conf = getCommonConfigs();
+ openSpout(spout, 0, conf);
+
+ // 1) read initial lines in file, then check if lock exists
+ List<String> res = runSpout(spout, "r5");
+ Assert.assertEquals(5, res.size());
+ List<String> lockFiles = listDir(spout.getLockDirPath());
+ Assert.assertEquals(1, lockFiles.size());
+
+ // 2) check log file content line count == tuples emitted + 1
+ List<String> lines = readTextFile(fs, lockFiles.get(0));
+ Assert.assertEquals(lines.size(), res.size() + 1);
+
+ // 3) read remaining lines in file, then ensure lock is gone
+ runSpout(spout, "r6");
+ lockFiles = listDir(spout.getLockDirPath());
+ Assert.assertEquals(0, lockFiles.size());
+
+ // 4) --- Create another input file and reverify same behavior ---
+ Path file2 = new Path(source.toString() + "/file2.txt");
+ createTextFile(file2, 10);
+
+ // 5) read initial lines in file, then check if lock exists
+ res = runSpout(spout, "r5");
+ Assert.assertEquals(15, res.size());
+ lockFiles = listDir(spout.getLockDirPath());
+ Assert.assertEquals(1, lockFiles.size());
+
+ // 6) check log file content line count == tuples emitted + 1
+ lines = readTextFile(fs, lockFiles.get(0));
+ Assert.assertEquals(6, lines.size());
+
+ // 7) read remaining lines in file, then ensure lock is gone
+ runSpout(spout, "r6");
+ lockFiles = listDir(spout.getLockDirPath());
+ Assert.assertEquals(0, lockFiles.size());
+ }
+ }
- @Test
- public void testReadFailures() throws Exception {
- // 1) create couple of input files to read
- Path file1 = new Path(source.toString() + "/file1.txt");
- Path file2 = new Path(source.toString() + "/file2.txt");
-
- createTextFile(file1, 6);
- createTextFile(file2, 7);
- Assert.assertEquals(2, listDir(source).size());
-
- // 2) run spout
- HdfsSpout spout = makeSpout(MockTextFailingReader.class.getName(), MockTextFailingReader.defaultFields);
- Map conf = getCommonConfigs();
- openSpout(spout, 0, conf);
+ @Test
+ public void testLockLoggingFreqCount() throws Exception {
+ Path file1 = new Path(source.toString() + "/file1.txt");
+ createTextFile(file1, 10);
- List<String> res = runSpout(spout, "r11");
- String[] expected = new String[] {"[line 0]","[line 1]","[line 2]","[line 0]","[line 1]","[line 2]"};
- Assert.assertArrayEquals(expected, res.toArray());
+ // 0) config spout to log progress in lock file for each tuple
+ try (AutoCloseableHdfsSpout closeableSpout = makeSpout(Configs.TEXT, TextFileReader.defaultFields)) {
+ HdfsSpout spout = closeableSpout.spout;
+ spout.setCommitFrequencyCount(2); // 1 lock log entry every 2 tuples
+ spout.setCommitFrequencySec(1000); // Effectively disable commits based on time
- // 3) make sure 6 lines (3 from each file) were read in all
- Assert.assertEquals(((MockCollector) spout.getCollector()).lines.size(), 6);
- ArrayList<Path> badFiles = HdfsUtils.listFilesByModificationTime(fs, badfiles, 0);
- Assert.assertEquals(badFiles.size(), 2);
- }
+ Map conf = getCommonConfigs();
+ openSpout(spout, 0, conf);
- // check lock creation/deletion and contents
- @Test
- public void testLocking() throws Exception {
- Path file1 = new Path(source.toString() + "/file1.txt");
- createTextFile(file1, 10);
+ // 1) read 5 lines in file,
+ runSpout(spout, "r5");
- // 0) config spout to log progress in lock file for each tuple
+ // 2) check log file contents
+ String lockFile = listDir(spout.getLockDirPath()).get(0);
+ List<String> lines = readTextFile(fs, lockFile);
+ Assert.assertEquals(lines.size(), 3);
- HdfsSpout spout = makeSpout(Configs.TEXT, TextFileReader.defaultFields);
- spout.setCommitFrequencyCount(1);
- spout.setCommitFrequencySec(1000); // effectively disable commits based on time
-
-
- Map conf = getCommonConfigs();
- openSpout(spout, 0, conf);
-
- // 1) read initial lines in file, then check if lock exists
- List<String> res = runSpout(spout, "r5");
- Assert.assertEquals(5, res.size());
- List<String> lockFiles = listDir(spout.getLockDirPath());
- Assert.assertEquals(1, lockFiles.size());
-
- // 2) check log file content line count == tuples emitted + 1
- List<String> lines = readTextFile(fs, lockFiles.get(0));
- Assert.assertEquals(lines.size(), res.size()+1);
-
- // 3) read remaining lines in file, then ensure lock is gone
- runSpout(spout, "r6");
- lockFiles = listDir(spout.getLockDirPath());
- Assert.assertEquals(0, lockFiles.size());
-
-
- // 4) --- Create another input file and reverify same behavior ---
- Path file2 = new Path(source.toString() + "/file2.txt");
- createTextFile(file2, 10);
-
- // 5) read initial lines in file, then check if lock exists
- res = runSpout(spout, "r5");
- Assert.assertEquals(15, res.size());
- lockFiles = listDir(spout.getLockDirPath());
- Assert.assertEquals(1, lockFiles.size());
-
- // 6) check log file content line count == tuples emitted + 1
- lines = readTextFile(fs, lockFiles.get(0));
- Assert.assertEquals(6, lines.size());
-
- // 7) read remaining lines in file, then ensure lock is gone
- runSpout(spout, "r6");
- lockFiles = listDir(spout.getLockDirPath());
- Assert.assertEquals(0, lockFiles.size());
- }
-
- @Test
- public void testLockLoggingFreqCount() throws Exception {
- Path file1 = new Path(source.toString() + "/file1.txt");
- createTextFile(file1, 10);
-
- // 0) config spout to log progress in lock file for each tuple
-
- HdfsSpout spout = makeSpout(Configs.TEXT, TextFileReader.defaultFields);
- spout.setCommitFrequencyCount(2); // 1 lock log entry every 2 tuples
- spout.setCommitFrequencySec(1000); // Effectively disable commits based on time
-
- Map conf = getCommonConfigs();
- openSpout(spout, 0, conf);
-
- // 1) read 5 lines in file,
- runSpout(spout, "r5");
-
- // 2) check log file contents
- String lockFile = listDir(spout.getLockDirPath()).get(0);
- List<String> lines = readTextFile(fs, lockFile);
- Assert.assertEquals(lines.size(), 3);
-
- // 3) read 6th line and see if another log entry was made
- runSpout(spout, "r1");
- lines = readTextFile(fs, lockFile);
- Assert.assertEquals(lines.size(), 4);
- }
-
- @Test
- public void testLockLoggingFreqSec() throws Exception {
- Path file1 = new Path(source.toString() + "/file1.txt");
- createTextFile(file1, 10);
-
- // 0) config spout to log progress in lock file for each tuple
- HdfsSpout spout = makeSpout(Configs.TEXT, TextFileReader.defaultFields);
- spout.setCommitFrequencyCount(0); // disable it
- spout.setCommitFrequencySec(2); // log every 2 sec
-
- Map conf = getCommonConfigs();
- openSpout(spout, 0, conf);
-
- // 1) read 5 lines in file
- runSpout(spout, "r5");
-
- // 2) check log file contents
- String lockFile = listDir(spout.getLockDirPath()).get(0);
- List<String> lines = readTextFile(fs, lockFile);
- Assert.assertEquals(lines.size(), 1);
- Thread.sleep(3000); // allow freq_sec to expire
-
- // 3) read another line and see if another log entry was made
- runSpout(spout, "r1");
- lines = readTextFile(fs, lockFile);
- Assert.assertEquals(2, lines.size());
- }
-
- private static List<String> readTextFile(FileSystem fs, String f) throws IOException {
- Path file = new Path(f);
- FSDataInputStream x = fs.open(file);
- BufferedReader reader = new BufferedReader(new InputStreamReader(x));
- String line = null;
- ArrayList<String> result = new ArrayList<>();
- while( (line = reader.readLine()) !=null )
- result.add( line );
- return result;
- }
-
-
- private Map getCommonConfigs() {
- Map conf = new HashMap();
- conf.put(Config.TOPOLOGY_ACKER_EXECUTORS, "0");
- return conf;
- }
-
- private HdfsSpout makeSpout(String readerType, String[] outputFields) {
- HdfsSpout spout = new HdfsSpout().withOutputFields(outputFields)
- .setReaderType(readerType)
- .setHdfsUri(hdfsCluster.getURI().toString())
- .setSourceDir(source.toString())
- .setArchiveDir(archive.toString())
- .setBadFilesDir(badfiles.toString());
-
- return spout;
- }
-
- private void openSpout(HdfsSpout spout, int spoutId, Map conf) {
- MockCollector collector = new MockCollector();
- spout.open(conf, new MockTopologyContext(spoutId), collector);
- }
-
- /**
- * Execute a sequence of calls on HdfsSpout.
- *
- * @param cmds: set of commands to run,
- * e.g. "r,r,r,r,a1,f2,...". The commands are:
- * r[N] - receive() called N times
- * aN - ack, item number: N
- * fN - fail, item number: N
- */
-
- private List<String> runSpout(HdfsSpout spout, String... cmds) {
- MockCollector collector = (MockCollector) spout.getCollector();
- for(String cmd : cmds) {
- if(cmd.startsWith("r")) {
- int count = 1;
- if(cmd.length() > 1) {
- count = Integer.parseInt(cmd.substring(1));
- }
- for(int i=0; i<count; ++i) {
- spout.nextTuple();
- }
+ // 3) read 6th line and see if another log entry was made
+ runSpout(spout, "r1");
+ lines = readTextFile(fs, lockFile);
+ Assert.assertEquals(lines.size(), 4);
}
- else if(cmd.startsWith("a")) {
- int n = Integer.parseInt(cmd.substring(1));
- Pair<HdfsSpout.MessageId, List<Object>> item = collector.items.get(n);
- spout.ack(item.getKey());
+ }
+
+ @Test
+ public void testLockLoggingFreqSec() throws Exception {
+ Path file1 = new Path(source.toString() + "/file1.txt");
+ createTextFile(file1, 10);
+
+ // 0) config spout to log progress in lock file for each tuple
+ try (AutoCloseableHdfsSpout closeableSpout = makeSpout(Configs.TEXT, TextFileReader.defaultFields)) {
+ HdfsSpout spout = closeableSpout.spout;
+ spout.setCommitFrequencyCount(0); // disable it
+ spout.setCommitFrequencySec(2); // log every 2 sec
+
+ Map conf = getCommonConfigs();
+ openSpout(spout, 0, conf);
+
+ // 1) read 5 lines in file
+ runSpout(spout, "r5");
+
+ // 2) check log file contents
+ String lockFile = listDir(spout.getLockDirPath()).get(0);
+ List<String> lines = readTextFile(fs, lockFile);
+ Assert.assertEquals(lines.size(), 1);
+ Thread.sleep(3000); // allow freq_sec to expire
+
+ // 3) read another line and see if another log entry was made
+ runSpout(spout, "r1");
+ lines = readTextFile(fs, lockFile);
+ Assert.assertEquals(2, lines.size());
}
- else if(cmd.startsWith("f")) {
- int n = Integer.parseInt(cmd.substring(1));
- Pair<HdfsSpout.MessageId, List<Object>> item = collector.items.get(n);
- spout.fail(item.getKey());
+ }
+
+ private static List<String> readTextFile(FileSystem fs, String f) throws IOException {
+ Path file = new Path(f);
+ FSDataInputStream x = fs.open(file);
+ BufferedReader reader = new BufferedReader(new InputStreamReader(x));
+ String line = null;
+ ArrayList<String> result = new ArrayList<>();
+ while ((line = reader.readLine()) != null) {
+ result.add(line);
}
- }
- return collector.lines;
+ return result;
}
- private void createTextFile(Path file, int lineCount) throws IOException {
- FSDataOutputStream os = fs.create(file);
- int size = 0;
- for (int i = 0; i < lineCount; i++) {
- os.writeBytes("line " + i + System.lineSeparator());
- String msg = "line " + i + System.lineSeparator();
- size += msg.getBytes().length;
+ private Map getCommonConfigs() {
+ Map conf = new HashMap();
+ conf.put(Config.TOPOLOGY_ACKER_EXECUTORS, "0");
+ return conf;
}
- os.close();
- }
+ private AutoCloseableHdfsSpout makeSpout(String readerType, String[] outputFields) {
+ HdfsSpout spout = new HdfsSpout().withOutputFields(outputFields)
+ .setReaderType(readerType)
+ .setHdfsUri(DFS_CLUSTER_RULE.getDfscluster().getURI().toString())
+ .setSourceDir(source.toString())
+ .setArchiveDir(archive.toString())
+ .setBadFilesDir(badfiles.toString());
+ return new AutoCloseableHdfsSpout(spout);
+ }
- private static void createSeqFile(FileSystem fs, Path file, int rowCount) throws IOException {
+ private static class AutoCloseableHdfsSpout implements AutoCloseable {
- Configuration conf = new Configuration();
- try {
- if(fs.exists(file)) {
- fs.delete(file, false);
- }
+ private final HdfsSpout spout;
- SequenceFile.Writer w = SequenceFile.createWriter(fs, conf, file, IntWritable.class, Text.class );
- for (int i = 0; i < rowCount; i++) {
- w.append(new IntWritable(i), new Text("line " + i));
- }
- w.close();
- System.out.println("done");
- } catch (IOException e) {
- e.printStackTrace();
+ public AutoCloseableHdfsSpout(HdfsSpout spout) {
+ this.spout = spout;
+ }
+ @Override
+ public void close() throws Exception {
+ spout.close();
+ }
}
- }
-
+ private void openSpout(HdfsSpout spout, int spoutId, Map conf) {
+ MockCollector collector = new MockCollector();
+ spout.open(conf, new MockTopologyContext(spoutId), collector);
+ }
- static class MockCollector extends SpoutOutputCollector {
- //comma separated offsets
- public ArrayList<String> lines;
- public ArrayList<Pair<HdfsSpout.MessageId, List<Object> > > items;
+ /**
+ * Execute a sequence of calls on HdfsSpout.
+ *
+ * @param cmds: set of commands to run, e.g. "r,r,r,r,a1,f2,...". The commands are: r[N] - receive() called N times aN - ack, item
+ * number: N fN - fail, item number: N
+ */
+ private List<String> runSpout(HdfsSpout spout, String... cmds) {
+ MockCollector collector = (MockCollector) spout.getCollector();
+ for (String cmd : cmds) {
+ if (cmd.startsWith("r")) {
+ int count = 1;
+ if (cmd.length() > 1) {
+ count = Integer.parseInt(cmd.substring(1));
+ }
+ for (int i = 0; i < count; ++i) {
+ spout.nextTuple();
+ }
+ } else if (cmd.startsWith("a")) {
+ int n = Integer.parseInt(cmd.substring(1));
+ Pair<HdfsSpout.MessageId, List<Object>> item = collector.items.get(n);
+ spout.ack(item.getKey());
+ } else if (cmd.startsWith("f")) {
+ int n = Integer.parseInt(cmd.substring(1));
+ Pair<HdfsSpout.MessageId, List<Object>> item = collector.items.get(n);
+ spout.fail(item.getKey());
+ }
+ }
+ return collector.lines;
+ }
- public MockCollector() {
- super(null);
- lines = new ArrayList<>();
- items = new ArrayList<>();
+ private void createTextFile(Path file, int lineCount) throws IOException {
+ FSDataOutputStream os = fs.create(file);
+ int size = 0;
+ for (int i = 0; i < lineCount; i++) {
+ os.writeBytes("line " + i + System.lineSeparator());
+ String msg = "line " + i + System.lineSeparator();
+ size += msg.getBytes().length;
+ }
+ os.close();
}
+ private static void createSeqFile(FileSystem fs, Path file, int rowCount) throws IOException {
+ Configuration conf = new Configuration();
+ try {
+ if (fs.exists(file)) {
+ fs.delete(file, false);
+ }
- @Override
- public List<Integer> emit(String streamId, List<Object> tuple, Object messageId) {
- lines.add(tuple.toString());
- items.add(HdfsUtils.Pair.of(messageId, tuple));
- return null;
- }
+ SequenceFile.Writer w = SequenceFile.createWriter(fs, conf, file, IntWritable.class, Text.class);
+ for (int i = 0; i < rowCount; i++) {
+ w.append(new IntWritable(i), new Text("line " + i));
+ }
+ w.close();
+ System.out.println("done");
+ } catch (IOException e) {
+ e.printStackTrace();
- @Override
- public void emitDirect(int arg0, String arg1, List<Object> arg2, Object arg3) {
- throw new UnsupportedOperationException("NOT Implemented");
+ }
}
- @Override
- public void reportError(Throwable arg0) {
- throw new UnsupportedOperationException("NOT Implemented");
- }
+ static class MockCollector extends SpoutOutputCollector {
+ //comma separated offsets
- @Override
- public long getPendingCount() {
- return 0;
- }
- } // class MockCollector
+ public ArrayList<String> lines;
+ public ArrayList<Pair<HdfsSpout.MessageId, List<Object>>> items;
+
+ public MockCollector() {
+ super(null);
+ lines = new ArrayList<>();
+ items = new ArrayList<>();
+ }
+ @Override
+ public List<Integer> emit(String streamId, List<Object> tuple, Object messageId) {
+ lines.add(tuple.toString());
+ items.add(HdfsUtils.Pair.of(messageId, tuple));
+ return null;
+ }
+ @Override
+ public void emitDirect(int arg0, String arg1, List<Object> arg2, Object arg3) {
+ throw new UnsupportedOperationException("NOT Implemented");
+ }
- // Throws IOExceptions for 3rd & 4th call to next(), succeeds on 5th, thereafter
- // throws ParseException. Effectively produces 3 lines (1,2 & 3) from each file read
- static class MockTextFailingReader extends TextFileReader {
- public static final String[] defaultFields = {"line"};
- int readAttempts = 0;
+ @Override
+ public void reportError(Throwable arg0) {
+ throw new UnsupportedOperationException("NOT Implemented");
+ }
- public MockTextFailingReader(FileSystem fs, Path file, Map conf) throws IOException {
- super(fs, file, conf);
- }
+ @Override
+ public long getPendingCount() {
+ return 0;
+ }
+ } // class MockCollector
- @Override
- public List<Object> next() throws IOException, ParseException {
- readAttempts++;
- if (readAttempts == 3 || readAttempts ==4) {
- throw new IOException("mock test exception");
- } else if (readAttempts > 5 ) {
- throw new ParseException("mock test exception", null);
- }
- return super.next();
- }
- }
+ // Throws IOExceptions for 3rd & 4th call to next(), succeeds on 5th, thereafter
+ // throws ParseException. Effectively produces 3 lines (1,2 & 3) from each file read
+ static class MockTextFailingReader extends TextFileReader {
- static class MockTopologyContext extends TopologyContext {
- private final int componentId;
+ public static final String[] defaultFields = {"line"};
+ int readAttempts = 0;
- public MockTopologyContext(int componentId) {
- // StormTopology topology, Map stormConf, Map<Integer, String> taskToComponent, Map<String, List<Integer>> componentToSortedTasks, Map<String, Map<String, Fields>> componentToStreamToFields, String stormId, String codeDir, String pidDir, Integer taskId, Integer workerPort, List<Integer> workerTasks, Map<String, Object> defaultResources, Map<String, Object> userResources, Map<String, Object> executorData, Map<Integer, Map<Integer, Map<String, IMetric>>> registeredMetrics, Atom openOrPrepareWasCalled
- super(null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null);
- this.componentId = componentId;
- }
+ public MockTextFailingReader(FileSystem fs, Path file, Map conf) throws IOException {
+ super(fs, file, conf);
+ }
- public String getThisComponentId() {
- return Integer.toString( componentId );
+ @Override
+ public List<Object> next() throws IOException, ParseException {
+ readAttempts++;
+ if (readAttempts == 3 || readAttempts == 4) {
+ throw new IOException("mock test exception");
+ } else if (readAttempts > 5) {
+ throw new ParseException("mock test exception", null);
+ }
+ return super.next();
+ }
}
- }
+ static class MockTopologyContext extends TopologyContext {
+
+ private final int componentId;
+
+ public MockTopologyContext(int componentId) {
+ // StormTopology topology, Map stormConf, Map<Integer, String> taskToComponent, Map<String, List<Integer>> componentToSortedTasks, Map<String, Map<String, Fields>> componentToStreamToFields, String stormId, String codeDir, String pidDir, Integer taskId, Integer workerPort, List<Integer> workerTasks, Map<String, Object> defaultResources, Map<String, Object> userResources, Map<String, Object> executorData, Map<Integer, Map<Integer, Map<String, IMetric>>> registeredMetrics, Atom openOrPrepareWasCalled
+ super(null, null, null, null, null, null, null, null, null, null, null, null, null, null, null, null);
+ this.componentId = componentId;
+ }
+
+ public String getThisComponentId() {
+ return Integer.toString(componentId);
+ }
+
+ }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/83adfd9d/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/spout/TestProgressTracker.java
----------------------------------------------------------------------
diff --git a/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/spout/TestProgressTracker.java b/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/spout/TestProgressTracker.java
index 0bb44af..48aa4b6 100644
--- a/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/spout/TestProgressTracker.java
+++ b/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/spout/TestProgressTracker.java
@@ -1,3 +1,4 @@
+
/**
* Licensed to the Apache Software Foundation (ASF) under one
* or more contributor license agreements. See the NOTICE file
@@ -9,13 +10,10 @@
* <p/>
* http://www.apache.org/licenses/LICENSE-2.0
* <p/>
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
+ * Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the specific language governing permissions
+ * and limitations under the License.
*/
-
package org.apache.storm.hdfs.spout;
import org.apache.hadoop.conf.Configuration;
@@ -33,92 +31,89 @@ import java.io.IOException;
public class TestProgressTracker {
- private FileSystem fs;
- private Configuration conf = new Configuration();
-
- @Rule
- public TemporaryFolder tempFolder = new TemporaryFolder();
- public File baseFolder;
-
- @Before
- public void setUp() throws Exception {
- fs = FileSystem.getLocal(conf);
- }
-
- @Test
- public void testBasic() throws Exception {
- ProgressTracker tracker = new ProgressTracker();
- baseFolder = tempFolder.newFolder("trackertest");
-
- Path file = new Path( baseFolder.toString() + Path.SEPARATOR + "testHeadTrimming.txt" );
- createTextFile(file, 10);
-
- // create reader and do some checks
- TextFileReader reader = new TextFileReader(fs, file, null);
- FileOffset pos0 = tracker.getCommitPosition();
- Assert.assertNull(pos0);
-
- TextFileReader.Offset currOffset = reader.getFileOffset();
- Assert.assertNotNull(currOffset);
- Assert.assertEquals(0, currOffset.charOffset);
-
- // read 1st line and ack
- Assert.assertNotNull(reader.next());
- TextFileReader.Offset pos1 = reader.getFileOffset();
- tracker.recordAckedOffset(pos1);
-
- TextFileReader.Offset pos1b = (TextFileReader.Offset) tracker.getCommitPosition();
- Assert.assertEquals(pos1, pos1b);
-
- // read 2nd line and ACK
- Assert.assertNotNull(reader.next());
- TextFileReader.Offset pos2 = reader.getFileOffset();
- tracker.recordAckedOffset(pos2);
-
- tracker.dumpState(System.err);
- TextFileReader.Offset pos2b = (TextFileReader.Offset) tracker.getCommitPosition();
- Assert.assertEquals(pos2, pos2b);
-
-
- // read lines 3..7, don't ACK .. commit pos should remain same
- Assert.assertNotNull(reader.next());//3
- TextFileReader.Offset pos3 = reader.getFileOffset();
- Assert.assertNotNull(reader.next());//4
- TextFileReader.Offset pos4 = reader.getFileOffset();
- Assert.assertNotNull(reader.next());//5
- TextFileReader.Offset pos5 = reader.getFileOffset();
- Assert.assertNotNull(reader.next());//6
- TextFileReader.Offset pos6 = reader.getFileOffset();
- Assert.assertNotNull(reader.next());//7
- TextFileReader.Offset pos7 = reader.getFileOffset();
-
- // now ack msg 5 and check
- tracker.recordAckedOffset(pos5);
- Assert.assertEquals(pos2, tracker.getCommitPosition()); // should remain unchanged @ 2
- tracker.recordAckedOffset(pos4);
- Assert.assertEquals(pos2, tracker.getCommitPosition()); // should remain unchanged @ 2
- tracker.recordAckedOffset(pos3);
- Assert.assertEquals(pos5, tracker.getCommitPosition()); // should be at 5
-
- tracker.recordAckedOffset(pos6);
- Assert.assertEquals(pos6, tracker.getCommitPosition()); // should be at 6
- tracker.recordAckedOffset(pos6); // double ack on same msg
- Assert.assertEquals(pos6, tracker.getCommitPosition()); // should still be at 6
-
- tracker.recordAckedOffset(pos7);
- Assert.assertEquals(pos7, tracker.getCommitPosition()); // should be at 7
-
- tracker.dumpState(System.err);
- }
-
-
-
- private void createTextFile(Path file, int lineCount) throws IOException {
- FSDataOutputStream os = fs.create(file);
- for (int i = 0; i < lineCount; i++) {
- os.writeBytes("line " + i + System.lineSeparator());
+ private FileSystem fs;
+ private Configuration conf = new Configuration();
+
+ @Rule
+ public TemporaryFolder tempFolder = new TemporaryFolder();
+ public File baseFolder;
+
+ @Before
+ public void setUp() throws Exception {
+ fs = FileSystem.getLocal(conf);
+ }
+
+ @Test
+ public void testBasic() throws Exception {
+ ProgressTracker tracker = new ProgressTracker();
+ baseFolder = tempFolder.newFolder("trackertest");
+
+ Path file = new Path(baseFolder.toString() + Path.SEPARATOR + "testHeadTrimming.txt");
+ createTextFile(file, 10);
+
+ // create reader and do some checks
+ TextFileReader reader = new TextFileReader(fs, file, null);
+ FileOffset pos0 = tracker.getCommitPosition();
+ Assert.assertNull(pos0);
+
+ TextFileReader.Offset currOffset = reader.getFileOffset();
+ Assert.assertNotNull(currOffset);
+ Assert.assertEquals(0, currOffset.charOffset);
+
+ // read 1st line and ack
+ Assert.assertNotNull(reader.next());
+ TextFileReader.Offset pos1 = reader.getFileOffset();
+ tracker.recordAckedOffset(pos1);
+
+ TextFileReader.Offset pos1b = (TextFileReader.Offset) tracker.getCommitPosition();
+ Assert.assertEquals(pos1, pos1b);
+
+ // read 2nd line and ACK
+ Assert.assertNotNull(reader.next());
+ TextFileReader.Offset pos2 = reader.getFileOffset();
+ tracker.recordAckedOffset(pos2);
+
+ tracker.dumpState(System.err);
+ TextFileReader.Offset pos2b = (TextFileReader.Offset) tracker.getCommitPosition();
+ Assert.assertEquals(pos2, pos2b);
+
+ // read lines 3..7, don't ACK .. commit pos should remain same
+ Assert.assertNotNull(reader.next());//3
+ TextFileReader.Offset pos3 = reader.getFileOffset();
+ Assert.assertNotNull(reader.next());//4
+ TextFileReader.Offset pos4 = reader.getFileOffset();
+ Assert.assertNotNull(reader.next());//5
+ TextFileReader.Offset pos5 = reader.getFileOffset();
+ Assert.assertNotNull(reader.next());//6
+ TextFileReader.Offset pos6 = reader.getFileOffset();
+ Assert.assertNotNull(reader.next());//7
+ TextFileReader.Offset pos7 = reader.getFileOffset();
+
+ // now ack msg 5 and check
+ tracker.recordAckedOffset(pos5);
+ Assert.assertEquals(pos2, tracker.getCommitPosition()); // should remain unchanged @ 2
+ tracker.recordAckedOffset(pos4);
+ Assert.assertEquals(pos2, tracker.getCommitPosition()); // should remain unchanged @ 2
+ tracker.recordAckedOffset(pos3);
+ Assert.assertEquals(pos5, tracker.getCommitPosition()); // should be at 5
+
+ tracker.recordAckedOffset(pos6);
+ Assert.assertEquals(pos6, tracker.getCommitPosition()); // should be at 6
+ tracker.recordAckedOffset(pos6); // double ack on same msg
+ Assert.assertEquals(pos6, tracker.getCommitPosition()); // should still be at 6
+
+ tracker.recordAckedOffset(pos7);
+ Assert.assertEquals(pos7, tracker.getCommitPosition()); // should be at 7
+
+ tracker.dumpState(System.err);
+ }
+
+ private void createTextFile(Path file, int lineCount) throws IOException {
+ try (FSDataOutputStream os = fs.create(file)) {
+ for (int i = 0; i < lineCount; i++) {
+ os.writeBytes("line " + i + System.lineSeparator());
+ }
+ }
}
- os.close();
- }
}
http://git-wip-us.apache.org/repos/asf/storm/blob/83adfd9d/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/testing/MiniDFSClusterRule.java
----------------------------------------------------------------------
diff --git a/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/testing/MiniDFSClusterRule.java b/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/testing/MiniDFSClusterRule.java
new file mode 100644
index 0000000..d80796a
--- /dev/null
+++ b/external/storm-hdfs/src/test/java/org/apache/storm/hdfs/testing/MiniDFSClusterRule.java
@@ -0,0 +1,78 @@
+/*
+ * Copyright 2017 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.storm.hdfs.testing;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.junit.rules.TestRule;
+import org.junit.runner.Description;
+import org.junit.runners.model.Statement;
+
+public class MiniDFSClusterRule implements TestRule {
+
+ private static final String TEST_BUILD_DATA = "test.build.data";
+
+ private final Java7Supplier<Configuration> hadoopConfSupplier;
+ private Configuration hadoopConf;
+ private MiniDFSCluster dfscluster;
+
+ public static interface Java7Supplier<T> {
+ T get();
+ }
+
+ public MiniDFSClusterRule() {
+ this(new Java7Supplier<Configuration>() {
+ @Override
+ public Configuration get() {
+ return new Configuration();
+ }
+ });
+ }
+
+ public MiniDFSClusterRule(Java7Supplier<Configuration> hadoopConfSupplier) {
+ this.hadoopConfSupplier = hadoopConfSupplier;
+ }
+
+ public Configuration getHadoopConf() {
+ return hadoopConf;
+ }
+
+ public MiniDFSCluster getDfscluster() {
+ return dfscluster;
+ }
+
+ @Override
+ public Statement apply(Statement base, Description description) {
+ return new Statement() {
+ @Override
+ public void evaluate() throws Throwable {
+ try {
+ System.setProperty(TEST_BUILD_DATA, "target/test/data");
+ hadoopConf = hadoopConfSupplier.get();
+ dfscluster = new MiniDFSCluster.Builder(hadoopConf).numDataNodes(3).build();
+ dfscluster.waitActive();
+ } finally {
+ if (dfscluster != null) {
+ dfscluster.shutdown();
+ }
+ System.clearProperty(TEST_BUILD_DATA);
+ }
+ }
+ };
+ }
+
+}
[35/50] [abbrv] storm git commit: STORM-2525: Fix flaky integration
tests
Posted by pt...@apache.org.
STORM-2525: Fix flaky integration tests
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/4d492874
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/4d492874
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/4d492874
Branch: refs/heads/metrics_v2
Commit: 4d492874282b7e9f94c6ce089d5b614d7dc4fba6
Parents: ef73328
Author: Stig Rohde Døssing <st...@gmail.com>
Authored: Sat May 20 19:46:03 2017 +0200
Committer: Stig Rohde Døssing <sr...@apache.org>
Committed: Mon Nov 13 19:21:31 2017 +0100
----------------------------------------------------------------------
integration-test/config/Vagrantfile | 8 +-
integration-test/config/cluster.xml | 101 -------------------
integration-test/config/install-storm.sh | 1 -
integration-test/config/install-zookeeper.sh | 1 +
integration-test/pom.xml | 5 +
integration-test/run-it.sh | 7 +-
.../topology/window/SlidingTimeCorrectness.java | 4 +-
.../window/SlidingWindowCorrectness.java | 2 -
.../window/TumblingTimeCorrectness.java | 4 +-
.../window/TumblingWindowCorrectness.java | 2 -
.../st/tests/window/SlidingWindowTest.java | 44 ++++----
.../org/apache/storm/st/wrapper/TopoWrap.java | 81 +++++++--------
pom.xml | 19 ++++
storm-core/pom.xml | 7 +-
.../storm/topology/WindowedBoltExecutor.java | 6 +-
.../apache/storm/windowing/EvictionContext.java | 2 +-
.../apache/storm/windowing/EvictionPolicy.java | 2 +-
.../storm/windowing/TimeEvictionPolicy.java | 2 +-
.../windowing/WatermarkCountEvictionPolicy.java | 11 +-
.../windowing/WatermarkTimeEvictionPolicy.java | 10 +-
.../apache/storm/windowing/WindowManager.java | 3 +
.../storm/windowing/WindowManagerTest.java | 48 ++++++++-
22 files changed, 184 insertions(+), 186 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/4d492874/integration-test/config/Vagrantfile
----------------------------------------------------------------------
diff --git a/integration-test/config/Vagrantfile b/integration-test/config/Vagrantfile
index 740d0b0..def461d 100644
--- a/integration-test/config/Vagrantfile
+++ b/integration-test/config/Vagrantfile
@@ -19,7 +19,7 @@
require 'uri'
# Vagrantfile API/syntax version. Don't touch unless you know what you're doing!
VAGRANTFILE_API_VERSION = "2"
-STORM_BOX_TYPE = "hashicorp/precise64"
+STORM_BOX_TYPE = "ubuntu/xenial64"
STORM_ZIP = Dir.glob("../../storm-dist/binary/**/*.zip")
if(STORM_ZIP.length != 1)
raise "Expected one storm-binary found: " + STORM_ZIP.join(",") + ". Did you run : cd ${STORM_SRC_DIR}/storm-dist/binary && mvn clean package -Dgpg.skip=true"
@@ -53,8 +53,8 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
}
end
- config.vm.synced_folder "../../", "/home/vagrant/build/vagrant/storm"
- config.vm.synced_folder "~/.m2", "/home/vagrant/.m2"
+ config.vm.synced_folder "../../", "/home/ubuntu/build/vagrant/storm"
+ config.vm.synced_folder "~/.m2", "/home/ubuntu/.m2"
config.vm.define "node1" do |node1|
node1.vm.provider "virtualbox" do |v|
@@ -62,7 +62,7 @@ Vagrant.configure(VAGRANTFILE_API_VERSION) do |config|
end
node1.vm.network "private_network", ip: "192.168.50.3"
node1.vm.hostname = "node1"
- node1.vm.provision :shell, :inline => "echo run integration test; whoami; env; cd /home/vagrant/build/vagrant/storm/; pwd; bash integration-test/run-it.sh", privileged: false
+ node1.vm.provision :shell, :inline => "echo run integration test; whoami; env; cd /home/ubuntu/build/vagrant/storm/; pwd; bash integration-test/run-it.sh", privileged: false
#node1.vm.provision :shell, :inline => "sudo ln -fs /vagrant/etc-hosts /etc/hosts"
end
http://git-wip-us.apache.org/repos/asf/storm/blob/4d492874/integration-test/config/cluster.xml
----------------------------------------------------------------------
diff --git a/integration-test/config/cluster.xml b/integration-test/config/cluster.xml
deleted file mode 100644
index 97968e4..0000000
--- a/integration-test/config/cluster.xml
+++ /dev/null
@@ -1,101 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-
-<!--
- ~ Licensed to the Apache Software Foundation (ASF) under one or more
- ~ contributor license agreements. See the NOTICE file distributed with
- ~ this work for additional information regarding copyright ownership.
- ~ The ASF licenses this file to You under the Apache License, Version 2.0
- ~ (the "License"); you may not use this file except in compliance with
- ~ the License. You may obtain a copy of the License at
- ~
- ~ http://www.apache.org/licenses/LICENSE-2.0
- ~
- ~ Unless required by applicable law or agreed to in writing, software
- ~ distributed under the License is distributed on an "AS IS" BASIS,
- ~ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- ~ See the License for the specific language governing permissions and
- ~ limitations under the License.
- -->
-
-<configuration scan="true" scanPeriod="60 seconds">
- <properties>
- <property name="pattern">%d{yyyy-MM-dd HH:mm:ss.SSS} %c{1.} %t [%p] %msg%n</property>
- <property name="patternMetrics">%d %-8r %m%n</property>
- </properties>
-
- <appender name="A1" class="ch.qos.logback.core.rolling.RollingFileAppender">
- <file>/var/log/storm/${logfile.name}</file>
- <rollingPolicy class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy">
- <fileNamePattern>/var/log/storm/${logfile.name}.%i</fileNamePattern>
- <minIndex>1</minIndex>
- <maxIndex>9</maxIndex>
- </rollingPolicy>
-
- <triggeringPolicy class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy">
- <maxFileSize>100MB</maxFileSize>
- </triggeringPolicy>
-
- <encoder>
- <pattern>${pattern}</pattern>
- </encoder>
- </appender>
-
- <appender name="ACCESS" class="ch.qos.logback.core.rolling.RollingFileAppender">
- <file>/var/log/storm/access.log</file>
- <rollingPolicy class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy">
- <fileNamePattern>/var/log/storm/access.log.%i</fileNamePattern>
- <minIndex>1</minIndex>
- <maxIndex>9</maxIndex>
- </rollingPolicy>
-
- <triggeringPolicy class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy">
- <maxFileSize>100MB</maxFileSize>
- </triggeringPolicy>
-
- <encoder>
- <pattern>${pattern}</pattern>
- </encoder>
- </appender>
-
- <appender name="METRICS" class="ch.qos.logback.core.rolling.RollingFileAppender">
- <file>/var/log/storm/metrics.log</file>
- <rollingPolicy class="ch.qos.logback.core.rolling.FixedWindowRollingPolicy">
- <fileNamePattern>metrics.log.%i</fileNamePattern>
- <minIndex>1</minIndex>
- <maxIndex>9</maxIndex>
- </rollingPolicy>
-
- <triggeringPolicy class="ch.qos.logback.core.rolling.SizeBasedTriggeringPolicy">
- <maxFileSize>2MB</maxFileSize>
- </triggeringPolicy>
-
- <encoder>
- <pattern>${patternMetrics}</pattern>
- </encoder>
- </appender>
-
- <root level="INFO">
- <appender-ref ref="A1"/>
- </root>
-
- <logger name="org.apache.storm.messaging.netty">
- <level value="WARN" />
- <appender-ref ref="A1" />
- </logger>
-
- <logger name="org.apache.storm">
- <level value="DEBUG" />
- <appender-ref ref="A1" />
- </logger>
-
- <logger name="org.apache.storm.security.auth.authorizer" additivity="false">
- <level value="INFO" />
- <appender-ref ref="ACCESS" />
- </logger>
-
- <logger name="org.apache.storm.metric.LoggingMetricsConsumer" additivity="false">
- <level value="INFO"/>
- <appender-ref ref="METRICS"/>
- </logger>
-
-</configuration>
http://git-wip-us.apache.org/repos/asf/storm/blob/4d492874/integration-test/config/install-storm.sh
----------------------------------------------------------------------
diff --git a/integration-test/config/install-storm.sh b/integration-test/config/install-storm.sh
index b08e2bc..8316c75 100644
--- a/integration-test/config/install-storm.sh
+++ b/integration-test/config/install-storm.sh
@@ -31,7 +31,6 @@ chown -R storm:storm /etc/storm
rm /usr/share/storm/conf/storm.yaml
cp "${SCRIPT_DIR}/storm.yaml" /usr/share/storm/conf/
-cp "${SCRIPT_DIR}/cluster.xml" /usr/share/storm/logback/
ln -s /usr/share/storm/conf/storm.yaml /etc/storm/conf/storm.yaml
mkdir /var/log/storm
http://git-wip-us.apache.org/repos/asf/storm/blob/4d492874/integration-test/config/install-zookeeper.sh
----------------------------------------------------------------------
diff --git a/integration-test/config/install-zookeeper.sh b/integration-test/config/install-zookeeper.sh
index 5f92f56..98253d7 100644
--- a/integration-test/config/install-zookeeper.sh
+++ b/integration-test/config/install-zookeeper.sh
@@ -14,6 +14,7 @@
# See the License for the specific language governing permissions and
# limitations under the License.
#
+# $1 is the Zookeeper version to install
apt-get --yes install zookeeper=$1 zookeeperd=$1
service zookeeper stop
echo maxClientCnxns=200 >> /etc/zookeeper/conf/zoo.cfg
http://git-wip-us.apache.org/repos/asf/storm/blob/4d492874/integration-test/pom.xml
----------------------------------------------------------------------
diff --git a/integration-test/pom.xml b/integration-test/pom.xml
index 0351235..bcf688d 100755
--- a/integration-test/pom.xml
+++ b/integration-test/pom.xml
@@ -88,6 +88,11 @@
<version>${project.version}</version>
</dependency>
<dependency>
+ <groupId>com.google.code.gson</groupId>
+ <artifactId>gson</artifactId>
+ <version>2.8.0</version>
+ </dependency>
+ <dependency>
<groupId>org.apache.storm</groupId>
<artifactId>storm-starter</artifactId>
<version>${project.version}</version>
http://git-wip-us.apache.org/repos/asf/storm/blob/4d492874/integration-test/run-it.sh
----------------------------------------------------------------------
diff --git a/integration-test/run-it.sh b/integration-test/run-it.sh
index eaed790..927c60d 100755
--- a/integration-test/run-it.sh
+++ b/integration-test/run-it.sh
@@ -32,11 +32,11 @@ function list_storm_processes() {
list_storm_processes || true
# increasing swap space so we can run lots of workers
-sudo dd if=/dev/zero of=/swapfile.img bs=8192 count=1M
+sudo dd if=/dev/zero of=/swapfile.img bs=4096 count=1M
sudo mkswap /swapfile.img
sudo swapon /swapfile.img
-if [[ "${USER}" == "vagrant" ]]; then # install oracle jdk8
+if [[ "${USER}" == "ubuntu" ]]; then # install oracle jdk8
sudo apt-get update
sudo apt-get -y install python-software-properties
sudo apt-add-repository -y ppa:webupd8team/java
@@ -44,12 +44,13 @@ if [[ "${USER}" == "vagrant" ]]; then # install oracle jdk8
echo "oracle-java8-installer shared/accepted-oracle-license-v1-1 select true" | sudo debconf-set-selections
sudo apt-get install -y oracle-java8-installer
sudo apt-get -y install maven
+ sudo apt-get install unzip
java -version
mvn --version
export MAVEN_OPTS="-Xmx3000m"
+ zookeeper_version=3.4.8*
else
( while true; do echo "heartbeat"; sleep 300; done ) & #heartbeat needed by travis ci
- (cd "${STORM_SRC_DIR}" && mvn clean install -DskipTests=true) || die "maven install command failed"
if [[ "${USER}" == "travis" ]]; then
( cd "${STORM_SRC_DIR}/storm-dist/binary" && mvn clean package -Dgpg.skip=true )
fi
http://git-wip-us.apache.org/repos/asf/storm/blob/4d492874/integration-test/src/main/java/org/apache/storm/st/topology/window/SlidingTimeCorrectness.java
----------------------------------------------------------------------
diff --git a/integration-test/src/main/java/org/apache/storm/st/topology/window/SlidingTimeCorrectness.java b/integration-test/src/main/java/org/apache/storm/st/topology/window/SlidingTimeCorrectness.java
index 430449b..8e9c6c9 100644
--- a/integration-test/src/main/java/org/apache/storm/st/topology/window/SlidingTimeCorrectness.java
+++ b/integration-test/src/main/java/org/apache/storm/st/topology/window/SlidingTimeCorrectness.java
@@ -114,7 +114,9 @@ public class SlidingTimeCorrectness implements TestableTopology {
@Override
public void nextTuple() {
- TimeUtil.sleepMilliSec(rng.nextInt(800));
+ //Emitting too quickly can lead to spurious test failures because the worker log may roll right before we read it
+ //Sleep a bit between emits
+ TimeUtil.sleepMilliSec(rng.nextInt(100));
currentNum++;
TimeData data = TimeData.newData(currentNum);
final Values tuple = data.getValues();
http://git-wip-us.apache.org/repos/asf/storm/blob/4d492874/integration-test/src/main/java/org/apache/storm/st/topology/window/SlidingWindowCorrectness.java
----------------------------------------------------------------------
diff --git a/integration-test/src/main/java/org/apache/storm/st/topology/window/SlidingWindowCorrectness.java b/integration-test/src/main/java/org/apache/storm/st/topology/window/SlidingWindowCorrectness.java
index 33ee004..c059035 100644
--- a/integration-test/src/main/java/org/apache/storm/st/topology/window/SlidingWindowCorrectness.java
+++ b/integration-test/src/main/java/org/apache/storm/st/topology/window/SlidingWindowCorrectness.java
@@ -39,7 +39,6 @@ import org.apache.storm.st.utils.TimeUtil;
import java.util.List;
import java.util.Map;
import java.util.Random;
-import java.util.concurrent.TimeUnit;
/**
* Computes sliding window sum
@@ -74,7 +73,6 @@ public class SlidingWindowCorrectness implements TestableTopology {
builder.setSpout(getSpoutName(), new IncrementingSpout(), 1);
builder.setBolt(getBoltName(),
new VerificationBolt()
- .withLag(new BaseWindowedBolt.Duration(10, TimeUnit.SECONDS))
.withWindow(new BaseWindowedBolt.Count(windowSize), new BaseWindowedBolt.Count(slideSize)),
1)
.shuffleGrouping(getSpoutName());
http://git-wip-us.apache.org/repos/asf/storm/blob/4d492874/integration-test/src/main/java/org/apache/storm/st/topology/window/TumblingTimeCorrectness.java
----------------------------------------------------------------------
diff --git a/integration-test/src/main/java/org/apache/storm/st/topology/window/TumblingTimeCorrectness.java b/integration-test/src/main/java/org/apache/storm/st/topology/window/TumblingTimeCorrectness.java
index 7cadd18..0c9e891 100644
--- a/integration-test/src/main/java/org/apache/storm/st/topology/window/TumblingTimeCorrectness.java
+++ b/integration-test/src/main/java/org/apache/storm/st/topology/window/TumblingTimeCorrectness.java
@@ -111,7 +111,9 @@ public class TumblingTimeCorrectness implements TestableTopology {
@Override
public void nextTuple() {
- TimeUtil.sleepMilliSec(rng.nextInt(800));
+ //Emitting too quickly can lead to spurious test failures because the worker log may roll right before we read it
+ //Sleep a bit between emits
+ TimeUtil.sleepMilliSec(rng.nextInt(100));
currentNum++;
TimeData data = TimeData.newData(currentNum);
final Values tuple = data.getValues();
http://git-wip-us.apache.org/repos/asf/storm/blob/4d492874/integration-test/src/main/java/org/apache/storm/st/topology/window/TumblingWindowCorrectness.java
----------------------------------------------------------------------
diff --git a/integration-test/src/main/java/org/apache/storm/st/topology/window/TumblingWindowCorrectness.java b/integration-test/src/main/java/org/apache/storm/st/topology/window/TumblingWindowCorrectness.java
index adcb9dd..b7f5dfa 100644
--- a/integration-test/src/main/java/org/apache/storm/st/topology/window/TumblingWindowCorrectness.java
+++ b/integration-test/src/main/java/org/apache/storm/st/topology/window/TumblingWindowCorrectness.java
@@ -39,7 +39,6 @@ import org.apache.storm.st.utils.TimeUtil;
import java.util.List;
import java.util.Map;
import java.util.Random;
-import java.util.concurrent.TimeUnit;
/**
* Computes sliding window sum
@@ -72,7 +71,6 @@ public class TumblingWindowCorrectness implements TestableTopology {
builder.setSpout(getSpoutName(), new IncrementingSpout(), 1);
builder.setBolt(getBoltName(),
new VerificationBolt()
- .withLag(new BaseWindowedBolt.Duration(10, TimeUnit.SECONDS))
.withTumblingWindow(new BaseWindowedBolt.Count(tumbleSize)), 1)
.shuffleGrouping(getSpoutName());
return builder.createTopology();
http://git-wip-us.apache.org/repos/asf/storm/blob/4d492874/integration-test/src/test/java/org/apache/storm/st/tests/window/SlidingWindowTest.java
----------------------------------------------------------------------
diff --git a/integration-test/src/test/java/org/apache/storm/st/tests/window/SlidingWindowTest.java b/integration-test/src/test/java/org/apache/storm/st/tests/window/SlidingWindowTest.java
index 29afe7a..da65ca5 100644
--- a/integration-test/src/test/java/org/apache/storm/st/tests/window/SlidingWindowTest.java
+++ b/integration-test/src/test/java/org/apache/storm/st/tests/window/SlidingWindowTest.java
@@ -17,6 +17,7 @@
package org.apache.storm.st.tests.window;
+import java.io.IOException;
import org.apache.storm.st.helper.AbstractTest;
import org.apache.storm.st.wrapper.LogData;
import org.apache.storm.st.wrapper.TopoWrap;
@@ -79,21 +80,25 @@ public final class SlidingWindowTest extends AbstractTest {
runAndVerifyCount(windowSize, slideSize, testable, topo);
}
- static void runAndVerifyCount(int windowSize, int slideSize, TestableTopology testable, TopoWrap topo) throws TException, MalformedURLException {
+ static void runAndVerifyCount(int windowSize, int slideSize, TestableTopology testable, TopoWrap topo) throws IOException, TException, MalformedURLException {
topo.submitSuccessfully();
- final int minSpoutEmits = 1000 + windowSize;
final int minBoltEmits = 5;
+ //Sliding windows should produce one window every slideSize tuples
+ //Wait for the spout to emit at least enough tuples to get minBoltEmit windows and at least one full window
+ final int minSpoutEmits = Math.max(windowSize, minBoltEmits * slideSize);
+
String boltName = testable.getBoltName();
String spoutName = testable.getSpoutName();
- topo.waitForProgress(minSpoutEmits, spoutName, 180);
- topo.waitForProgress(minBoltEmits, boltName, 180);
+ //Waiting for spout tuples isn't strictly necessary since we also wait for bolt emits, but do it anyway
+ topo.assertProgress(minSpoutEmits, spoutName, 180);
+ topo.assertProgress(minBoltEmits, boltName, 180);
List<TopoWrap.ExecutorURL> boltUrls = topo.getLogUrls(boltName);
log.info(boltUrls.toString());
final List<LogData> allBoltData = topo.getLogData(boltName);
final List<LogData> allSpoutData = topo.getLogData(spoutName);
Assert.assertTrue(allBoltData.size() >= minBoltEmits,
"Expecting min " + minBoltEmits + " bolt emits, found: " + allBoltData.size() + " \n\t" + allBoltData);
- final int numberOfWindows = allBoltData.size() - windowSize / slideSize;
+ final int numberOfWindows = allBoltData.size();
for(int i = 0; i < numberOfWindows; ++i ) {
log.info("Comparing window: " + (i + 1) + " of " + numberOfWindows);
final int toIndex = (i + 1) * slideSize;
@@ -143,28 +148,29 @@ public final class SlidingWindowTest extends AbstractTest {
runAndVerifyTime(windowSec, slideSec, testable, topo);
}
- static void runAndVerifyTime(int windowSec, int slideSec, TestableTopology testable, TopoWrap topo) throws TException, java.net.MalformedURLException {
+ static void runAndVerifyTime(int windowSec, int slideSec, TestableTopology testable, TopoWrap topo) throws IOException, TException, java.net.MalformedURLException {
topo.submitSuccessfully();
- final int minSpoutEmits = 1000 + windowSec;
+ final int minSpoutEmits = 100;
final int minBoltEmits = 5;
String boltName = testable.getBoltName();
String spoutName = testable.getSpoutName();
- topo.waitForProgress(minSpoutEmits, spoutName, 60 + 10 * (windowSec + slideSec));
- topo.waitForProgress(minBoltEmits, boltName, 60 + 10 * (windowSec + slideSec));
- final List<TimeData> allSpoutData = topo.getLogData(spoutName, TimeData.CLS);
- final List<LogData> allBoltLog = topo.getLogData(boltName);
- final List<TimeDataWindow> allBoltData = topo.getLogData(boltName, TimeDataWindow.CLS);
- Assert.assertTrue(allBoltLog.size() >= minBoltEmits,
- "Expecting min " + minBoltEmits + " bolt emits, found: " + allBoltLog.size() + " \n\t" + allBoltLog);
- final DateTime firstEndTime = TimeUtil.ceil(new DateTime(allSpoutData.get(0).getDate()).withZone(DateTimeZone.UTC), slideSec);
- final int numberOfWindows = allBoltLog.size() - windowSec / slideSec;
+ //Waiting for spout tuples isn't strictly necessary since we also wait for bolt emits, but do it anyway
+ topo.assertProgress(minSpoutEmits, spoutName, 60 + 10 * (windowSec + slideSec));
+ topo.assertProgress(minBoltEmits, boltName, 60 + 10 * (windowSec + slideSec));
+ final List<TimeData> allSpoutDataDeserialized = topo.getLogData(spoutName, TimeData.CLS);
+ final List<LogData> allBoltData = topo.getLogData(boltName);
+ final List<TimeDataWindow> allBoltDataDeserialized = topo.deserializeLogData(allBoltData, TimeDataWindow.CLS);
+ Assert.assertTrue(allBoltData.size() >= minBoltEmits,
+ "Expecting min " + minBoltEmits + " bolt emits, found: " + allBoltData.size() + " \n\t" + allBoltData);
+ final DateTime firstEndTime = TimeUtil.ceil(new DateTime(allSpoutDataDeserialized.get(0).getDate()).withZone(DateTimeZone.UTC), slideSec);
+ final int numberOfWindows = allBoltData.size();
for(int i = 0; i < numberOfWindows; ++i ) {
final DateTime toDate = firstEndTime.plusSeconds(i * slideSec);
final DateTime fromDate = toDate.minusSeconds(windowSec);
log.info("Comparing window: " + fromDate + " to " + toDate + " iter " + (i+1) + "/" + numberOfWindows);
- final TimeDataWindow computedWindow = TimeDataWindow.newInstance(allSpoutData,fromDate, toDate);
- final LogData oneBoltLog = allBoltLog.get(i);
- final TimeDataWindow actualWindow = allBoltData.get(i);
+ final TimeDataWindow computedWindow = TimeDataWindow.newInstance(allSpoutDataDeserialized,fromDate, toDate);
+ final LogData oneBoltLog = allBoltData.get(i);
+ final TimeDataWindow actualWindow = allBoltDataDeserialized.get(i);
log.info("Actual window: " + actualWindow.getDescription());
log.info("Computed window: " + computedWindow.getDescription());
for (TimeData oneLog : computedWindow) {
http://git-wip-us.apache.org/repos/asf/storm/blob/4d492874/integration-test/src/test/java/org/apache/storm/st/wrapper/TopoWrap.java
----------------------------------------------------------------------
diff --git a/integration-test/src/test/java/org/apache/storm/st/wrapper/TopoWrap.java b/integration-test/src/test/java/org/apache/storm/st/wrapper/TopoWrap.java
index 5e58637..4d5c3a8 100644
--- a/integration-test/src/test/java/org/apache/storm/st/wrapper/TopoWrap.java
+++ b/integration-test/src/test/java/org/apache/storm/st/wrapper/TopoWrap.java
@@ -53,6 +53,7 @@ import java.io.File;
import java.io.IOException;
import java.net.MalformedURLException;
import java.net.URL;
+import java.nio.charset.StandardCharsets;
import java.text.NumberFormat;
import java.util.ArrayList;
import java.util.Arrays;
@@ -65,6 +66,8 @@ import java.util.Locale;
import java.util.Map;
import java.util.Set;
import java.util.regex.Pattern;
+import org.apache.storm.Config;
+import org.apache.storm.utils.Utils;
public class TopoWrap {
private static Logger log = LoggerFactory.getLogger(TopoWrap.class);
@@ -97,6 +100,9 @@ public class TopoWrap {
submitConf.put("storm.zookeeper.topology.auth.scheme", "digest");
submitConf.put("topology.workers", 3);
submitConf.put("topology.debug", true);
+ //Set the metrics sample rate to 1 to force update the executor stats every time something happens
+ //This is necessary because getAllTimeEmittedCount relies on the executor emit stats to be accurate
+ submitConf.put(Config.TOPOLOGY_STATS_SAMPLE_RATE, 1);
return submitConf;
}
@@ -180,7 +186,7 @@ public class TopoWrap {
Map<String, Long> allTime = emitted.get(since);
if (allTime == null)
return 0L;
- return allTime.get("default");
+ return allTime.get(Utils.DEFAULT_STREAM_ID);
}
});
return sum(ackCounts).longValue();
@@ -213,7 +219,7 @@ public class TopoWrap {
log.info(getInfo().toString());
long emitCount = getAllTimeEmittedCount(componentName);
log.info("Count for component " + componentName + " is " + emitCount);
- if (emitCount > minEmits) {
+ if (emitCount >= minEmits) {
break;
}
TimeUtil.sleepSec(10);
@@ -280,8 +286,13 @@ public class TopoWrap {
}
}
- public <T extends FromJson<T>> List<T> getLogData(final String componentId, final FromJson<T> cls) throws TException, MalformedURLException {
+ public <T extends FromJson<T>> List<T> getLogData(final String componentId, final FromJson<T> cls)
+ throws IOException, TException, MalformedURLException {
final List<LogData> logData = getLogData(componentId);
+ return deserializeLogData(logData, cls);
+ }
+
+ public <T extends FromJson<T>> List<T> deserializeLogData(final List<LogData> logData, final FromJson<T> cls) {
final List<T> data = new ArrayList<>(
Collections2.transform(logData, new Function<LogData, T>() {
@Nullable
@@ -294,7 +305,7 @@ public class TopoWrap {
return data;
}
- public List<LogData> getLogData(final String componentId) throws TException, MalformedURLException {
+ public List<LogData> getLogData(final String componentId) throws IOException, TException, MalformedURLException {
final String logs = getLogs(componentId);
final String dateRegex = "\\d{4}-\\d{2}-\\d{2} \\d{2}:\\d{2}:\\d{2}\\.\\d{3}";
Pattern pattern = Pattern.compile("(?=\\n" + dateRegex + ")");
@@ -318,48 +329,38 @@ public class TopoWrap {
return sortedLogs;
}
- public String getLogs(final String componentId) throws TException, MalformedURLException {
+ public String getLogs(final String componentId) throws IOException, TException, MalformedURLException {
log.info("Fetching logs for componentId = " + componentId);
List<ExecutorURL> exclaim2Urls = getLogUrls(componentId);
log.info("Found " + exclaim2Urls.size() + " urls: " + exclaim2Urls.toString());
- Collection<String> urlOuputs = Collections2.transform(exclaim2Urls, new Function<ExecutorURL, String>() {
- @Nullable
- @Override
- public String apply(@Nullable ExecutorURL executorURL) {
- if (executorURL == null || executorURL.getDownloadUrl() == null) {
- return "";
- }
- String warnMessage = "Couldn't fetch executorURL: " + executorURL;
+ List<String> urlContents = new ArrayList<>();
+ for(ExecutorURL executorUrl : exclaim2Urls) {
+ if(executorUrl == null || executorUrl.getDownloadUrl() == null) {
+ continue;
+ }
+ log.info("Fetching: " + executorUrl);
+ URL downloadUrl = executorUrl.downloadUrl;
+ String urlContent = IOUtils.toString(downloadUrl, StandardCharsets.UTF_8);
+ urlContents.add(urlContent);
+ if (urlContent.length() < 500) {
+ log.info("Fetched: " + urlContent);
+ } else {
+ log.info("Fetched: " + NumberFormat.getNumberInstance(Locale.US).format(urlContent.length()) + " bytes.");
+ }
+ if (System.getProperty("regression.downloadWorkerLogs").equalsIgnoreCase("true")) {
+ final String userDir = System.getProperty("user.dir");
+ final File target = new File(userDir, "target");
+ final File logDir = new File(target, "logs");
+ final File logFile = new File(logDir, downloadUrl.getHost() + "-" + downloadUrl.getFile().split("/")[2]);
try {
- log.info("Fetching: " + executorURL);
- final URL downloadUrl = executorURL.downloadUrl;
- final String urlContent = IOUtils.toString(downloadUrl);
- if (urlContent.length() < 500) {
- log.info("Fetched: " + urlContent);
- } else {
- log.info("Fetched: " + NumberFormat.getNumberInstance(Locale.US).format(urlContent.length()) + " bytes.");
- }
- if (System.getProperty("regression.downloadWorkerLogs").equalsIgnoreCase("true")) {
- final String userDir = System.getProperty("user.dir");
- final File target = new File(userDir, "target");
- final File logDir = new File(target, "logs");
- final File logFile = new File(logDir, downloadUrl.getHost() + "-" + downloadUrl.getFile().split("/")[2]);
- try {
- FileUtils.forceMkdir(logDir);
- FileUtils.write(logFile, urlContent);
- } catch (Throwable throwable) {
- log.info("Caught exteption: " + ExceptionUtils.getFullStackTrace(throwable));
- }
- }
- return urlContent;
- } catch (IOException e) {
- log.warn(warnMessage);
+ FileUtils.forceMkdir(logDir);
+ FileUtils.write(logFile, urlContent, StandardCharsets.UTF_8);
+ } catch (Throwable throwable) {
+ log.info("Caught exception: " + ExceptionUtils.getFullStackTrace(throwable));
}
- String stars = StringUtils.repeat("*", 30);
- return stars + " " + warnMessage + " " + stars;
}
- });
- return StringUtils.join(urlOuputs, '\n');
+ }
+ return StringUtils.join(urlContents, '\n');
}
private Number sum(Collection<? extends Number> nums) {
http://git-wip-us.apache.org/repos/asf/storm/blob/4d492874/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 3b0d39b..facb74f 100644
--- a/pom.xml
+++ b/pom.xml
@@ -267,6 +267,7 @@
<junit.version>4.11</junit.version>
<metrics-clojure.version>2.5.1</metrics-clojure.version>
<hdrhistogram.version>2.1.7</hdrhistogram.version>
+ <hamcrest.version>1.3</hamcrest.version>
<calcite.version>1.11.0</calcite.version>
@@ -916,6 +917,24 @@
</dependency>
<dependency>
<groupId>org.mockito</groupId>
+ <artifactId>mockito-core</artifactId>
+ <version>${mockito.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.hamcrest</groupId>
+ <artifactId>hamcrest-core</artifactId>
+ <version>${hamcrest.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.hamcrest</groupId>
+ <artifactId>hamcrest-library</artifactId>
+ <version>${hamcrest.version}</version>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.mockito</groupId>
<artifactId>mockito-all</artifactId>
<version>${mockito.version}</version>
<scope>test</scope>
http://git-wip-us.apache.org/repos/asf/storm/blob/4d492874/storm-core/pom.xml
----------------------------------------------------------------------
diff --git a/storm-core/pom.xml b/storm-core/pom.xml
index 3d714c0..7c6f310 100644
--- a/storm-core/pom.xml
+++ b/storm-core/pom.xml
@@ -256,7 +256,12 @@
</dependency>
<dependency>
<groupId>org.mockito</groupId>
- <artifactId>mockito-all</artifactId>
+ <artifactId>mockito-core</artifactId>
+ <scope>test</scope>
+ </dependency>
+ <dependency>
+ <groupId>org.hamcrest</groupId>
+ <artifactId>hamcrest-library</artifactId>
<scope>test</scope>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/storm/blob/4d492874/storm-core/src/jvm/org/apache/storm/topology/WindowedBoltExecutor.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/topology/WindowedBoltExecutor.java b/storm-core/src/jvm/org/apache/storm/topology/WindowedBoltExecutor.java
index c9afc67..48bcb69 100644
--- a/storm-core/src/jvm/org/apache/storm/topology/WindowedBoltExecutor.java
+++ b/storm-core/src/jvm/org/apache/storm/topology/WindowedBoltExecutor.java
@@ -199,8 +199,7 @@ public class WindowedBoltExecutor implements IRichBolt {
// validate
validate(stormConf, windowLengthCount, windowLengthDuration,
slidingIntervalCount, slidingIntervalDuration);
- evictionPolicy = getEvictionPolicy(windowLengthCount, windowLengthDuration,
- manager);
+ evictionPolicy = getEvictionPolicy(windowLengthCount, windowLengthDuration);
triggerPolicy = getTriggerPolicy(slidingIntervalCount, slidingIntervalDuration,
manager, evictionPolicy);
manager.setEvictionPolicy(evictionPolicy);
@@ -251,8 +250,7 @@ public class WindowedBoltExecutor implements IRichBolt {
}
}
- private EvictionPolicy<Tuple> getEvictionPolicy(Count windowLengthCount, Duration windowLengthDuration,
- WindowManager<Tuple> manager) {
+ private EvictionPolicy<Tuple> getEvictionPolicy(Count windowLengthCount, Duration windowLengthDuration) {
if (windowLengthCount != null) {
if (isTupleTs()) {
return new WatermarkCountEvictionPolicy<>(windowLengthCount.value);
http://git-wip-us.apache.org/repos/asf/storm/blob/4d492874/storm-core/src/jvm/org/apache/storm/windowing/EvictionContext.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/windowing/EvictionContext.java b/storm-core/src/jvm/org/apache/storm/windowing/EvictionContext.java
index c5a578a..7582e26 100644
--- a/storm-core/src/jvm/org/apache/storm/windowing/EvictionContext.java
+++ b/storm-core/src/jvm/org/apache/storm/windowing/EvictionContext.java
@@ -46,7 +46,7 @@ public interface EvictionContext {
Long getSlidingInterval();
/**
- * Returns the current count of events in the queue up to the reference tim
+ * Returns the current count of events in the queue up to the reference time
* based on which count based evictions can be performed.
*
* @return the current count
http://git-wip-us.apache.org/repos/asf/storm/blob/4d492874/storm-core/src/jvm/org/apache/storm/windowing/EvictionPolicy.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/windowing/EvictionPolicy.java b/storm-core/src/jvm/org/apache/storm/windowing/EvictionPolicy.java
index 05e4d93..e7d1e40 100644
--- a/storm-core/src/jvm/org/apache/storm/windowing/EvictionPolicy.java
+++ b/storm-core/src/jvm/org/apache/storm/windowing/EvictionPolicy.java
@@ -27,7 +27,7 @@ public interface EvictionPolicy<T> {
/**
* The action to be taken when {@link EvictionPolicy#evict(Event)} is invoked.
*/
- enum Action {
+ public enum Action {
/**
* expire the event and remove it from the queue
*/
http://git-wip-us.apache.org/repos/asf/storm/blob/4d492874/storm-core/src/jvm/org/apache/storm/windowing/TimeEvictionPolicy.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/windowing/TimeEvictionPolicy.java b/storm-core/src/jvm/org/apache/storm/windowing/TimeEvictionPolicy.java
index 0ab28a1..95329ca 100644
--- a/storm-core/src/jvm/org/apache/storm/windowing/TimeEvictionPolicy.java
+++ b/storm-core/src/jvm/org/apache/storm/windowing/TimeEvictionPolicy.java
@@ -49,7 +49,7 @@ public class TimeEvictionPolicy<T> implements EvictionPolicy<T> {
* {@inheritDoc}
*/
@Override
- public Action evict(Event<T> event) {
+ public Action evict(Event<T> event) {
long now = referenceTime == null ? System.currentTimeMillis() : referenceTime;
long diff = now - event.getTimestamp();
if (diff >= (windowLength + delta)) {
http://git-wip-us.apache.org/repos/asf/storm/blob/4d492874/storm-core/src/jvm/org/apache/storm/windowing/WatermarkCountEvictionPolicy.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/windowing/WatermarkCountEvictionPolicy.java b/storm-core/src/jvm/org/apache/storm/windowing/WatermarkCountEvictionPolicy.java
index 74240bb..676ccdb 100644
--- a/storm-core/src/jvm/org/apache/storm/windowing/WatermarkCountEvictionPolicy.java
+++ b/storm-core/src/jvm/org/apache/storm/windowing/WatermarkCountEvictionPolicy.java
@@ -19,7 +19,7 @@ package org.apache.storm.windowing;
/**
* An eviction policy that tracks count based on watermark ts and
- * evicts events upto the watermark based on a threshold count.
+ * evicts events up to the watermark based on a threshold count.
*
* @param <T> the type of event tracked by this policy.
*/
@@ -30,6 +30,7 @@ public class WatermarkCountEvictionPolicy<T> extends CountEvictionPolicy<T> {
*/
private long referenceTime;
private long processed = 0L;
+ private EvictionContext context;
public WatermarkCountEvictionPolicy(int count) {
super(count);
@@ -37,6 +38,13 @@ public class WatermarkCountEvictionPolicy<T> extends CountEvictionPolicy<T> {
@Override
public Action evict(Event<T> event) {
+ if(context == null) {
+ //It is possible to get asked about eviction before we have a context, due to WindowManager.compactWindow.
+ //In this case we should hold on to all the events. When the first watermark is received, the context will be set,
+ //and the events will be reevaluated for eviction
+ return Action.STOP;
+ }
+
Action action;
if (event.getTimestamp() <= referenceTime && processed < currentCount.get()) {
action = super.evict(event);
@@ -56,6 +64,7 @@ public class WatermarkCountEvictionPolicy<T> extends CountEvictionPolicy<T> {
@Override
public void setContext(EvictionContext context) {
+ this.context = context;
referenceTime = context.getReferenceTime();
if (context.getCurrentCount() != null) {
currentCount.set(context.getCurrentCount());
http://git-wip-us.apache.org/repos/asf/storm/blob/4d492874/storm-core/src/jvm/org/apache/storm/windowing/WatermarkTimeEvictionPolicy.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/windowing/WatermarkTimeEvictionPolicy.java b/storm-core/src/jvm/org/apache/storm/windowing/WatermarkTimeEvictionPolicy.java
index 53361d2..981f514 100644
--- a/storm-core/src/jvm/org/apache/storm/windowing/WatermarkTimeEvictionPolicy.java
+++ b/storm-core/src/jvm/org/apache/storm/windowing/WatermarkTimeEvictionPolicy.java
@@ -58,7 +58,15 @@ public class WatermarkTimeEvictionPolicy<T> extends TimeEvictionPolicy<T> {
*/
@Override
public Action evict(Event<T> event) {
- long diff = referenceTime - event.getTimestamp();
+ if(evictionContext == null) {
+ //It is possible to get asked about eviction before we have a context, due to WindowManager.compactWindow.
+ //In this case we should hold on to all the events. When the first watermark is received, the context will be set,
+ //and the events will be reevaluated for eviction
+ return Action.STOP;
+ }
+
+ long referenceTime = evictionContext.getReferenceTime();
+ long diff = referenceTime - event.getTimestamp();
if (diff < -lag) {
return Action.STOP;
} else if (diff < 0) {
http://git-wip-us.apache.org/repos/asf/storm/blob/4d492874/storm-core/src/jvm/org/apache/storm/windowing/WindowManager.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/windowing/WindowManager.java b/storm-core/src/jvm/org/apache/storm/windowing/WindowManager.java
index 792509e..a734497 100644
--- a/storm-core/src/jvm/org/apache/storm/windowing/WindowManager.java
+++ b/storm-core/src/jvm/org/apache/storm/windowing/WindowManager.java
@@ -46,6 +46,9 @@ public class WindowManager<T> implements TriggerHandler {
/**
* Expire old events every EXPIRE_EVENTS_THRESHOLD to
* keep the window size in check.
+ *
+ * Note that if the eviction policy is based on watermarks, events will not be evicted until a new
+ * watermark would cause them to be considered expired anyway, regardless of this limit
*/
public static final int EXPIRE_EVENTS_THRESHOLD = 100;
http://git-wip-us.apache.org/repos/asf/storm/blob/4d492874/storm-core/test/jvm/org/apache/storm/windowing/WindowManagerTest.java
----------------------------------------------------------------------
diff --git a/storm-core/test/jvm/org/apache/storm/windowing/WindowManagerTest.java b/storm-core/test/jvm/org/apache/storm/windowing/WindowManagerTest.java
index 6645566..a9874cd 100644
--- a/storm-core/test/jvm/org/apache/storm/windowing/WindowManagerTest.java
+++ b/storm-core/test/jvm/org/apache/storm/windowing/WindowManagerTest.java
@@ -29,9 +29,12 @@ import java.util.List;
import java.util.Set;
import java.util.concurrent.TimeUnit;
-import static org.apache.storm.topology.base.BaseWindowedBolt.Count;
import static org.apache.storm.topology.base.BaseWindowedBolt.Duration;
+import static org.hamcrest.CoreMatchers.equalTo;
+import static org.hamcrest.CoreMatchers.is;
+import static org.hamcrest.Matchers.empty;
import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertThat;
import static org.junit.Assert.assertTrue;
/**
@@ -155,7 +158,48 @@ public class WindowManagerTest {
// window should be compacted and events should be expired.
assertEquals(seq(1, threshold - windowLength), listener.onExpiryEvents);
}
-
+
+ private void testEvictBeforeWatermarkForWatermarkEvictionPolicy(EvictionPolicy watermarkEvictionPolicy, int windowLength) throws Exception {
+ /**
+ * The watermark eviction policy must not evict tuples until the first watermark has been received.
+ * The policies can't make a meaningful decision prior to the first watermark, so the safe decision
+ * is to postpone eviction.
+ */
+ int threshold = WindowManager.EXPIRE_EVENTS_THRESHOLD;
+ windowManager.setEvictionPolicy(watermarkEvictionPolicy);
+ WatermarkCountTriggerPolicy triggerPolicy = new WatermarkCountTriggerPolicy(windowLength, windowManager,
+ watermarkEvictionPolicy, windowManager);
+ triggerPolicy.start();
+ windowManager.setTriggerPolicy(triggerPolicy);
+ for (int i : seq(1, threshold)) {
+ windowManager.add(i, i);
+ }
+ assertThat("The watermark eviction policies should never evict events before the first watermark is received", listener.onExpiryEvents, is(empty()));
+ windowManager.add(new WaterMarkEvent<Integer>(threshold));
+ // The events should be put in a window when the first watermark is received
+ assertEquals(seq(1, threshold), listener.onActivationEvents);
+ //Now add some more events and a new watermark, and check that the previous events are expired
+ for(int i : seq(threshold+1, threshold*2)) {
+ windowManager.add(i, i);
+ }
+ windowManager.add(new WaterMarkEvent<Integer>(threshold + windowLength+1));
+ //All the events should be expired when the next watermark is received
+ assertThat("All the events should be expired after the second watermark", listener.onExpiryEvents, equalTo(seq(1, threshold)));
+ }
+
+ @Test
+ public void testExpireThresholdWithWatermarkCountEvictionPolicy() throws Exception {
+ int windowLength = WindowManager.EXPIRE_EVENTS_THRESHOLD;
+ EvictionPolicy watermarkCountEvictionPolicy = new WatermarkCountEvictionPolicy(windowLength);
+ testEvictBeforeWatermarkForWatermarkEvictionPolicy(watermarkCountEvictionPolicy, windowLength);
+ }
+
+ @Test
+ public void testExpireThresholdWithWatermarkTimeEvictionPolicy() throws Exception {
+ int windowLength = WindowManager.EXPIRE_EVENTS_THRESHOLD;
+ EvictionPolicy watermarkTimeEvictionPolicy = new WatermarkTimeEvictionPolicy(windowLength);
+ testEvictBeforeWatermarkForWatermarkEvictionPolicy(watermarkTimeEvictionPolicy, windowLength);
+ }
@Test
public void testTimeBasedWindow() throws Exception {
[36/50] [abbrv] storm git commit: Merge branch 'STORM-2525-1.x' of
https://github.com/srdo/storm into asfgit-1.x-branch
Posted by pt...@apache.org.
Merge branch 'STORM-2525-1.x' of https://github.com/srdo/storm into asfgit-1.x-branch
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/7e177122
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/7e177122
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/7e177122
Branch: refs/heads/metrics_v2
Commit: 7e1771220969c68e20c4905d9c6afdfbf7a72bc8
Parents: c632424 4d49287
Author: Stig Rohde Døssing <sr...@apache.org>
Authored: Mon Nov 13 19:23:37 2017 +0100
Committer: Stig Rohde Døssing <sr...@apache.org>
Committed: Mon Nov 13 19:23:37 2017 +0100
----------------------------------------------------------------------
integration-test/config/Vagrantfile | 8 +-
integration-test/config/cluster.xml | 101 -------------------
integration-test/config/install-storm.sh | 1 -
integration-test/config/install-zookeeper.sh | 1 +
integration-test/pom.xml | 5 +
integration-test/run-it.sh | 7 +-
.../topology/window/SlidingTimeCorrectness.java | 4 +-
.../window/SlidingWindowCorrectness.java | 2 -
.../window/TumblingTimeCorrectness.java | 4 +-
.../window/TumblingWindowCorrectness.java | 2 -
.../st/tests/window/SlidingWindowTest.java | 44 ++++----
.../org/apache/storm/st/wrapper/TopoWrap.java | 81 +++++++--------
pom.xml | 19 ++++
storm-core/pom.xml | 7 +-
.../storm/topology/WindowedBoltExecutor.java | 6 +-
.../apache/storm/windowing/EvictionContext.java | 2 +-
.../apache/storm/windowing/EvictionPolicy.java | 2 +-
.../storm/windowing/TimeEvictionPolicy.java | 2 +-
.../windowing/WatermarkCountEvictionPolicy.java | 11 +-
.../windowing/WatermarkTimeEvictionPolicy.java | 10 +-
.../apache/storm/windowing/WindowManager.java | 3 +
.../storm/windowing/WindowManagerTest.java | 48 ++++++++-
22 files changed, 184 insertions(+), 186 deletions(-)
----------------------------------------------------------------------
[20/50] [abbrv] storm git commit: Merge branch 'STORM-2786-1.x' of
https://github.com/revans2/incubator-storm into STORM-2786
Posted by pt...@apache.org.
Merge branch 'STORM-2786-1.x' of https://github.com/revans2/incubator-storm into STORM-2786
STORM-2786: Enable tick tuples for ackers
This closes #2383
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/502ed7e7
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/502ed7e7
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/502ed7e7
Branch: refs/heads/metrics_v2
Commit: 502ed7e7a0a1b459848de69fb5d0b32cf6059082
Parents: 38e308e da04189
Author: Robert Evans <ev...@yahoo-inc.com>
Authored: Wed Oct 25 16:57:22 2017 -0500
Committer: Robert Evans <ev...@yahoo-inc.com>
Committed: Wed Oct 25 16:57:22 2017 -0500
----------------------------------------------------------------------
.../src/clj/org/apache/storm/daemon/executor.clj | 19 ++++++++++---------
1 file changed, 10 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
[14/50] [abbrv] storm git commit: [STORM-2607] Offset consumer + 1
Posted by pt...@apache.org.
[STORM-2607] Offset consumer + 1
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/4fc47b2a
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/4fc47b2a
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/4fc47b2a
Branch: refs/heads/metrics_v2
Commit: 4fc47b2a841bf05ec89c4ccbf089d95c28c965a1
Parents: 39e12aa
Author: Rodolfo Ribeiro <ro...@b2wdigital.com>
Authored: Thu Jun 29 12:06:20 2017 -0300
Committer: Stig Rohde Døssing <sr...@apache.org>
Committed: Tue Oct 17 07:20:38 2017 +0200
----------------------------------------------------------------------
.../kafka/spout/internal/OffsetManager.java | 20 ++++++++++++--------
.../kafka/spout/SingleTopicKafkaSpoutTest.java | 2 +-
2 files changed, 13 insertions(+), 9 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/4fc47b2a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/internal/OffsetManager.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/internal/OffsetManager.java b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/internal/OffsetManager.java
index b6d36d8..1c474e3 100755
--- a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/internal/OffsetManager.java
+++ b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/internal/OffsetManager.java
@@ -71,16 +71,18 @@ public class OffsetManager {
boolean found = false;
long currOffset;
long nextCommitOffset = committedOffset;
+ long lastOffMessageOffset = committedOffset;
KafkaSpoutMessageId nextCommitMsg = null; // this is a convenience variable to make it faster to create OffsetAndMetadata
for (KafkaSpoutMessageId currAckedMsg : ackedMsgs) { // complexity is that of a linear scan on a TreeMap
currOffset = currAckedMsg.offset();
- if (currOffset == nextCommitOffset + 1) { // found the next offset to commit
+ if (currOffset == lastOffMessageOffset + 1) { // found the next offset to commit
found = true;
nextCommitMsg = currAckedMsg;
- nextCommitOffset = currOffset;
- } else if (currOffset > nextCommitOffset + 1) {
- if (emittedOffsets.contains(nextCommitOffset + 1)) {
+ lastOffMessageOffset = currOffset;
+ nextCommitOffset = lastOffMessageOffset + 1;
+ } else if (currOffset > lastOffMessageOffset + 1) {
+ if (emittedOffsets.contains(lastOffMessageOffset + 1)) {
LOG.debug("topic-partition [{}] has non-continuous offset [{}]. It will be processed in a subsequent batch.", tp, currOffset);
break;
} else {
@@ -92,11 +94,12 @@ public class OffsetManager {
first element after committedOffset in the ascending ordered emitted set.
*/
LOG.debug("Processed non contiguous offset. (committedOffset+1) is no longer part of the topic. Committed: [{}], Processed: [{}]", committedOffset, currOffset);
- final Long nextEmittedOffset = emittedOffsets.ceiling(nextCommitOffset + 1);
+ final Long nextEmittedOffset = emittedOffsets.ceiling(lastOffMessageOffset);
if (nextEmittedOffset != null && currOffset == nextEmittedOffset) {
found = true;
nextCommitMsg = currAckedMsg;
- nextCommitOffset = currOffset;
+ lastOffMessageOffset = currOffset;
+ nextCommitOffset = lastOffMessageOffset + 1;
} else {
LOG.debug("topic-partition [{}] has non-continuous offset [{}]. Next Offset to commit should be [{}]", tp, currOffset, nextEmittedOffset);
break;
@@ -112,7 +115,8 @@ public class OffsetManager {
OffsetAndMetadata nextCommitOffsetAndMetadata = null;
if (found) {
nextCommitOffsetAndMetadata = new OffsetAndMetadata(nextCommitOffset, nextCommitMsg.getMetadata(Thread.currentThread()));
- LOG.debug("topic-partition [{}] has offsets [{}-{}] ready to be committed", tp, committedOffset + 1, nextCommitOffsetAndMetadata.offset());
+ LOG.debug("topic-partition [{}] has offsets [{}-{}] ready to be committed",
+ tp, earliestUncommittedOffset, nextCommitOffsetAndMetadata.offset() - 1);
} else {
LOG.debug("topic-partition [{}] has NO offsets ready to be committed", tp);
}
@@ -131,7 +135,7 @@ public class OffsetManager {
*/
public long commit(OffsetAndMetadata committedOffset) {
final long preCommitCommittedOffsets = this.committedOffset;
- long numCommittedOffsets = 0;
+ final long numCommittedOffsets = committedOffset.offset() - this.committedOffset - 1;
this.committedOffset = committedOffset.offset();
for (Iterator<KafkaSpoutMessageId> iterator = ackedMsgs.iterator(); iterator.hasNext();) {
if (iterator.next().offset() <= committedOffset.offset()) {
http://git-wip-us.apache.org/repos/asf/storm/blob/4fc47b2a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/SingleTopicKafkaSpoutTest.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/SingleTopicKafkaSpoutTest.java b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/SingleTopicKafkaSpoutTest.java
index 75fa6b8..59921eb 100644
--- a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/SingleTopicKafkaSpoutTest.java
+++ b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/SingleTopicKafkaSpoutTest.java
@@ -118,7 +118,7 @@ public class SingleTopicKafkaSpoutTest {
Map<TopicPartition, OffsetAndMetadata> commits = commitCapture.getValue();
assertThat("Expected commits for only one topic partition", commits.entrySet().size(), is(1));
OffsetAndMetadata offset = commits.entrySet().iterator().next().getValue();
- assertThat("Expected committed offset to cover all emitted messages", offset.offset(), is(messageCount - 1));
+ assertThat("Expected committed offset to cover all emitted messages", offset.offset(), is(messageCount));
}
@Test
[50/50] [abbrv] storm git commit: Merge branch '1.x-branch' into
metrics_v2
Posted by pt...@apache.org.
Merge branch '1.x-branch' into metrics_v2
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/99bcf684
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/99bcf684
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/99bcf684
Branch: refs/heads/metrics_v2
Commit: 99bcf68449df63a83e124afaadaa61f61da15d22
Parents: a47b5fb 7962639
Author: P. Taylor Goetz <pt...@gmail.com>
Authored: Mon Nov 20 13:52:24 2017 -0500
Committer: P. Taylor Goetz <pt...@gmail.com>
Committed: Mon Nov 20 13:52:24 2017 -0500
----------------------------------------------------------------------
bin/storm.py | 1 -
docs/Trident-API-Overview.md | 2 +-
docs/storm-kafka-client.md | 46 +-
.../storm/hbase/security/HBaseSecurityUtil.java | 2 +-
.../state/TridentResultSetValuesMapper.java | 6 +-
.../apache/storm/druid/bolt/DruidBeamBolt.java | 13 +-
external/storm-eventhubs/pom.xml | 11 -
external/storm-hbase/README.md | 22 +-
.../storm/hdfs/blobstore/HdfsBlobStore.java | 107 +-
.../storm/hdfs/blobstore/HdfsBlobStoreImpl.java | 14 +-
.../storm/hdfs/bolt/AbstractHdfsBolt.java | 48 +-
.../org/apache/storm/hdfs/spout/HdfsSpout.java | 1484 +++++++++---------
.../hdfs/avro/TestFixedAvroSerializer.java | 13 +-
.../hdfs/avro/TestGenericAvroSerializer.java | 11 +-
.../storm/hdfs/blobstore/BlobStoreTest.java | 897 ++++++-----
.../hdfs/blobstore/HdfsBlobStoreImplTest.java | 288 ++--
.../hdfs/bolt/AvroGenericRecordBoltTest.java | 142 +-
.../apache/storm/hdfs/bolt/TestHdfsBolt.java | 108 +-
.../storm/hdfs/bolt/TestSequenceFileBolt.java | 71 +-
.../apache/storm/hdfs/spout/TestDirLock.java | 281 ++--
.../apache/storm/hdfs/spout/TestFileLock.java | 679 ++++----
.../storm/hdfs/spout/TestHdfsSemantics.java | 311 ++--
.../apache/storm/hdfs/spout/TestHdfsSpout.java | 1235 +++++++--------
.../storm/hdfs/spout/TestProgressTracker.java | 179 +--
.../storm/hdfs/testing/MiniDFSClusterRule.java | 78 +
.../apache/storm/jms/spout/JmsSpoutTest.java | 72 +-
.../apache/storm/kafka/spout/KafkaSpout.java | 362 +++--
.../storm/kafka/spout/KafkaSpoutConfig.java | 167 +-
.../KafkaSpoutRetryExponentialBackoff.java | 32 +-
.../kafka/spout/KafkaSpoutRetryService.java | 6 +-
.../apache/storm/kafka/spout/Subscription.java | 5 +-
.../kafka/spout/internal/OffsetManager.java | 102 +-
.../trident/KafkaTridentSpoutBatchMetadata.java | 78 +-
.../spout/trident/KafkaTridentSpoutEmitter.java | 108 +-
.../spout/trident/KafkaTridentSpoutManager.java | 10 +-
.../spout/trident/KafkaTridentSpoutOpaque.java | 18 +-
.../KafkaTridentSpoutOpaqueCoordinator.java | 26 +-
.../spout/trident/TopicPartitionSerializer.java | 47 +
.../storm/kafka/trident/TridentKafkaState.java | 9 +
.../storm/kafka/spout/KafkaSpoutCommitTest.java | 131 --
.../storm/kafka/spout/KafkaSpoutConfigTest.java | 26 +-
.../storm/kafka/spout/KafkaSpoutEmitTest.java | 147 +-
.../KafkaSpoutLogCompactionSupportTest.java | 223 +++
.../spout/KafkaSpoutMessagingGuaranteeTest.java | 207 +++
.../kafka/spout/KafkaSpoutRebalanceTest.java | 79 +-
.../KafkaSpoutRetryExponentialBackoffTest.java | 292 ++++
.../kafka/spout/KafkaSpoutRetryLimitTest.java | 27 +-
.../kafka/spout/MaxUncommittedOffsetTest.java | 64 +-
.../kafka/spout/SingleTopicKafkaSpoutTest.java | 153 +-
.../spout/SingleTopicKafkaUnitSetupHelper.java | 67 +
.../SpoutWithMockedConsumerSetupHelper.java | 136 +-
.../SingleTopicKafkaSpoutConfiguration.java | 25 +-
.../kafka/spout/internal/OffsetManagerTest.java | 187 +++
.../KafkaTridentSpoutBatchMetadataTest.java | 66 +
external/storm-kafka-monitor/pom.xml | 6 -
external/storm-kafka/pom.xml | 29 -
.../apache/storm/kafka/PartitionManager.java | 21 +-
external/storm-kinesis/pom.xml | 11 -
integration-test/README.md | 2 +-
integration-test/config/Vagrantfile | 8 +-
integration-test/config/cluster.xml | 101 --
integration-test/config/install-storm.sh | 1 -
integration-test/config/install-zookeeper.sh | 1 +
integration-test/pom.xml | 5 +
integration-test/run-it.sh | 7 +-
.../topology/window/SlidingTimeCorrectness.java | 4 +-
.../window/SlidingWindowCorrectness.java | 2 -
.../window/TumblingTimeCorrectness.java | 10 +-
.../window/TumblingWindowCorrectness.java | 4 +-
.../test/java/org/apache/storm/st/DemoTest.java | 3 +-
.../st/tests/window/SlidingWindowTest.java | 51 +-
.../st/tests/window/TumblingWindowTest.java | 9 +-
.../apache/storm/st/wrapper/StormCluster.java | 25 +-
.../org/apache/storm/st/wrapper/TopoWrap.java | 87 +-
pom.xml | 47 +-
storm-core/pom.xml | 37 +-
.../clj/org/apache/storm/daemon/executor.clj | 19 +-
.../clj/org/apache/storm/daemon/logviewer.clj | 12 +-
storm-core/src/clj/org/apache/storm/ui/core.clj | 3 +-
.../src/clj/org/apache/storm/ui/helpers.clj | 21 +-
storm-core/src/jvm/org/apache/storm/Config.java | 6 +-
.../apache/storm/blobstore/BlobStoreUtils.java | 8 +-
.../storm/blobstore/LocalFsBlobStore.java | 6 +-
.../apache/storm/testing/FixedTupleSpout.java | 18 +-
.../storm/topology/OutputFieldsGetter.java | 3 +
.../storm/topology/WindowedBoltExecutor.java | 9 +-
.../spout/IOpaquePartitionedTridentSpout.java | 22 +-
.../topology/state/TransactionalState.java | 14 +-
.../jvm/org/apache/storm/utils/ConfigUtils.java | 16 +-
.../apache/storm/windowing/EvictionContext.java | 2 +-
.../apache/storm/windowing/EvictionPolicy.java | 2 +-
.../storm/windowing/TimeEvictionPolicy.java | 2 +-
.../storm/windowing/TimeTriggerPolicy.java | 8 +-
.../windowing/WaterMarkEventGenerator.java | 25 +-
.../windowing/WatermarkCountEvictionPolicy.java | 11 +-
.../windowing/WatermarkTimeEvictionPolicy.java | 10 +-
.../apache/storm/windowing/WindowManager.java | 3 +
.../org/apache/storm/integration_test.clj | 53 +-
.../clj/org/apache/storm/transactional_test.clj | 4 +-
.../clj/org/apache/storm/trident/state_test.clj | 4 +-
.../storm/blobstore/BlobStoreUtilsTest.java | 135 ++
.../storm/blobstore/BlobSynchronizerTest.java | 23 +-
.../blobstore/MockZookeeperClientBuilder.java | 100 ++
.../windowing/WaterMarkEventGeneratorTest.java | 6 +
.../storm/windowing/WindowManagerTest.java | 48 +-
storm-core/test/resources/log4j2-test.xml | 2 +
106 files changed, 5938 insertions(+), 4049 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/99bcf684/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/99bcf684/storm-core/pom.xml
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/99bcf684/storm-core/src/clj/org/apache/storm/daemon/executor.clj
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/99bcf684/storm-core/src/jvm/org/apache/storm/Config.java
----------------------------------------------------------------------
[38/50] [abbrv] storm git commit: Merge branch 'STORM-2811-1.x' of
https://github.com/srdo/storm into STORM-2811-1.x-merge
Posted by pt...@apache.org.
Merge branch 'STORM-2811-1.x' of https://github.com/srdo/storm into STORM-2811-1.x-merge
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/a5ed0f91
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/a5ed0f91
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/a5ed0f91
Branch: refs/heads/metrics_v2
Commit: a5ed0f918af8aba055258ce783f54eb21586deab
Parents: c632424 bbd582b
Author: Jungtaek Lim <ka...@gmail.com>
Authored: Tue Nov 14 08:02:01 2017 +0900
Committer: Jungtaek Lim <ka...@gmail.com>
Committed: Tue Nov 14 08:02:01 2017 +0900
----------------------------------------------------------------------
integration-test/config/Vagrantfile | 8 +-
integration-test/config/cluster.xml | 101 -------------------
integration-test/config/install-storm.sh | 1 -
integration-test/config/install-zookeeper.sh | 1 +
integration-test/pom.xml | 5 +
integration-test/run-it.sh | 7 +-
.../topology/window/SlidingTimeCorrectness.java | 4 +-
.../window/SlidingWindowCorrectness.java | 2 -
.../window/TumblingTimeCorrectness.java | 4 +-
.../window/TumblingWindowCorrectness.java | 2 -
.../test/java/org/apache/storm/st/DemoTest.java | 1 +
.../st/tests/window/SlidingWindowTest.java | 45 +++++----
.../st/tests/window/TumblingWindowTest.java | 3 +-
.../apache/storm/st/wrapper/StormCluster.java | 2 +-
.../org/apache/storm/st/wrapper/TopoWrap.java | 81 +++++++--------
pom.xml | 19 ++++
storm-core/pom.xml | 7 +-
.../storm/topology/WindowedBoltExecutor.java | 6 +-
.../apache/storm/windowing/EvictionContext.java | 2 +-
.../apache/storm/windowing/EvictionPolicy.java | 2 +-
.../storm/windowing/TimeEvictionPolicy.java | 2 +-
.../windowing/WatermarkCountEvictionPolicy.java | 11 +-
.../windowing/WatermarkTimeEvictionPolicy.java | 10 +-
.../apache/storm/windowing/WindowManager.java | 3 +
.../storm/windowing/WindowManagerTest.java | 48 ++++++++-
25 files changed, 189 insertions(+), 188 deletions(-)
----------------------------------------------------------------------
[04/50] [abbrv] storm git commit: STORM-2648/STORM-2357: Add
storm-kafka-client support for at-most-once processing and a toggle for
whether messages should be emitted with a message id when not using
at-least-once
Posted by pt...@apache.org.
STORM-2648/STORM-2357: Add storm-kafka-client support for at-most-once processing and a toggle for whether messages should be emitted with a message id when not using at-least-once
* Minor refactor of emit statements
* Add tests for at-most-once and any-times mode, deduplicate some test code in other tests
* Fix rebase conflicts and fix leaking state through unit test retry service
* Update storm-kafka-client doc
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/11a7a157
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/11a7a157
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/11a7a157
Branch: refs/heads/metrics_v2
Commit: 11a7a15746010e0baaa5a4db70374dff1d4ab800
Parents: fca692d
Author: Stig Rohde Døssing <sr...@apache.org>
Authored: Mon Jul 31 20:26:55 2017 +0200
Committer: Stig Rohde Døssing <sr...@apache.org>
Committed: Mon Oct 2 18:08:12 2017 +0200
----------------------------------------------------------------------
docs/storm-kafka-client.md | 31 ++-
.../apache/storm/kafka/spout/KafkaSpout.java | 175 +++++++++-------
.../storm/kafka/spout/KafkaSpoutConfig.java | 99 ++++++++-
.../storm/kafka/spout/KafkaSpoutCommitTest.java | 5 +-
.../storm/kafka/spout/KafkaSpoutEmitTest.java | 8 +-
.../spout/KafkaSpoutMessagingGuaranteeTest.java | 207 +++++++++++++++++++
.../kafka/spout/KafkaSpoutRebalanceTest.java | 8 +-
.../kafka/spout/KafkaSpoutRetryLimitTest.java | 5 +-
.../kafka/spout/MaxUncommittedOffsetTest.java | 32 +--
.../kafka/spout/SingleTopicKafkaSpoutTest.java | 47 ++---
.../spout/SingleTopicKafkaUnitSetupHelper.java | 67 ++++++
.../SingleTopicKafkaSpoutConfiguration.java | 23 +--
12 files changed, 522 insertions(+), 185 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/11a7a157/docs/storm-kafka-client.md
----------------------------------------------------------------------
diff --git a/docs/storm-kafka-client.md b/docs/storm-kafka-client.md
index 93d622e..841e8ef 100644
--- a/docs/storm-kafka-client.md
+++ b/docs/storm-kafka-client.md
@@ -341,33 +341,32 @@ Depending on the structure of your Kafka cluster, distribution of the data, and
### Default values
-Currently the Kafka spout has has the following default values, which have shown to give good performance in the test environment as described in this [blog post] (https://hortonworks.com/blog/microbenchmarking-storm-1-0-performance/)
+Currently the Kafka spout has has the following default values, which have been shown to give good performance in the test environment as described in this [blog post] (https://hortonworks.com/blog/microbenchmarking-storm-1-0-performance/)
* poll.timeout.ms = 200
* offset.commit.period.ms = 30000 (30s)
* max.uncommitted.offsets = 10000000
<br/>
-# Kafka AutoCommitMode
+# Messaging reliability modes
-If reliability isn't important to you -- that is, you don't care about losing tuples in failure situations --, and want to remove the overhead of tuple tracking, then you can run a KafkaSpout with AutoCommitMode.
-
-To enable it, you need to:
-
-* set Config.TOPOLOGY_ACKERS to 0;
-* enable *AutoCommitMode* in Kafka consumer configuration;
-
-Here's one example to set AutoCommitMode in KafkaSpout:
+In some cases you may not need or want the spout to guarantee at-least-once processing of messages. The spout also supports at-most-once and any-times modes. At-most-once guarantees that any tuple emitted to the topology will never be reemitted. Any-times makes no guarantees, but may reduce the overhead of committing offsets to Kafka in cases where you truly don't care how many times a message is processed.
+To set the processing guarantee, use the KafkaSpoutConfig.Builder.setProcessingGuarantee method, e.g.
```java
KafkaSpoutConfig<String, String> kafkaConf = KafkaSpoutConfig
- .builder(String bootstrapServers, String ... topics)
- .setProp(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "true")
- .setFirstPollOffsetStrategy(FirstPollOffsetStrategy.EARLIEST)
- .build();
+ .builder(String bootstrapServers, String ... topics)
+ .setProcessingGuarantee(ProcessingGuarantee.AT_MOST_ONCE)
```
-*Note that it's not exactly At-Most-Once in Storm, as offset is committed periodically by Kafka consumer, some tuples could be replayed when KafkaSpout is crashed.*
-
+The spout will disable tuple tracking for emitted tuples by default when you use at-most-once or any-times. In some cases you may want to enable tracking anyway, because tuple tracking is necessary for some features of Storm, e.g. showing complete latency in Storm UI, or enabling backpressure through the `Config.TOPOLOGY_MAX_SPOUT_PENDING` parameter.
+If you need to enable tracking, use the KafkaSpoutConfig.Builder.setForceEnableTupleTracking method, e.g.
+```java
+KafkaSpoutConfig<String, String> kafkaConf = KafkaSpoutConfig
+ .builder(String bootstrapServers, String ... topics)
+ .setProcessingGuarantee(ProcessingGuarantee.AT_MOST_ONCE)
+ .setForceEnableTupleTracking(true)
+```
+Note that this setting has no effect in at-least-once mode, where tuple tracking is always enabled.
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/storm/blob/11a7a157/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpout.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpout.java b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpout.java
index 9f806b5..fbd869c 100644
--- a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpout.java
+++ b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpout.java
@@ -52,10 +52,12 @@ import org.apache.storm.spout.SpoutOutputCollector;
import org.apache.storm.task.TopologyContext;
import org.apache.storm.topology.OutputFieldsDeclarer;
import org.apache.storm.topology.base.BaseRichSpout;
+import org.apache.storm.utils.Utils;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
public class KafkaSpout<K, V> extends BaseRichSpout {
+
private static final long serialVersionUID = 4151921085047987154L;
//Initial delay for the commit and subscription refresh timers
public static final long TIMER_DELAY_MS = 500;
@@ -66,26 +68,36 @@ public class KafkaSpout<K, V> extends BaseRichSpout {
// Kafka
private final KafkaSpoutConfig<K, V> kafkaSpoutConfig;
- private KafkaConsumerFactory kafkaConsumerFactory;
+ private KafkaConsumerFactory<K, V> kafkaConsumerFactory;
private transient KafkaConsumer<K, V> kafkaConsumer;
- private transient boolean consumerAutoCommitMode;
// Bookkeeping
- private transient FirstPollOffsetStrategy firstPollOffsetStrategy; // Strategy to determine the fetch offset of the first realized by the spout upon activation
- private transient KafkaSpoutRetryService retryService; // Class that has the logic to handle tuple failure
- private transient KafkaTupleListener tupleListener; // Handles tuple events (emit, ack etc.)
- private transient Timer commitTimer; // timer == null for auto commit mode
- private transient boolean initialized; // Flag indicating that the spout is still undergoing initialization process.
+ // Strategy to determine the fetch offset of the first realized by the spout upon activation
+ private transient FirstPollOffsetStrategy firstPollOffsetStrategy;
+ // Class that has the logic to handle tuple failure.
+ private transient KafkaSpoutRetryService retryService;
+ // Handles tuple events (emit, ack etc.)
+ private transient KafkaTupleListener tupleListener;
+ // timer == null for modes other than at-least-once
+ private transient Timer commitTimer;
+ // Flag indicating that the spout is still undergoing initialization process.
+ private transient boolean initialized;
// Initialization is only complete after the first call to KafkaSpoutConsumerRebalanceListener.onPartitionsAssigned()
- private transient Map<TopicPartition, OffsetManager> offsetManagers;// Tuples that were successfully acked/emitted. These tuples will be committed periodically when the commit timer expires, or after a consumer rebalance, or during close/deactivate
- private transient Set<KafkaSpoutMessageId> emitted; // Tuples that have been emitted but that are "on the wire", i.e. pending being acked or failed. Not used if it's AutoCommitMode
- private transient Iterator<ConsumerRecord<K, V>> waitingToEmit; // Records that have been polled and are queued to be emitted in the nextTuple() call. One record is emitted per nextTuple()
- private transient long numUncommittedOffsets; // Number of offsets that have been polled and emitted but not yet been committed. Not used if auto commit mode is enabled.
- private transient Timer refreshSubscriptionTimer; // Triggers when a subscription should be refreshed
+ // Tuples that were successfully acked/emitted. These tuples will be committed periodically when the commit timer expires,
+ //or after a consumer rebalance, or during close/deactivate. Always empty if not using at-least-once mode.
+ private transient Map<TopicPartition, OffsetManager> offsetManagers;
+ // Tuples that have been emitted but that are "on the wire", i.e. pending being acked or failed.
+ // Always empty if not using at-least-once mode.
+ private transient Set<KafkaSpoutMessageId> emitted;
+ // Records that have been polled and are queued to be emitted in the nextTuple() call. One record is emitted per nextTuple()
+ private transient Iterator<ConsumerRecord<K, V>> waitingToEmit;
+ // Number of offsets that have been polled and emitted but not yet been committed. Not used if auto commit mode is enabled.
+ private transient long numUncommittedOffsets;
+ // Triggers when a subscription should be refreshed
+ private transient Timer refreshSubscriptionTimer;
private transient TopologyContext context;
-
public KafkaSpout(KafkaSpoutConfig<K, V> kafkaSpoutConfig) {
this(kafkaSpoutConfig, new KafkaConsumerFactoryDefault<K, V>());
}
@@ -107,9 +119,7 @@ public class KafkaSpout<K, V> extends BaseRichSpout {
// Offset management
firstPollOffsetStrategy = kafkaSpoutConfig.getFirstPollOffsetStrategy();
- // with AutoCommitMode, offset will be periodically committed in the background by Kafka consumer
- consumerAutoCommitMode = kafkaSpoutConfig.isConsumerAutoCommitMode();
-
+
// Retries management
retryService = kafkaSpoutConfig.getRetryService();
@@ -117,7 +127,8 @@ public class KafkaSpout<K, V> extends BaseRichSpout {
tupleListener.open(conf, context);
- if (!consumerAutoCommitMode) { // If it is auto commit, no need to commit offsets manually
+ if (isAtLeastOnce()) {
+ // Only used if the spout commits offsets for acked tuples
commitTimer = new Timer(TIMER_DELAY_MS, kafkaSpoutConfig.getOffsetsCommitPeriodMs(), TimeUnit.MILLISECONDS);
}
refreshSubscriptionTimer = new Timer(TIMER_DELAY_MS, kafkaSpoutConfig.getPartitionRefreshPeriodMs(), TimeUnit.MILLISECONDS);
@@ -129,14 +140,18 @@ public class KafkaSpout<K, V> extends BaseRichSpout {
LOG.info("Kafka Spout opened with the following configuration: {}", kafkaSpoutConfig);
}
- // =========== Consumer Rebalance Listener - On the same thread as the caller ===========
+ private boolean isAtLeastOnce() {
+ return kafkaSpoutConfig.getProcessingGuarantee() == KafkaSpoutConfig.ProcessingGuarantee.AT_LEAST_ONCE;
+ }
+ // =========== Consumer Rebalance Listener - On the same thread as the caller ===========
private class KafkaSpoutConsumerRebalanceListener implements ConsumerRebalanceListener {
+
@Override
public void onPartitionsRevoked(Collection<TopicPartition> partitions) {
LOG.info("Partitions revoked. [consumer-group={}, consumer={}, topic-partitions={}]",
- kafkaSpoutConfig.getConsumerGroupId(), kafkaConsumer, partitions);
- if (!consumerAutoCommitMode && initialized) {
+ kafkaSpoutConfig.getConsumerGroupId(), kafkaConsumer, partitions);
+ if (isAtLeastOnce() && initialized) {
initialized = false;
commitOffsetsForAckedTuples();
}
@@ -145,36 +160,39 @@ public class KafkaSpout<K, V> extends BaseRichSpout {
@Override
public void onPartitionsAssigned(Collection<TopicPartition> partitions) {
LOG.info("Partitions reassignment. [task-ID={}, consumer-group={}, consumer={}, topic-partitions={}]",
- context.getThisTaskId(), kafkaSpoutConfig.getConsumerGroupId(), kafkaConsumer, partitions);
+ context.getThisTaskId(), kafkaSpoutConfig.getConsumerGroupId(), kafkaConsumer, partitions);
tupleListener.onPartitionsReassigned(partitions);
initialize(partitions);
}
private void initialize(Collection<TopicPartition> partitions) {
- if (!consumerAutoCommitMode) {
+ if (isAtLeastOnce()) {
offsetManagers.keySet().retainAll(partitions); // remove from acked all partitions that are no longer assigned to this spout
- }
-
- retryService.retainAll(partitions);
-
- /*
- * Emitted messages for partitions that are no longer assigned to this spout can't
- * be acked and should not be retried, hence remove them from emitted collection.
- */
- Set<TopicPartition> partitionsSet = new HashSet<>(partitions);
- Iterator<KafkaSpoutMessageId> msgIdIterator = emitted.iterator();
- while (msgIdIterator.hasNext()) {
- KafkaSpoutMessageId msgId = msgIdIterator.next();
- if (!partitionsSet.contains(msgId.getTopicPartition())) {
- msgIdIterator.remove();
+ retryService.retainAll(partitions);
+
+ /*
+ * Emitted messages for partitions that are no longer assigned to this spout can't
+ * be acked and should not be retried, hence remove them from emitted collection.
+ */
+ Set<TopicPartition> partitionsSet = new HashSet<>(partitions);
+ Iterator<KafkaSpoutMessageId> msgIdIterator = emitted.iterator();
+ while (msgIdIterator.hasNext()) {
+ KafkaSpoutMessageId msgId = msgIdIterator.next();
+ if (!partitionsSet.contains(msgId.getTopicPartition())) {
+ msgIdIterator.remove();
+ }
}
}
for (TopicPartition tp : partitions) {
final OffsetAndMetadata committedOffset = kafkaConsumer.committed(tp);
final long fetchOffset = doSeek(tp, committedOffset);
- setAcked(tp, fetchOffset);
+ // Add offset managers for the new partitions.
+ // If this partition was previously assigned to this spout, leave the acked offsets as they were to resume where it left off
+ if (isAtLeastOnce() && !offsetManagers.containsKey(tp)) {
+ offsetManagers.put(tp, new OffsetManager(tp, fetchOffset));
+ }
}
initialized = true;
LOG.info("Initialization complete");
@@ -209,15 +227,7 @@ public class KafkaSpout<K, V> extends BaseRichSpout {
}
}
- private void setAcked(TopicPartition tp, long fetchOffset) {
- // If this partition was previously assigned to this spout, leave the acked offsets as they were to resume where it left off
- if (!consumerAutoCommitMode && !offsetManagers.containsKey(tp)) {
- offsetManagers.put(tp, new OffsetManager(tp, fetchOffset));
- }
- }
-
// ======== Next Tuple =======
-
@Override
public void nextTuple() {
try {
@@ -252,24 +262,25 @@ public class KafkaSpout<K, V> extends BaseRichSpout {
}
private boolean commit() {
- return !consumerAutoCommitMode && commitTimer.isExpiredResetOnTrue(); // timer != null for non auto commit mode
+ return isAtLeastOnce() && commitTimer.isExpiredResetOnTrue(); // timer != null for non auto commit mode
}
private boolean poll() {
final int maxUncommittedOffsets = kafkaSpoutConfig.getMaxUncommittedOffsets();
final int readyMessageCount = retryService.readyMessageCount();
- final boolean poll = !waitingToEmit() &&
+ final boolean poll = !waitingToEmit()
//Check that the number of uncommitted, nonretriable tuples is less than the maxUncommittedOffsets limit
- //Accounting for retriable tuples this way still guarantees that the limit is followed on a per partition basis, and prevents locking up the spout when there are too many retriable tuples
- (numUncommittedOffsets - readyMessageCount < maxUncommittedOffsets ||
- consumerAutoCommitMode);
-
+ //Accounting for retriable tuples this way still guarantees that the limit is followed on a per partition basis,
+ //and prevents locking up the spout when there are too many retriable tuples
+ && (numUncommittedOffsets - readyMessageCount < maxUncommittedOffsets
+ || !isAtLeastOnce());
+
if (!poll) {
if (waitingToEmit()) {
LOG.debug("Not polling. Tuples waiting to be emitted. [{}] uncommitted offsets across all topic partitions", numUncommittedOffsets);
}
- if (numUncommittedOffsets >= maxUncommittedOffsets && !consumerAutoCommitMode) {
+ if (numUncommittedOffsets >= maxUncommittedOffsets && isAtLeastOnce()) {
LOG.debug("Not polling. [{}] uncommitted offsets across all topic partitions has reached the threshold of [{}]", numUncommittedOffsets, maxUncommittedOffsets);
}
}
@@ -297,6 +308,10 @@ public class KafkaSpout<K, V> extends BaseRichSpout {
final ConsumerRecords<K, V> consumerRecords = kafkaConsumer.poll(kafkaSpoutConfig.getPollTimeoutMs());
final int numPolledRecords = consumerRecords.count();
LOG.debug("Polled [{}] records from Kafka. [{}] uncommitted offsets across all topic partitions", numPolledRecords, numUncommittedOffsets);
+ if (kafkaSpoutConfig.getProcessingGuarantee() == KafkaSpoutConfig.ProcessingGuarantee.AT_MOST_ONCE) {
+ //Commit polled records immediately to ensure delivery is at-most-once.
+ kafkaConsumer.commitSync();
+ }
return consumerRecords;
}
@@ -335,11 +350,14 @@ public class KafkaSpout<K, V> extends BaseRichSpout {
final boolean isScheduled = retryService.isScheduled(msgId);
// not scheduled <=> never failed (i.e. never emitted), or scheduled and ready to be retried
if (!isScheduled || retryService.isReady(msgId)) {
- if (consumerAutoCommitMode) {
- if (tuple instanceof KafkaTuple) {
- collector.emit(((KafkaTuple) tuple).getStream(), tuple);
+ String stream = tuple instanceof KafkaTuple ? ((KafkaTuple) tuple).getStream() : Utils.DEFAULT_STREAM_ID;
+ if (!isAtLeastOnce()) {
+ if (kafkaSpoutConfig.getForceEnableTupleTracking()) {
+ collector.emit(stream, tuple, msgId);
+ LOG.trace("Emitted tuple [{}] for record [{}] with msgId [{}]", tuple, record, msgId);
} else {
- collector.emit(tuple);
+ collector.emit(stream, tuple);
+ LOG.trace("Emitted tuple [{}] for record [{}]", tuple, record);
}
} else {
emitted.add(msgId);
@@ -349,15 +367,10 @@ public class KafkaSpout<K, V> extends BaseRichSpout {
} else { //New tuple, hence increment the uncommitted offset counter
numUncommittedOffsets++;
}
-
- if (tuple instanceof KafkaTuple) {
- collector.emit(((KafkaTuple) tuple).getStream(), tuple, msgId);
- } else {
- collector.emit(tuple, msgId);
- }
+ collector.emit(stream, tuple, msgId);
tupleListener.onEmit(tuple, msgId);
+ LOG.trace("Emitted tuple [{}] for record [{}] with msgId [{}]", tuple, record, msgId);
}
- LOG.trace("Emitted tuple [{}] for record [{}] with msgId [{}]", tuple, record, msgId);
return true;
}
} else {
@@ -407,31 +420,37 @@ public class KafkaSpout<K, V> extends BaseRichSpout {
}
// ======== Ack =======
-
@Override
public void ack(Object messageId) {
+ if (!isAtLeastOnce()) {
+ // Only need to keep track of acked tuples if commits are done based on acks
+ return;
+ }
+
final KafkaSpoutMessageId msgId = (KafkaSpoutMessageId) messageId;
if (!emitted.contains(msgId)) {
if (msgId.isEmitted()) {
- LOG.debug("Received ack for message [{}], associated with tuple emitted for a ConsumerRecord that " +
- "came from a topic-partition that this consumer group instance is no longer tracking " +
- "due to rebalance/partition reassignment. No action taken.", msgId);
+ LOG.debug("Received ack for message [{}], associated with tuple emitted for a ConsumerRecord that "
+ + "came from a topic-partition that this consumer group instance is no longer tracking "
+ + "due to rebalance/partition reassignment. No action taken.", msgId);
} else {
LOG.debug("Received direct ack for message [{}], associated with null tuple", msgId);
}
} else {
- if (!consumerAutoCommitMode) { // Only need to keep track of acked tuples if commits are not done automatically
- offsetManagers.get(msgId.getTopicPartition()).addToAckMsgs(msgId);
- }
+ offsetManagers.get(msgId.getTopicPartition()).addToAckMsgs(msgId);
emitted.remove(msgId);
}
tupleListener.onAck(msgId);
}
// ======== Fail =======
-
@Override
public void fail(Object messageId) {
+ if (!isAtLeastOnce()) {
+ // Only need to keep track of failed tuples if commits are done based on acks
+ return;
+ }
+
final KafkaSpoutMessageId msgId = (KafkaSpoutMessageId) messageId;
if (!emitted.contains(msgId)) {
LOG.debug("Received fail for tuple this spout is no longer tracking. Partitions may have been reassigned. Ignoring message [{}]", msgId);
@@ -451,7 +470,6 @@ public class KafkaSpout<K, V> extends BaseRichSpout {
}
// ======== Activate / Deactivate / Close / Declare Outputs =======
-
@Override
public void activate() {
try {
@@ -487,7 +505,7 @@ public class KafkaSpout<K, V> extends BaseRichSpout {
private void shutdown() {
try {
- if (!consumerAutoCommitMode) {
+ if (isAtLeastOnce()) {
commitOffsetsForAckedTuples();
}
} finally {
@@ -506,10 +524,10 @@ public class KafkaSpout<K, V> extends BaseRichSpout {
@Override
public String toString() {
- return "KafkaSpout{" +
- "offsetManagers =" + offsetManagers +
- ", emitted=" + emitted +
- "}";
+ return "KafkaSpout{"
+ + "offsetManagers =" + offsetManagers
+ + ", emitted=" + emitted
+ + "}";
}
@Override
@@ -532,6 +550,3 @@ public class KafkaSpout<K, V> extends BaseRichSpout {
return kafkaSpoutConfig.getSubscription().getTopicsString();
}
}
-
-
-
http://git-wip-us.apache.org/repos/asf/storm/blob/11a7a157/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpoutConfig.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpoutConfig.java b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpoutConfig.java
index 5cad0f4..b7de812 100644
--- a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpoutConfig.java
+++ b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpoutConfig.java
@@ -33,14 +33,18 @@ import org.apache.kafka.clients.consumer.ConsumerRecord;
import org.apache.kafka.clients.consumer.KafkaConsumer;
import org.apache.kafka.common.serialization.Deserializer;
import org.apache.kafka.common.serialization.StringDeserializer;
+import org.apache.storm.Config;
import org.apache.storm.kafka.spout.KafkaSpoutRetryExponentialBackoff.TimeInterval;
import org.apache.storm.tuple.Fields;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
/**
* KafkaSpoutConfig defines the required configuration to connect a consumer to a consumer group, as well as the subscribing topics
*/
public class KafkaSpoutConfig<K, V> implements Serializable {
+ private static final Logger LOG = LoggerFactory.getLogger(KafkaSpoutConfig.class);
private static final long serialVersionUID = 141902646130682494L;
// 200ms
public static final long DEFAULT_POLL_TIMEOUT_MS = 200;
@@ -56,6 +60,7 @@ public class KafkaSpoutConfig<K, V> implements Serializable {
public static final KafkaSpoutRetryService DEFAULT_RETRY_SERVICE =
new KafkaSpoutRetryExponentialBackoff(TimeInterval.seconds(0), TimeInterval.milliSeconds(2),
DEFAULT_MAX_RETRIES, TimeInterval.seconds(10));
+ public static final ProcessingGuarantee DEFAULT_PROCESSING_GUARANTEE = ProcessingGuarantee.AT_LEAST_ONCE;
public static final KafkaTupleListener DEFAULT_TUPLE_LISTENER = new EmptyKafkaTupleListener();
@@ -77,6 +82,8 @@ public class KafkaSpoutConfig<K, V> implements Serializable {
private final Class<? extends Deserializer<K>> keyDesClazz;
private final SerializableDeserializer<V> valueDes;
private final Class<? extends Deserializer<V>> valueDesClazz;
+ private final ProcessingGuarantee processingGuarantee;
+ private final boolean forceEnableTupleTracking;
/**
* Creates a new KafkaSpoutConfig using a Builder.
@@ -84,7 +91,8 @@ public class KafkaSpoutConfig<K, V> implements Serializable {
* @param builder The Builder to construct the KafkaSpoutConfig from
*/
public KafkaSpoutConfig(Builder<K, V> builder) {
- this.kafkaProps = setDefaultsAndGetKafkaProps(builder.kafkaProps);
+ setAutoCommitMode(builder);
+ this.kafkaProps = builder.kafkaProps;
this.subscription = builder.subscription;
this.translator = builder.translator;
this.pollTimeoutMs = builder.pollTimeoutMs;
@@ -99,6 +107,8 @@ public class KafkaSpoutConfig<K, V> implements Serializable {
this.keyDesClazz = builder.keyDesClazz;
this.valueDes = builder.valueDes;
this.valueDesClazz = builder.valueDesClazz;
+ this.processingGuarantee = builder.processingGuarantee;
+ this.forceEnableTupleTracking = builder.forceEnableTupleTracking;
}
/**
@@ -124,6 +134,25 @@ public class KafkaSpoutConfig<K, V> implements Serializable {
UNCOMMITTED_LATEST
}
+ /**
+ * The processing guarantee supported by the spout. This parameter affects when the spout commits offsets to Kafka, marking them as
+ * processed.
+ *
+ * <ul>
+ * <li>AT_LEAST_ONCE means that the Kafka spout considers an offset ready for commit once a tuple corresponding to that offset has been
+ * acked on the spout. This corresponds to an at-least-once guarantee.</li>
+ * <li>ANY_TIMES means that the Kafka spout may commit polled offsets at any time. This means the message may be processed any number of
+ * times (including 0), and causes the spout to enable auto offset committing on the underlying consumer.</li>
+ * <li>AT_MOST_ONCE means that the spout will commit polled offsets before emitting them to the topology. This guarantees at-most-once
+ * processing.</li>
+ * </ul>
+ */
+ public static enum ProcessingGuarantee {
+ AT_LEAST_ONCE,
+ ANY_TIMES,
+ AT_MOST_ONCE
+ }
+
public static class Builder<K, V> {
private final Map<String, Object> kafkaProps;
@@ -141,6 +170,8 @@ public class KafkaSpoutConfig<K, V> implements Serializable {
private KafkaTupleListener tupleListener = DEFAULT_TUPLE_LISTENER;
private long partitionRefreshPeriodMs = DEFAULT_PARTITION_REFRESH_PERIOD_MS;
private boolean emitNullTuples = false;
+ private ProcessingGuarantee processingGuarantee = DEFAULT_PROCESSING_GUARANTEE;
+ private boolean forceEnableTupleTracking = false;
public Builder(String bootstrapServers, String... topics) {
this(bootstrapServers, (SerializableDeserializer) null, (SerializableDeserializer) null, new ManualPartitionSubscription(new RoundRobinManualPartitioner(), new NamedTopicFilter(topics)));
@@ -482,6 +513,8 @@ public class KafkaSpoutConfig<K, V> implements Serializable {
/**
* Specifies the period, in milliseconds, the offset commit task is periodically called. Default is 15s.
*
+ * <p>This setting only has an effect if the configured {@link ProcessingGuarantee} is {@link ProcessingGuarantee#AT_LEAST_ONCE}.
+ *
* @param offsetCommitPeriodMs time in ms
*/
public Builder<K, V> setOffsetCommitPeriodMs(long offsetCommitPeriodMs) {
@@ -495,6 +528,8 @@ public class KafkaSpoutConfig<K, V> implements Serializable {
* below the threshold. The default is {@link #DEFAULT_MAX_UNCOMMITTED_OFFSETS}. Note that this limit can in some cases be exceeded,
* but no partition will exceed this limit by more than maxPollRecords - 1.
*
+ * <p>This setting only has an effect if the configured {@link ProcessingGuarantee} is {@link ProcessingGuarantee#AT_LEAST_ONCE}.
+ *
* @param maxUncommittedOffsets max number of records that can be be pending commit
*/
public Builder<K, V> setMaxUncommittedOffsets(int maxUncommittedOffsets) {
@@ -516,6 +551,8 @@ public class KafkaSpoutConfig<K, V> implements Serializable {
/**
* Sets the retry service for the spout to use.
*
+ * <p>This setting only has an effect if the configured {@link ProcessingGuarantee} is {@link ProcessingGuarantee#AT_LEAST_ONCE}.
+ *
* @param retryService the new retry service
* @return the builder (this).
*/
@@ -593,6 +630,32 @@ public class KafkaSpoutConfig<K, V> implements Serializable {
return this;
}
+ /**
+ * Specifies which processing guarantee the spout should offer. Refer to the documentation for {@link ProcessingGuarantee}.
+ *
+ * @param processingGuarantee The processing guarantee the spout should offer.
+ */
+ public Builder<K, V> setProcessingGuarantee(ProcessingGuarantee processingGuarantee) {
+ this.processingGuarantee = processingGuarantee;
+ return this;
+ }
+
+ /**
+ * Specifies whether the spout should require Storm to track emitted tuples when using a {@link ProcessingGuarantee} other than
+ * {@link ProcessingGuarantee#AT_LEAST_ONCE}. The spout will always track emitted tuples when offering at-least-once guarantees
+ * regardless of this setting. This setting is false by default.
+ *
+ * <p>Enabling tracking can be useful even in cases where reliability is not a concern, because it allows
+ * {@link Config#TOPOLOGY_MAX_SPOUT_PENDING} to have an effect, and enables some spout metrics (e.g. complete-latency) that would
+ * otherwise be disabled.
+ *
+ * @param forceEnableTupleTracking true if Storm should track emitted tuples, false otherwise
+ */
+ public Builder<K, V> setForceEnableTupleTracking(boolean forceEnableTupleTracking) {
+ this.forceEnableTupleTracking = forceEnableTupleTracking;
+ return this;
+ }
+
public KafkaSpoutConfig<K, V> build() {
return new KafkaSpoutConfig<>(this);
}
@@ -637,12 +700,24 @@ public class KafkaSpoutConfig<K, V> implements Serializable {
return builder;
}
- private static Map<String, Object> setDefaultsAndGetKafkaProps(Map<String, Object> kafkaProps) {
- // set defaults for properties not specified
- if (!kafkaProps.containsKey(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG)) {
- kafkaProps.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "false");
+ private static void setAutoCommitMode(Builder<?, ?> builder) {
+ if (builder.kafkaProps.containsKey(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG)) {
+ LOG.warn("Do not set " + ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG + " manually."
+ + " Instead use KafkaSpoutConfig.Builder.setProcessingGuarantee."
+ + " This will be treated as an error in the next major release."
+ + " For now the spout will be configured to behave like it would have in pre-1.2.0 releases.");
+ boolean enableAutoCommit = (boolean)builder.kafkaProps.get(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG);
+ if(enableAutoCommit) {
+ builder.processingGuarantee = ProcessingGuarantee.ANY_TIMES;
+ } else {
+ builder.processingGuarantee = ProcessingGuarantee.AT_LEAST_ONCE;
+ }
+ }
+ if (builder.processingGuarantee == ProcessingGuarantee.ANY_TIMES) {
+ builder.kafkaProps.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "true");
+ } else {
+ builder.kafkaProps.put(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG, "false");
}
- return kafkaProps;
}
/**
@@ -700,10 +775,22 @@ public class KafkaSpoutConfig<K, V> implements Serializable {
return offsetCommitPeriodMs;
}
+ /**
+ * @deprecated Use {@link #getProcessingGuarantee()} instead.
+ */
+ @Deprecated
public boolean isConsumerAutoCommitMode() {
return kafkaProps.get(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG) == null // default is false
|| Boolean.valueOf((String) kafkaProps.get(ConsumerConfig.ENABLE_AUTO_COMMIT_CONFIG));
}
+
+ public ProcessingGuarantee getProcessingGuarantee() {
+ return processingGuarantee;
+ }
+
+ public boolean getForceEnableTupleTracking() {
+ return forceEnableTupleTracking;
+ }
public String getConsumerGroupId() {
return (String) kafkaProps.get(ConsumerConfig.GROUP_ID_CONFIG);
http://git-wip-us.apache.org/repos/asf/storm/blob/11a7a157/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutCommitTest.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutCommitTest.java b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutCommitTest.java
index 17ba378..afc9b82 100644
--- a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutCommitTest.java
+++ b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutCommitTest.java
@@ -15,7 +15,6 @@
*/
package org.apache.storm.kafka.spout;
-import static org.apache.storm.kafka.spout.builders.SingleTopicKafkaSpoutConfiguration.getKafkaSpoutConfigBuilder;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.mockito.Matchers.*;
@@ -41,6 +40,8 @@ import org.mockito.Captor;
import org.mockito.InOrder;
import org.mockito.MockitoAnnotations;
+import static org.apache.storm.kafka.spout.builders.SingleTopicKafkaSpoutConfiguration.createKafkaSpoutConfigBuilder;
+
public class KafkaSpoutCommitTest {
private final long offsetCommitPeriodMs = 2_000;
@@ -57,7 +58,7 @@ public class KafkaSpoutCommitTest {
@Before
public void setUp() {
MockitoAnnotations.initMocks(this);
- spoutConfig = getKafkaSpoutConfigBuilder(-1)
+ spoutConfig = createKafkaSpoutConfigBuilder(-1)
.setOffsetCommitPeriodMs(offsetCommitPeriodMs)
.build();
consumerMock = mock(KafkaConsumer.class);
http://git-wip-us.apache.org/repos/asf/storm/blob/11a7a157/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutEmitTest.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutEmitTest.java b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutEmitTest.java
index e8e93b0..3b1ce2d 100755
--- a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutEmitTest.java
+++ b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutEmitTest.java
@@ -15,8 +15,6 @@
*/
package org.apache.storm.kafka.spout;
-import static org.hamcrest.CoreMatchers.is;
-import static org.junit.Assert.assertThat;
import static org.mockito.Matchers.anyLong;
import static org.mockito.Matchers.anyObject;
import static org.mockito.Mockito.mock;
@@ -40,18 +38,18 @@ import org.apache.storm.task.TopologyContext;
import org.junit.Test;
import org.mockito.ArgumentCaptor;
-import static org.apache.storm.kafka.spout.builders.SingleTopicKafkaSpoutConfiguration.getKafkaSpoutConfigBuilder;
import static org.mockito.Matchers.anyList;
import static org.mockito.Matchers.anyString;
import static org.mockito.Mockito.inOrder;
import static org.mockito.Mockito.never;
-import java.util.HashSet;
import org.apache.storm.utils.Time;
import org.apache.storm.utils.Time.SimulatedTime;
import org.junit.Before;
import org.mockito.InOrder;
+import static org.apache.storm.kafka.spout.builders.SingleTopicKafkaSpoutConfiguration.createKafkaSpoutConfigBuilder;
+
public class KafkaSpoutEmitTest {
private final long offsetCommitPeriodMs = 2_000;
@@ -64,7 +62,7 @@ public class KafkaSpoutEmitTest {
@Before
public void setUp() {
- spoutConfig = getKafkaSpoutConfigBuilder(-1)
+ spoutConfig = createKafkaSpoutConfigBuilder(-1)
.setOffsetCommitPeriodMs(offsetCommitPeriodMs)
.build();
consumerMock = mock(KafkaConsumer.class);
http://git-wip-us.apache.org/repos/asf/storm/blob/11a7a157/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutMessagingGuaranteeTest.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutMessagingGuaranteeTest.java b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutMessagingGuaranteeTest.java
new file mode 100644
index 0000000..1f23cc5
--- /dev/null
+++ b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutMessagingGuaranteeTest.java
@@ -0,0 +1,207 @@
+/*
+ * Copyright 2017 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.storm.kafka.spout;
+
+import static org.apache.storm.kafka.spout.builders.SingleTopicKafkaSpoutConfiguration.createKafkaSpoutConfigBuilder;
+import static org.hamcrest.CoreMatchers.not;
+import static org.hamcrest.CoreMatchers.nullValue;
+import static org.junit.Assert.assertThat;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyList;
+import static org.mockito.Matchers.anyLong;
+import static org.mockito.Matchers.eq;
+import static org.mockito.Mockito.inOrder;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.never;
+import static org.mockito.Mockito.reset;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.Map;
+import org.apache.kafka.clients.consumer.ConsumerRecords;
+import org.apache.kafka.clients.consumer.KafkaConsumer;
+import org.apache.kafka.common.TopicPartition;
+import org.apache.storm.kafka.spout.builders.SingleTopicKafkaSpoutConfiguration;
+import org.apache.storm.spout.SpoutOutputCollector;
+import org.apache.storm.task.TopologyContext;
+import org.apache.storm.utils.Time;
+import org.apache.storm.utils.Time.SimulatedTime;
+import org.junit.Before;
+import org.junit.Test;
+import org.mockito.ArgumentCaptor;
+import org.mockito.InOrder;
+
+public class KafkaSpoutMessagingGuaranteeTest {
+
+ private final TopologyContext contextMock = mock(TopologyContext.class);
+ private final SpoutOutputCollector collectorMock = mock(SpoutOutputCollector.class);
+ private final Map<String, Object> conf = new HashMap<>();
+ private final TopicPartition partition = new TopicPartition(SingleTopicKafkaSpoutConfiguration.TOPIC, 1);
+ private KafkaConsumer<String, String> consumerMock;
+
+ @Before
+ public void setUp() {
+ consumerMock = mock(KafkaConsumer.class);
+ }
+
+ @Test
+ public void testAtMostOnceModeCommitsBeforeEmit() throws Exception {
+ //At-most-once mode must commit tuples before they are emitted to the topology to ensure that a spout crash won't cause replays.
+ KafkaSpoutConfig<String, String> spoutConfig = createKafkaSpoutConfigBuilder(-1)
+ .setProcessingGuarantee(KafkaSpoutConfig.ProcessingGuarantee.AT_MOST_ONCE)
+ .build();
+ KafkaSpout<String, String> spout = SpoutWithMockedConsumerSetupHelper.setupSpout(spoutConfig, conf, contextMock, collectorMock, consumerMock, Collections.singleton(partition));
+
+ when(consumerMock.poll(anyLong())).thenReturn(new ConsumerRecords<>(Collections.singletonMap(partition,
+ SpoutWithMockedConsumerSetupHelper.<String, String>createRecords(partition, 0, 1))));
+
+ spout.nextTuple();
+
+ //The spout should have emitted the tuple, and must have committed it before emit
+ InOrder inOrder = inOrder(consumerMock, collectorMock);
+ inOrder.verify(consumerMock).poll(anyLong());
+ inOrder.verify(consumerMock).commitSync();
+ inOrder.verify(collectorMock).emit(eq(SingleTopicKafkaSpoutConfiguration.STREAM), anyList());
+ }
+
+ private void doTestModeDisregardsMaxUncommittedOffsets(KafkaSpoutConfig<String, String> spoutConfig) {
+ KafkaSpout<String, String> spout = SpoutWithMockedConsumerSetupHelper.setupSpout(spoutConfig, conf, contextMock, collectorMock, consumerMock, Collections.singleton(partition));
+
+ when(consumerMock.poll(anyLong()))
+ .thenReturn(new ConsumerRecords<>(Collections.singletonMap(partition,
+ SpoutWithMockedConsumerSetupHelper.<String, String>createRecords(partition, 0, spoutConfig.getMaxUncommittedOffsets()))))
+ .thenReturn(new ConsumerRecords<>(Collections.singletonMap(partition,
+ SpoutWithMockedConsumerSetupHelper.<String, String>createRecords(partition, spoutConfig.getMaxUncommittedOffsets() - 1, spoutConfig.getMaxUncommittedOffsets()))));
+
+ for (int i = 0; i < spoutConfig.getMaxUncommittedOffsets() * 2; i++) {
+ spout.nextTuple();
+ }
+
+ verify(consumerMock, times(2)).poll(anyLong());
+ verify(collectorMock, times(spoutConfig.getMaxUncommittedOffsets() * 2)).emit(eq(SingleTopicKafkaSpoutConfiguration.STREAM), anyList());
+ }
+
+ @Test
+ public void testAtMostOnceModeDisregardsMaxUncommittedOffsets() throws Exception {
+ //The maxUncommittedOffsets limit should not be enforced, since it is only meaningful in at-least-once mode
+ KafkaSpoutConfig<String, String> spoutConfig = createKafkaSpoutConfigBuilder(-1)
+ .setProcessingGuarantee(KafkaSpoutConfig.ProcessingGuarantee.AT_MOST_ONCE)
+ .build();
+ doTestModeDisregardsMaxUncommittedOffsets(spoutConfig);
+ }
+
+ @Test
+ public void testAnyTimesModeDisregardsMaxUncommittedOffsets() throws Exception {
+ //The maxUncommittedOffsets limit should not be enforced, since it is only meaningful in at-least-once mode
+ KafkaSpoutConfig<String, String> spoutConfig = createKafkaSpoutConfigBuilder(-1)
+ .setProcessingGuarantee(KafkaSpoutConfig.ProcessingGuarantee.ANY_TIMES)
+ .build();
+ doTestModeDisregardsMaxUncommittedOffsets(spoutConfig);
+ }
+
+ private void doTestModeCannotReplayTuples(KafkaSpoutConfig<String, String> spoutConfig) {
+ KafkaSpout<String, String> spout = SpoutWithMockedConsumerSetupHelper.setupSpout(spoutConfig, conf, contextMock, collectorMock, consumerMock, Collections.singleton(partition));
+
+ when(consumerMock.poll(anyLong())).thenReturn(new ConsumerRecords<>(Collections.singletonMap(partition,
+ SpoutWithMockedConsumerSetupHelper.<String, String>createRecords(partition, 0, 1))));
+
+ spout.nextTuple();
+
+ ArgumentCaptor<KafkaSpoutMessageId> msgIdCaptor = ArgumentCaptor.forClass(KafkaSpoutMessageId.class);
+ verify(collectorMock).emit(eq(SingleTopicKafkaSpoutConfiguration.STREAM), anyList(), msgIdCaptor.capture());
+ assertThat("Should have captured a message id", msgIdCaptor.getValue(), not(nullValue()));
+
+ spout.fail(msgIdCaptor.getValue());
+
+ reset(consumerMock);
+
+ when(consumerMock.poll(anyLong())).thenReturn(new ConsumerRecords<>(Collections.singletonMap(partition,
+ SpoutWithMockedConsumerSetupHelper.<String, String>createRecords(partition, 1, 1))));
+
+ spout.nextTuple();
+
+ //The consumer should not be seeking to retry the failed tuple, it should just be continuing from the current position
+ verify(consumerMock, never()).seek(eq(partition), anyLong());
+ }
+
+ @Test
+ public void testAtMostOnceModeCannotReplayTuples() throws Exception {
+ //When tuple tracking is enabled, the spout must not replay tuples in at-most-once mode
+ KafkaSpoutConfig<String, String> spoutConfig = createKafkaSpoutConfigBuilder(-1)
+ .setProcessingGuarantee(KafkaSpoutConfig.ProcessingGuarantee.AT_MOST_ONCE)
+ .setForceEnableTupleTracking(true)
+ .build();
+ doTestModeCannotReplayTuples(spoutConfig);
+ }
+
+ @Test
+ public void testAnyTimesModeCannotReplayTuples() throws Exception {
+ //When tuple tracking is enabled, the spout must not replay tuples in any-times mode
+ KafkaSpoutConfig<String, String> spoutConfig = createKafkaSpoutConfigBuilder(-1)
+ .setProcessingGuarantee(KafkaSpoutConfig.ProcessingGuarantee.ANY_TIMES)
+ .setForceEnableTupleTracking(true)
+ .build();
+ doTestModeCannotReplayTuples(spoutConfig);
+ }
+
+ private void doTestModeDoesNotCommitAckedTuples(KafkaSpoutConfig<String, String> spoutConfig) {
+ try (SimulatedTime time = new SimulatedTime()) {
+ KafkaSpout<String, String> spout = SpoutWithMockedConsumerSetupHelper.setupSpout(spoutConfig, conf, contextMock, collectorMock, consumerMock, Collections.singleton(partition));
+
+ when(consumerMock.poll(anyLong())).thenReturn(new ConsumerRecords<>(Collections.singletonMap(partition,
+ SpoutWithMockedConsumerSetupHelper.<String, String>createRecords(partition, 0, 1))));
+
+ spout.nextTuple();
+
+ ArgumentCaptor<KafkaSpoutMessageId> msgIdCaptor = ArgumentCaptor.forClass(KafkaSpoutMessageId.class);
+ verify(collectorMock).emit(eq(SingleTopicKafkaSpoutConfiguration.STREAM), anyList(), msgIdCaptor.capture());
+ assertThat("Should have captured a message id", msgIdCaptor.getValue(), not(nullValue()));
+
+ spout.ack(msgIdCaptor.getValue());
+
+ Time.advanceTime(spoutConfig.getOffsetsCommitPeriodMs());
+
+ spout.nextTuple();
+
+ verify(consumerMock, never()).commitSync(any(Map.class));
+ }
+ }
+
+ @Test
+ public void testAtMostOnceModeDoesNotCommitAckedTuples() throws Exception {
+ //When tuple tracking is enabled, the spout must not commit acked tuples in at-most-once mode because they were committed before being emitted
+ KafkaSpoutConfig<String, String> spoutConfig = createKafkaSpoutConfigBuilder(-1)
+ .setProcessingGuarantee(KafkaSpoutConfig.ProcessingGuarantee.AT_MOST_ONCE)
+ .setForceEnableTupleTracking(true)
+ .build();
+ doTestModeDoesNotCommitAckedTuples(spoutConfig);
+ }
+
+ @Test
+ public void testAnyTimesModeDoesNotCommitAckedTuples() throws Exception {
+ //When tuple tracking is enabled, the spout must not commit acked tuples in any-times mode because committing is managed by the consumer
+ KafkaSpoutConfig<String, String> spoutConfig = createKafkaSpoutConfigBuilder(-1)
+ .setProcessingGuarantee(KafkaSpoutConfig.ProcessingGuarantee.ANY_TIMES)
+ .setForceEnableTupleTracking(true)
+ .build();
+ doTestModeDoesNotCommitAckedTuples(spoutConfig);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/11a7a157/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutRebalanceTest.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutRebalanceTest.java b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutRebalanceTest.java
index 8996190..ab57052 100644
--- a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutRebalanceTest.java
+++ b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutRebalanceTest.java
@@ -15,7 +15,6 @@
*/
package org.apache.storm.kafka.spout;
-import static org.apache.storm.kafka.spout.builders.SingleTopicKafkaSpoutConfiguration.getKafkaSpoutConfigBuilder;
import static org.hamcrest.CoreMatchers.not;
import static org.hamcrest.Matchers.hasKey;
import static org.junit.Assert.assertThat;
@@ -54,6 +53,9 @@ import org.mockito.Captor;
import org.mockito.Mockito;
import org.mockito.MockitoAnnotations;
+import static org.apache.storm.kafka.spout.builders.SingleTopicKafkaSpoutConfiguration.createKafkaSpoutConfigBuilder;
+import static org.apache.storm.kafka.spout.builders.SingleTopicKafkaSpoutConfiguration.createKafkaSpoutConfigBuilder;
+
public class KafkaSpoutRebalanceTest {
@Captor
@@ -131,7 +133,7 @@ public class KafkaSpoutRebalanceTest {
doNothing()
.when(subscriptionMock)
.subscribe(any(KafkaConsumer.class), rebalanceListenerCapture.capture(), any(TopologyContext.class));
- KafkaSpout<String, String> spout = new KafkaSpout<>(getKafkaSpoutConfigBuilder(subscriptionMock, -1)
+ KafkaSpout<String, String> spout = new KafkaSpout<>(createKafkaSpoutConfigBuilder(subscriptionMock, -1)
.setOffsetCommitPeriodMs(offsetCommitPeriodMs)
.build(), consumerFactoryMock);
String topic = SingleTopicKafkaSpoutConfiguration.TOPIC;
@@ -168,7 +170,7 @@ public class KafkaSpoutRebalanceTest {
.when(subscriptionMock)
.subscribe(any(KafkaConsumer.class), rebalanceListenerCapture.capture(), any(TopologyContext.class));
KafkaSpoutRetryService retryServiceMock = mock(KafkaSpoutRetryService.class);
- KafkaSpout<String, String> spout = new KafkaSpout<>(getKafkaSpoutConfigBuilder(subscriptionMock, -1)
+ KafkaSpout<String, String> spout = new KafkaSpout<>(createKafkaSpoutConfigBuilder(subscriptionMock, -1)
.setOffsetCommitPeriodMs(10)
.setRetry(retryServiceMock)
.build(), consumerFactoryMock);
http://git-wip-us.apache.org/repos/asf/storm/blob/11a7a157/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutRetryLimitTest.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutRetryLimitTest.java b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutRetryLimitTest.java
index 79f7398..39fa42c 100644
--- a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutRetryLimitTest.java
+++ b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/KafkaSpoutRetryLimitTest.java
@@ -15,7 +15,6 @@
*/
package org.apache.storm.kafka.spout;
-import static org.apache.storm.kafka.spout.builders.SingleTopicKafkaSpoutConfiguration.getKafkaSpoutConfigBuilder;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.mockito.Matchers.*;
@@ -45,6 +44,8 @@ import org.mockito.Captor;
import org.mockito.InOrder;
import org.mockito.MockitoAnnotations;
+import static org.apache.storm.kafka.spout.builders.SingleTopicKafkaSpoutConfiguration.createKafkaSpoutConfigBuilder;
+
public class KafkaSpoutRetryLimitTest {
private final long offsetCommitPeriodMs = 2_000;
@@ -65,7 +66,7 @@ public class KafkaSpoutRetryLimitTest {
@Before
public void setUp() {
MockitoAnnotations.initMocks(this);
- spoutConfig = getKafkaSpoutConfigBuilder(-1)
+ spoutConfig = createKafkaSpoutConfigBuilder(-1)
.setOffsetCommitPeriodMs(offsetCommitPeriodMs)
.setRetry(ZERO_RETRIES_RETRY_SERVICE)
.build();
http://git-wip-us.apache.org/repos/asf/storm/blob/11a7a157/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/MaxUncommittedOffsetTest.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/MaxUncommittedOffsetTest.java b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/MaxUncommittedOffsetTest.java
index ccb2a6c..80a8e1d 100755
--- a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/MaxUncommittedOffsetTest.java
+++ b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/MaxUncommittedOffsetTest.java
@@ -15,14 +15,12 @@
*/
package org.apache.storm.kafka.spout;
-import static org.apache.storm.kafka.spout.builders.SingleTopicKafkaSpoutConfiguration.getKafkaSpoutConfigBuilder;
import static org.hamcrest.CoreMatchers.everyItem;
import static org.hamcrest.CoreMatchers.is;
import static org.hamcrest.Matchers.greaterThanOrEqualTo;
import static org.hamcrest.Matchers.isIn;
import static org.hamcrest.Matchers.lessThanOrEqualTo;
import static org.junit.Assert.assertThat;
-import static org.mockito.Matchers.any;
import static org.mockito.Matchers.anyList;
import static org.mockito.Matchers.anyObject;
import static org.mockito.Matchers.anyString;
@@ -30,15 +28,12 @@ import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.reset;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.when;
import java.util.ArrayList;
-import java.util.Collections;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
import org.apache.kafka.clients.consumer.ConsumerConfig;
-import org.apache.kafka.clients.producer.ProducerRecord;
import org.apache.storm.kafka.KafkaUnitRule;
import org.apache.storm.kafka.spout.builders.SingleTopicKafkaSpoutConfiguration;
import org.apache.storm.spout.SpoutOutputCollector;
@@ -50,6 +45,8 @@ import org.junit.Test;
import org.mockito.ArgumentCaptor;
import org.mockito.MockitoAnnotations;
+import static org.apache.storm.kafka.spout.builders.SingleTopicKafkaSpoutConfiguration.createKafkaSpoutConfigBuilder;
+
public class MaxUncommittedOffsetTest {
@Rule
@@ -63,7 +60,7 @@ public class MaxUncommittedOffsetTest {
private final int maxUncommittedOffsets = 10;
private final int maxPollRecords = 5;
private final int initialRetryDelaySecs = 60;
- private final KafkaSpoutConfig<String, String> spoutConfig = getKafkaSpoutConfigBuilder(kafkaUnitRule.getKafkaUnit().getKafkaPort())
+ private final KafkaSpoutConfig<String, String> spoutConfig = createKafkaSpoutConfigBuilder(kafkaUnitRule.getKafkaUnit().getKafkaPort())
.setOffsetCommitPeriodMs(commitOffsetPeriodMs)
.setProp(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, maxPollRecords)
.setMaxUncommittedOffsets(maxUncommittedOffsets)
@@ -84,30 +81,15 @@ public class MaxUncommittedOffsetTest {
this.spout = new KafkaSpout<>(spoutConfig);
}
- private void populateTopicData(String topicName, int msgCount) throws Exception {
- kafkaUnitRule.getKafkaUnit().createTopic(topicName);
-
- for (int i = 0; i < msgCount; i++) {
- ProducerRecord<String, String> producerRecord = new ProducerRecord<>(
- topicName, Integer.toString(i),
- Integer.toString(i));
-
- kafkaUnitRule.getKafkaUnit().sendMessage(producerRecord);
- }
- }
-
- private void initializeSpout(int msgCount) throws Exception {
- populateTopicData(SingleTopicKafkaSpoutConfiguration.TOPIC, msgCount);
- when(topologyContext.getThisTaskIndex()).thenReturn(0);
- when(topologyContext.getComponentTasks(anyString())).thenReturn(Collections.singletonList(0));
- spout.open(conf, topologyContext, collector);
- spout.activate();
+ private void prepareSpout(int msgCount) throws Exception {
+ SingleTopicKafkaUnitSetupHelper.populateTopicData(kafkaUnitRule.getKafkaUnit(), SingleTopicKafkaSpoutConfiguration.TOPIC, msgCount);
+ SingleTopicKafkaUnitSetupHelper.initializeSpout(spout, conf, topologyContext, collector);
}
private ArgumentCaptor<KafkaSpoutMessageId> emitMaxUncommittedOffsetsMessagesAndCheckNoMoreAreEmitted(int messageCount) throws Exception {
assertThat("The message count is less than maxUncommittedOffsets. This test is not meaningful with this configuration.", messageCount, greaterThanOrEqualTo(maxUncommittedOffsets));
//The spout must respect maxUncommittedOffsets when requesting/emitting tuples
- initializeSpout(messageCount);
+ prepareSpout(messageCount);
//Try to emit all messages. Ensure only maxUncommittedOffsets are emitted
ArgumentCaptor<KafkaSpoutMessageId> messageIds = ArgumentCaptor.forClass(KafkaSpoutMessageId.class);
http://git-wip-us.apache.org/repos/asf/storm/blob/11a7a157/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/SingleTopicKafkaSpoutTest.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/SingleTopicKafkaSpoutTest.java b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/SingleTopicKafkaSpoutTest.java
index 7759b3c..cbbb391 100644
--- a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/SingleTopicKafkaSpoutTest.java
+++ b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/SingleTopicKafkaSpoutTest.java
@@ -17,10 +17,8 @@
*/
package org.apache.storm.kafka.spout;
-import static org.apache.storm.kafka.spout.builders.SingleTopicKafkaSpoutConfiguration.getKafkaSpoutConfigBuilder;
import org.apache.kafka.clients.consumer.OffsetAndMetadata;
-import org.apache.kafka.clients.producer.ProducerRecord;
import org.apache.storm.kafka.KafkaUnitRule;
import org.apache.storm.kafka.spout.builders.SingleTopicKafkaSpoutConfiguration;
import org.apache.storm.spout.SpoutOutputCollector;
@@ -31,12 +29,9 @@ import org.junit.Test;
import org.mockito.ArgumentCaptor;
import java.util.Map;
-import java.util.concurrent.ExecutionException;
-import java.util.concurrent.TimeoutException;
import static org.hamcrest.CoreMatchers.is;
import static org.hamcrest.MatcherAssert.assertThat;
-import static org.mockito.Matchers.any;
import static org.mockito.Matchers.anyList;
import static org.mockito.Matchers.anyListOf;
import static org.mockito.Matchers.anyObject;
@@ -48,9 +43,7 @@ import static org.mockito.Mockito.reset;
import static org.mockito.Mockito.spy;
import static org.mockito.Mockito.times;
import static org.mockito.Mockito.verify;
-import static org.mockito.Mockito.when;
-import java.util.Collections;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
@@ -65,6 +58,8 @@ import org.junit.Before;
import org.mockito.Captor;
import org.mockito.MockitoAnnotations;
+import static org.apache.storm.kafka.spout.builders.SingleTopicKafkaSpoutConfiguration.createKafkaSpoutConfigBuilder;
+
public class SingleTopicKafkaSpoutTest {
@Rule
@@ -77,15 +72,15 @@ public class SingleTopicKafkaSpoutTest {
private final Map<String, Object> conf = new HashMap<>();
private final SpoutOutputCollector collector = mock(SpoutOutputCollector.class);
private final long commitOffsetPeriodMs = 2_000;
+ private final int maxRetries = 3;
private KafkaConsumer<String, String> consumerSpy;
private KafkaConsumerFactory<String, String> consumerFactory;
private KafkaSpout<String, String> spout;
- private int maxRetries = 3;
@Before
public void setUp() {
MockitoAnnotations.initMocks(this);
- KafkaSpoutConfig<String, String> spoutConfig = getKafkaSpoutConfigBuilder(kafkaUnitRule.getKafkaUnit().getKafkaPort())
+ KafkaSpoutConfig<String, String> spoutConfig = createKafkaSpoutConfigBuilder(kafkaUnitRule.getKafkaUnit().getKafkaPort())
.setOffsetCommitPeriodMs(commitOffsetPeriodMs)
.setRetry(new KafkaSpoutRetryExponentialBackoff(KafkaSpoutRetryExponentialBackoff.TimeInterval.seconds(0), KafkaSpoutRetryExponentialBackoff.TimeInterval.seconds(0),
maxRetries, KafkaSpoutRetryExponentialBackoff.TimeInterval.seconds(0)))
@@ -101,25 +96,11 @@ public class SingleTopicKafkaSpoutTest {
this.spout = new KafkaSpout<>(spoutConfig, consumerFactory);
}
- void populateTopicData(String topicName, int msgCount) throws InterruptedException, ExecutionException, TimeoutException {
- kafkaUnitRule.getKafkaUnit().createTopic(topicName);
-
- for (int i = 0; i < msgCount; i++) {
- ProducerRecord<String, String> producerRecord = new ProducerRecord<>(
- topicName, Integer.toString(i),
- Integer.toString(i));
- kafkaUnitRule.getKafkaUnit().sendMessage(producerRecord);
- }
+ private void prepareSpout(int messageCount) throws Exception {
+ SingleTopicKafkaUnitSetupHelper.populateTopicData(kafkaUnitRule.getKafkaUnit(), SingleTopicKafkaSpoutConfiguration.TOPIC, messageCount);
+ SingleTopicKafkaUnitSetupHelper.initializeSpout(spout, conf, topologyContext, collector);
}
-
- private void initializeSpout(int msgCount) throws InterruptedException, ExecutionException, TimeoutException {
- populateTopicData(SingleTopicKafkaSpoutConfiguration.TOPIC, msgCount);
- when(topologyContext.getThisTaskIndex()).thenReturn(0);
- when(topologyContext.getComponentTasks(anyString())).thenReturn(Collections.singletonList(0));
- spout.open(conf, topologyContext, collector);
- spout.activate();
- }
-
+
/*
* Asserts that commitSync has been called once,
* that there are only commits on one topic,
@@ -137,7 +118,7 @@ public class SingleTopicKafkaSpoutTest {
public void shouldContinueWithSlowDoubleAcks() throws Exception {
try (SimulatedTime simulatedTime = new SimulatedTime()) {
int messageCount = 20;
- initializeSpout(messageCount);
+ prepareSpout(messageCount);
//play 1st tuple
ArgumentCaptor<Object> messageIdToDoubleAck = ArgumentCaptor.forClass(Object.class);
@@ -178,7 +159,7 @@ public class SingleTopicKafkaSpoutTest {
public void shouldEmitAllMessages() throws Exception {
try (SimulatedTime simulatedTime = new SimulatedTime()) {
int messageCount = 10;
- initializeSpout(messageCount);
+ prepareSpout(messageCount);
//Emit all messages and check that they are emitted. Ack the messages too
for(int i = 0; i < messageCount; i++) {
@@ -206,7 +187,7 @@ public class SingleTopicKafkaSpoutTest {
public void shouldReplayInOrderFailedMessages() throws Exception {
try (SimulatedTime simulatedTime = new SimulatedTime()) {
int messageCount = 10;
- initializeSpout(messageCount);
+ prepareSpout(messageCount);
//play and ack 1 tuple
ArgumentCaptor<Object> messageIdAcked = ArgumentCaptor.forClass(Object.class);
@@ -249,7 +230,7 @@ public class SingleTopicKafkaSpoutTest {
public void shouldReplayFirstTupleFailedOutOfOrder() throws Exception {
try (SimulatedTime simulatedTime = new SimulatedTime()) {
int messageCount = 10;
- initializeSpout(messageCount);
+ prepareSpout(messageCount);
//play 1st tuple
ArgumentCaptor<Object> messageIdToFail = ArgumentCaptor.forClass(Object.class);
@@ -296,7 +277,7 @@ public class SingleTopicKafkaSpoutTest {
//The spout must reemit retriable tuples, even if they fail out of order.
//The spout should be able to skip tuples it has already emitted when retrying messages, even if those tuples are also retries.
int messageCount = 10;
- initializeSpout(messageCount);
+ prepareSpout(messageCount);
//play all tuples
for (int i = 0; i < messageCount; i++) {
@@ -329,7 +310,7 @@ public class SingleTopicKafkaSpoutTest {
public void shouldDropMessagesAfterMaxRetriesAreReached() throws Exception {
//Check that if one message fails repeatedly, the retry cap limits how many times the message can be reemitted
int messageCount = 1;
- initializeSpout(messageCount);
+ prepareSpout(messageCount);
//Emit and fail the same tuple until we've reached retry limit
for (int i = 0; i <= maxRetries; i++) {
http://git-wip-us.apache.org/repos/asf/storm/blob/11a7a157/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/SingleTopicKafkaUnitSetupHelper.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/SingleTopicKafkaUnitSetupHelper.java b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/SingleTopicKafkaUnitSetupHelper.java
new file mode 100644
index 0000000..81fe362
--- /dev/null
+++ b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/SingleTopicKafkaUnitSetupHelper.java
@@ -0,0 +1,67 @@
+/*
+ * Copyright 2017 The Apache Software Foundation.
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.storm.kafka.spout;
+
+import static org.mockito.Matchers.anyString;
+import static org.mockito.Mockito.when;
+
+import java.util.Collections;
+import java.util.Map;
+import org.apache.kafka.clients.producer.ProducerRecord;
+import org.apache.storm.kafka.KafkaUnit;
+import org.apache.storm.spout.SpoutOutputCollector;
+import org.apache.storm.task.TopologyContext;
+
+public class SingleTopicKafkaUnitSetupHelper {
+
+ /**
+ * Using the given KafkaUnit instance, put some messages in the specified topic.
+ *
+ * @param kafkaUnit The KafkaUnit instance to use
+ * @param topicName The topic to produce messages for
+ * @param msgCount The number of messages to produce
+ */
+ public static void populateTopicData(KafkaUnit kafkaUnit, String topicName, int msgCount) throws Exception {
+ kafkaUnit.createTopic(topicName);
+
+ for (int i = 0; i < msgCount; i++) {
+ ProducerRecord<String, String> producerRecord = new ProducerRecord<>(
+ topicName, Integer.toString(i),
+ Integer.toString(i));
+ kafkaUnit.sendMessage(producerRecord);
+ }
+ }
+
+ /**
+ * Open and activate a KafkaSpout that acts as a single-task/executor spout.
+ *
+ * @param <K> Kafka key type
+ * @param <V> Kafka value type
+ * @param spout The spout to prepare
+ * @param topoConf The topoConf
+ * @param topoContextMock The TopologyContext mock
+ * @param collectorMock The output collector mock
+ */
+ public static <K, V> void initializeSpout(KafkaSpout<K, V> spout, Map<String, Object> topoConf, TopologyContext topoContextMock,
+ SpoutOutputCollector collectorMock) throws Exception {
+ when(topoContextMock.getThisTaskIndex()).thenReturn(0);
+ when(topoContextMock.getComponentTasks(anyString())).thenReturn(Collections.singletonList(0));
+ spout.open(topoConf, topoContextMock, collectorMock);
+ spout.activate();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/storm/blob/11a7a157/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/builders/SingleTopicKafkaSpoutConfiguration.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/builders/SingleTopicKafkaSpoutConfiguration.java b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/builders/SingleTopicKafkaSpoutConfiguration.java
index 1ab4966..f4cb833 100644
--- a/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/builders/SingleTopicKafkaSpoutConfiguration.java
+++ b/external/storm-kafka-client/src/test/java/org/apache/storm/kafka/spout/builders/SingleTopicKafkaSpoutConfiguration.java
@@ -41,13 +41,6 @@ public class SingleTopicKafkaSpoutConfiguration {
public static final String STREAM = "test_stream";
public static final String TOPIC = "test";
- /**
- * Retry in a tight loop (keep unit tests fasts).
- */
- public static final KafkaSpoutRetryService UNIT_TEST_RETRY_SERVICE =
- new KafkaSpoutRetryExponentialBackoff(KafkaSpoutRetryExponentialBackoff.TimeInterval.seconds(0), KafkaSpoutRetryExponentialBackoff.TimeInterval.milliSeconds(0),
- DEFAULT_MAX_RETRIES, KafkaSpoutRetryExponentialBackoff.TimeInterval.milliSeconds(0));
-
public static Config getConfig() {
Config config = new Config();
config.setDebug(true);
@@ -56,7 +49,7 @@ public class SingleTopicKafkaSpoutConfiguration {
public static StormTopology getTopologyKafkaSpout(int port) {
final TopologyBuilder tp = new TopologyBuilder();
- tp.setSpout("kafka_spout", new KafkaSpout<>(getKafkaSpoutConfigBuilder(port).build()), 1);
+ tp.setSpout("kafka_spout", new KafkaSpout<>(SingleTopicKafkaSpoutConfiguration.createKafkaSpoutConfigBuilder(port).build()), 1);
tp.setBolt("kafka_bolt", new KafkaSpoutTestBolt()).shuffleGrouping("kafka_spout", STREAM);
return tp.createTopology();
}
@@ -68,11 +61,11 @@ public class SingleTopicKafkaSpoutConfiguration {
}
};
- public static KafkaSpoutConfig.Builder<String, String> getKafkaSpoutConfigBuilder(int port) {
+ public static KafkaSpoutConfig.Builder<String, String> createKafkaSpoutConfigBuilder(int port) {
return setCommonSpoutConfig(KafkaSpoutConfig.builder("127.0.0.1:" + port, TOPIC));
}
- public static KafkaSpoutConfig.Builder<String, String> getKafkaSpoutConfigBuilder(Subscription subscription, int port) {
+ public static KafkaSpoutConfig.Builder<String, String> createKafkaSpoutConfigBuilder(Subscription subscription, int port) {
return setCommonSpoutConfig(new KafkaSpoutConfig.Builder<String, String>("127.0.0.1:" + port, subscription));
}
@@ -82,14 +75,18 @@ public class SingleTopicKafkaSpoutConfiguration {
new Fields("topic", "key", "value"), STREAM)
.setProp(ConsumerConfig.GROUP_ID_CONFIG, "kafkaSpoutTestGroup")
.setProp(ConsumerConfig.MAX_POLL_RECORDS_CONFIG, 5)
- .setRetry(getRetryService())
+ .setRetry(getNoDelayRetryService())
.setOffsetCommitPeriodMs(10_000)
.setFirstPollOffsetStrategy(EARLIEST)
.setMaxUncommittedOffsets(250)
.setPollTimeoutMs(1000);
}
- protected static KafkaSpoutRetryService getRetryService() {
- return UNIT_TEST_RETRY_SERVICE;
+ protected static KafkaSpoutRetryService getNoDelayRetryService() {
+ /**
+ * Retry in a tight loop (keep unit tests fasts).
+ */
+ return new KafkaSpoutRetryExponentialBackoff(KafkaSpoutRetryExponentialBackoff.TimeInterval.seconds(0), KafkaSpoutRetryExponentialBackoff.TimeInterval.milliSeconds(0),
+ DEFAULT_MAX_RETRIES, KafkaSpoutRetryExponentialBackoff.TimeInterval.milliSeconds(0));
}
}
[27/50] [abbrv] storm git commit: Merge branch
'1.x-branch_STORM-2787_KSInitFlag' of https://github.com/hmcl/storm-apache
into STORM-2787-on-top-of-STORM-2781-merge-1.x
Posted by pt...@apache.org.
Merge branch '1.x-branch_STORM-2787_KSInitFlag' of https://github.com/hmcl/storm-apache into STORM-2787-on-top-of-STORM-2781-merge-1.x
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/5d0889b8
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/5d0889b8
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/5d0889b8
Branch: refs/heads/metrics_v2
Commit: 5d0889b848991613f1c2618afa6330ecbb76a3c9
Parents: 96864e8 7d4ac07
Author: Jungtaek Lim <ka...@gmail.com>
Authored: Mon Oct 30 09:47:41 2017 +0900
Committer: Jungtaek Lim <ka...@gmail.com>
Committed: Mon Oct 30 09:47:41 2017 +0900
----------------------------------------------------------------------
docs/storm-kafka-client.md | 37 +++++--
.../apache/storm/kafka/spout/KafkaSpout.java | 100 +++++++++++--------
.../storm/kafka/spout/KafkaSpoutConfig.java | 55 ++++++----
.../spout/KafkaSpoutMessagingGuaranteeTest.java | 14 +--
4 files changed, 125 insertions(+), 81 deletions(-)
----------------------------------------------------------------------
[19/50] [abbrv] storm git commit: Merge branch 'STORM-2706-1.x' of
https://github.com/srdo/storm into STORM-2706
Posted by pt...@apache.org.
Merge branch 'STORM-2706-1.x' of https://github.com/srdo/storm into STORM-2706
STORM-2706: Upgrade to Curator 4.0.0 (1.x)
This closes #2384
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/38e308e3
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/38e308e3
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/38e308e3
Branch: refs/heads/metrics_v2
Commit: 38e308e3ae79282c6efc7826fda7e3d37455820f
Parents: 9aff9f9 ef6fa09
Author: Robert Evans <ev...@yahoo-inc.com>
Authored: Wed Oct 25 13:21:12 2017 -0500
Committer: Robert Evans <ev...@yahoo-inc.com>
Committed: Wed Oct 25 13:21:12 2017 -0500
----------------------------------------------------------------------
external/storm-eventhubs/pom.xml | 11 -------
external/storm-kafka-monitor/pom.xml | 6 ----
external/storm-kafka/pom.xml | 29 -------------------
external/storm-kinesis/pom.xml | 11 -------
pom.xml | 22 ++++----------
storm-core/pom.xml | 30 --------------------
.../clj/org/apache/storm/transactional_test.clj | 4 +--
.../clj/org/apache/storm/trident/state_test.clj | 4 +--
.../storm/blobstore/BlobSynchronizerTest.java | 23 +++++++--------
9 files changed, 21 insertions(+), 119 deletions(-)
----------------------------------------------------------------------
[17/50] [abbrv] storm git commit: STORM-2706: Upgrade to Curator 4.0.0
Posted by pt...@apache.org.
STORM-2706: Upgrade to Curator 4.0.0
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/ef6fa09c
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/ef6fa09c
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/ef6fa09c
Branch: refs/heads/metrics_v2
Commit: ef6fa09c948d5a22bc27318794214ea5b89ed70d
Parents: 9aff9f9
Author: Stig Rohde Døssing <sr...@apache.org>
Authored: Tue Oct 17 23:10:52 2017 +0200
Committer: Stig Rohde Døssing <sr...@apache.org>
Committed: Mon Oct 23 20:27:29 2017 +0200
----------------------------------------------------------------------
external/storm-eventhubs/pom.xml | 11 -------
external/storm-kafka-monitor/pom.xml | 6 ----
external/storm-kafka/pom.xml | 29 -------------------
external/storm-kinesis/pom.xml | 11 -------
pom.xml | 22 ++++----------
storm-core/pom.xml | 30 --------------------
.../clj/org/apache/storm/transactional_test.clj | 4 +--
.../clj/org/apache/storm/trident/state_test.clj | 4 +--
.../storm/blobstore/BlobSynchronizerTest.java | 23 +++++++--------
9 files changed, 21 insertions(+), 119 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/ef6fa09c/external/storm-eventhubs/pom.xml
----------------------------------------------------------------------
diff --git a/external/storm-eventhubs/pom.xml b/external/storm-eventhubs/pom.xml
index 0809b6e..e13f305 100755
--- a/external/storm-eventhubs/pom.xml
+++ b/external/storm-eventhubs/pom.xml
@@ -67,17 +67,6 @@
<dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-framework</artifactId>
- <version>${curator.version}</version>
- <exclusions>
- <exclusion>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- </exclusions>
</dependency>
<dependency>
<groupId>org.apache.qpid</groupId>
http://git-wip-us.apache.org/repos/asf/storm/blob/ef6fa09c/external/storm-kafka-monitor/pom.xml
----------------------------------------------------------------------
diff --git a/external/storm-kafka-monitor/pom.xml b/external/storm-kafka-monitor/pom.xml
index c66e701..76e7c87 100644
--- a/external/storm-kafka-monitor/pom.xml
+++ b/external/storm-kafka-monitor/pom.xml
@@ -57,12 +57,6 @@
<dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-framework</artifactId>
- <exclusions>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- </exclusions>
</dependency>
<dependency>
<groupId>com.googlecode.json-simple</groupId>
http://git-wip-us.apache.org/repos/asf/storm/blob/ef6fa09c/external/storm-kafka/pom.xml
----------------------------------------------------------------------
diff --git a/external/storm-kafka/pom.xml b/external/storm-kafka/pom.xml
index f9cd57e..3fa532c 100644
--- a/external/storm-kafka/pom.xml
+++ b/external/storm-kafka/pom.xml
@@ -77,44 +77,15 @@
<dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-framework</artifactId>
- <version>${curator.version}</version>
- <exclusions>
- <exclusion>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- </exclusions>
</dependency>
<dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-recipes</artifactId>
- <version>${curator.version}</version>
- <exclusions>
- <exclusion>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- </exclusion>
- </exclusions>
<scope>test</scope>
</dependency>
<dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-test</artifactId>
- <version>${curator.version}</version>
- <exclusions>
- <exclusion>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.testng</groupId>
- <artifactId>testng</artifactId>
- </exclusion>
- </exclusions>
<scope>test</scope>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/storm/blob/ef6fa09c/external/storm-kinesis/pom.xml
----------------------------------------------------------------------
diff --git a/external/storm-kinesis/pom.xml b/external/storm-kinesis/pom.xml
index f97a3a3..88c67d2 100644
--- a/external/storm-kinesis/pom.xml
+++ b/external/storm-kinesis/pom.xml
@@ -45,17 +45,6 @@
<dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-framework</artifactId>
- <version>${curator.version}</version>
- <exclusions>
- <exclusion>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- </exclusions>
</dependency>
<dependency>
<groupId>com.googlecode.json-simple</groupId>
http://git-wip-us.apache.org/repos/asf/storm/blob/ef6fa09c/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index 21ec2bc..3b0d39b 100644
--- a/pom.xml
+++ b/pom.xml
@@ -229,7 +229,8 @@
<commons-codec.version>1.6</commons-codec.version>
<commons-cli.version>1.3.1</commons-cli.version>
<clj-time.version>0.8.0</clj-time.version>
- <curator.version>2.12.0</curator.version>
+ <curator.version>4.0.0</curator.version>
+ <curator-test.version>2.12.0</curator-test.version>
<json-simple.version>1.1</json-simple.version>
<ring.version>1.3.1</ring.version>
<ring-json.version>0.3.1</ring-json.version>
@@ -702,10 +703,6 @@
<version>${curator.version}</version>
<exclusions>
<exclusion>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- </exclusion>
- <exclusion>
<groupId>org.jboss.netty</groupId>
<artifactId>netty</artifactId>
</exclusion>
@@ -715,16 +712,6 @@
<groupId>org.apache.curator</groupId>
<artifactId>curator-recipes</artifactId>
<version>${curator.version}</version>
- <exclusions>
- <exclusion>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- </exclusions>
</dependency>
<dependency>
<groupId>org.apache.curator</groupId>
@@ -734,7 +721,10 @@
<dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-test</artifactId>
- <version>${curator.version}</version>
+ <!-- curator-test is not compatible with Zookeeper 3.4.x.
+ Curator works around this by using an older curator-test jar.
+ See https://github.com/apache/curator/tree/6ba4de36d4e8b2b65d45c005a6a92dd85c3c497f/curator-test-zk34-->
+ <version>${curator-test.version}</version>
<scope>test</scope>
</dependency>
<dependency>
http://git-wip-us.apache.org/repos/asf/storm/blob/ef6fa09c/storm-core/pom.xml
----------------------------------------------------------------------
diff --git a/storm-core/pom.xml b/storm-core/pom.xml
index 0497bdc..3d714c0 100644
--- a/storm-core/pom.xml
+++ b/storm-core/pom.xml
@@ -188,44 +188,14 @@
<groupId>org.apache.curator</groupId>
<artifactId>curator-framework</artifactId>
<scope>compile</scope>
- <exclusions>
- <exclusion>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- </exclusions>
</dependency>
<dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-recipes</artifactId>
- <exclusions>
- <exclusion>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- </exclusions>
</dependency>
<dependency>
<groupId>org.apache.curator</groupId>
<artifactId>curator-test</artifactId>
- <exclusions>
- <exclusion>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- </exclusions>
</dependency>
<dependency>
<groupId>com.googlecode.json-simple</groupId>
http://git-wip-us.apache.org/repos/asf/storm/blob/ef6fa09c/storm-core/test/clj/org/apache/storm/transactional_test.clj
----------------------------------------------------------------------
diff --git a/storm-core/test/clj/org/apache/storm/transactional_test.clj b/storm-core/test/clj/org/apache/storm/transactional_test.clj
index 2920ea5..68d009a 100644
--- a/storm-core/test/clj/org/apache/storm/transactional_test.clj
+++ b/storm-core/test/clj/org/apache/storm/transactional_test.clj
@@ -31,7 +31,7 @@
IdentityBolt CountingCommitBolt OpaqueMemoryTransactionalSpout])
(:import [org.apache.storm.utils ZookeeperAuthInfo])
(:import [org.apache.curator.framework CuratorFramework])
- (:import [org.apache.curator.framework.api CreateBuilder ProtectACLCreateModePathAndBytesable])
+ (:import [org.apache.curator.framework.api CreateBuilder ProtectACLCreateModeStatPathAndBytesable])
(:import [org.apache.zookeeper CreateMode ZooDefs ZooDefs$Ids])
(:import [org.mockito Matchers Mockito])
(:import [org.mockito.exceptions.base MockitoAssertionError])
@@ -722,7 +722,7 @@
(testing "Creates ZooKeeper nodes with the correct ACLs"
(let [curator (Mockito/mock CuratorFramework)
builder0 (Mockito/mock CreateBuilder)
- builder1 (Mockito/mock ProtectACLCreateModePathAndBytesable)
+ builder1 (Mockito/mock ProtectACLCreateModeStatPathAndBytesable)
expectedAcls ZooDefs$Ids/CREATOR_ALL_ACL]
(. (Mockito/when (.create curator)) (thenReturn builder0))
(. (Mockito/when (.creatingParentsIfNeeded builder0)) (thenReturn builder1))
http://git-wip-us.apache.org/repos/asf/storm/blob/ef6fa09c/storm-core/test/clj/org/apache/storm/trident/state_test.clj
----------------------------------------------------------------------
diff --git a/storm-core/test/clj/org/apache/storm/trident/state_test.clj b/storm-core/test/clj/org/apache/storm/trident/state_test.clj
index 58a0cb8..2aba02c 100644
--- a/storm-core/test/clj/org/apache/storm/trident/state_test.clj
+++ b/storm-core/test/clj/org/apache/storm/trident/state_test.clj
@@ -24,7 +24,7 @@
(:import [org.apache.storm.trident.testing MemoryBackingMap MemoryMapState])
(:import [org.apache.storm.utils ZookeeperAuthInfo])
(:import [org.apache.curator.framework CuratorFramework])
- (:import [org.apache.curator.framework.api CreateBuilder ProtectACLCreateModePathAndBytesable])
+ (:import [org.apache.curator.framework.api CreateBuilder ProtectACLCreateModeStatPathAndBytesable])
(:import [org.apache.zookeeper CreateMode ZooDefs ZooDefs$Ids])
(:import [org.mockito Matchers Mockito])
(:import [org.mockito.exceptions.base MockitoAssertionError])
@@ -114,7 +114,7 @@
(testing "Creates ZooKeeper nodes with the correct ACLs"
(let [curator (Mockito/mock CuratorFramework)
builder0 (Mockito/mock CreateBuilder)
- builder1 (Mockito/mock ProtectACLCreateModePathAndBytesable)
+ builder1 (Mockito/mock ProtectACLCreateModeStatPathAndBytesable)
expectedAcls ZooDefs$Ids/CREATOR_ALL_ACL]
(. (Mockito/when (.create curator)) (thenReturn builder0))
(. (Mockito/when (.creatingParentsIfNeeded builder0)) (thenReturn builder1))
http://git-wip-us.apache.org/repos/asf/storm/blob/ef6fa09c/storm-core/test/jvm/org/apache/storm/blobstore/BlobSynchronizerTest.java
----------------------------------------------------------------------
diff --git a/storm-core/test/jvm/org/apache/storm/blobstore/BlobSynchronizerTest.java b/storm-core/test/jvm/org/apache/storm/blobstore/BlobSynchronizerTest.java
index 40201a8..5392b73 100644
--- a/storm-core/test/jvm/org/apache/storm/blobstore/BlobSynchronizerTest.java
+++ b/storm-core/test/jvm/org/apache/storm/blobstore/BlobSynchronizerTest.java
@@ -111,18 +111,17 @@ public class BlobSynchronizerTest {
@Test
public void testNimbodesWithLatestVersionOfBlob() throws Exception {
- TestingServer server = new TestingServer();
- CuratorFramework zkClient = CuratorFrameworkFactory.newClient(server.getConnectString(), new ExponentialBackoffRetry(1000, 3));
- zkClient.start();
- // Creating nimbus hosts containing latest version of blob
- zkClient.create().creatingParentContainersIfNeeded().forPath("/blobstore/key1/nimbus1:7800-1");
- zkClient.create().creatingParentContainersIfNeeded().forPath("/blobstore/key1/nimbus2:7800-2");
- Set<NimbusInfo> set = BlobStoreUtils.getNimbodesWithLatestSequenceNumberOfBlob(zkClient, "key1");
- assertEquals("Failed to get the correct nimbus hosts with latest blob version", (set.iterator().next()).getHost(),"nimbus2");
- zkClient.delete().deletingChildrenIfNeeded().forPath("/blobstore/key1/nimbus1:7800-1");
- zkClient.delete().deletingChildrenIfNeeded().forPath("/blobstore/key1/nimbus2:7800-2");
- zkClient.close();
- server.close();
+ try (TestingServer server = new TestingServer();
+ CuratorFramework zkClient = CuratorFrameworkFactory.newClient(server.getConnectString(), new ExponentialBackoffRetry(1000, 3));) {
+ zkClient.start();
+ // Creating nimbus hosts containing latest version of blob
+ zkClient.create().creatingParentContainersIfNeeded().forPath("/blobstore/key1/nimbus1:7800-1");
+ zkClient.create().creatingParentContainersIfNeeded().forPath("/blobstore/key1/nimbus2:7800-2");
+ Set<NimbusInfo> set = BlobStoreUtils.getNimbodesWithLatestSequenceNumberOfBlob(zkClient, "key1");
+ assertEquals("Failed to get the correct nimbus hosts with latest blob version", (set.iterator().next()).getHost(), "nimbus2");
+ zkClient.delete().deletingChildrenIfNeeded().forPath("/blobstore/key1/nimbus1:7800-1");
+ zkClient.delete().deletingChildrenIfNeeded().forPath("/blobstore/key1/nimbus2:7800-2");
+ }
}
@Test
[26/50] [abbrv] storm git commit: STORM-2787: storm-kafka-client
KafkaSpout method onPartitionsRevoked(...) should set initialized flag
independently of processing guarantees
Posted by pt...@apache.org.
STORM-2787: storm-kafka-client KafkaSpout method onPartitionsRevoked(...) should set initialized flag independently of processing guarantees
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/7d4ac076
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/7d4ac076
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/7d4ac076
Branch: refs/heads/metrics_v2
Commit: 7d4ac07684a7405d6539a3bd0cb7da985736bac7
Parents: ac16fe1
Author: Hugo Louro <hm...@gmail.com>
Authored: Tue Oct 24 23:52:54 2017 -0700
Committer: Hugo Louro <hm...@gmail.com>
Committed: Sun Oct 29 16:15:52 2017 -0700
----------------------------------------------------------------------
.../main/java/org/apache/storm/kafka/spout/KafkaSpout.java | 8 +++++---
1 file changed, 5 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/7d4ac076/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpout.java
----------------------------------------------------------------------
diff --git a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpout.java b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpout.java
index 0a4d788..7decd0f 100644
--- a/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpout.java
+++ b/external/storm-kafka-client/src/main/java/org/apache/storm/kafka/spout/KafkaSpout.java
@@ -151,11 +151,13 @@ public class KafkaSpout<K, V> extends BaseRichSpout {
@Override
public void onPartitionsRevoked(Collection<TopicPartition> partitions) {
+ initialized = false;
+ previousAssignment = partitions;
+
LOG.info("Partitions revoked. [consumer-group={}, consumer={}, topic-partitions={}]",
kafkaSpoutConfig.getConsumerGroupId(), kafkaConsumer, partitions);
- previousAssignment = partitions;
- if (isAtLeastOnceProcessing() && initialized) {
- initialized = false;
+
+ if (isAtLeastOnceProcessing()) {
commitOffsetsForAckedTuples();
}
}
[33/50] [abbrv] storm git commit: Merge branch 'STORM-2549-1.x' of
https://github.com/srdo/storm into asfgit-1.x-branch
Posted by pt...@apache.org.
Merge branch 'STORM-2549-1.x' of https://github.com/srdo/storm into asfgit-1.x-branch
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/ef73328d
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/ef73328d
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/ef73328d
Branch: refs/heads/metrics_v2
Commit: ef73328df9f3f70cdf0d492dca3cfb9fea6b342e
Parents: c70d7d4 07ecff1
Author: Stig Rohde Døssing <sr...@apache.org>
Authored: Sat Nov 11 18:53:41 2017 +0100
Committer: Stig Rohde Døssing <sr...@apache.org>
Committed: Sat Nov 11 18:53:41 2017 +0100
----------------------------------------------------------------------
.../apache/storm/kafka/spout/KafkaSpout.java | 111 +++++++++------
.../storm/kafka/spout/KafkaSpoutConfig.java | 11 +-
.../kafka/spout/internal/OffsetManager.java | 22 +++
.../storm/kafka/spout/KafkaSpoutCommitTest.java | 12 +-
.../storm/kafka/spout/KafkaSpoutEmitTest.java | 138 +++++++------------
.../kafka/spout/KafkaSpoutRebalanceTest.java | 15 +-
.../kafka/spout/KafkaSpoutRetryLimitTest.java | 11 +-
.../kafka/spout/MaxUncommittedOffsetTest.java | 32 +++--
.../kafka/spout/SingleTopicKafkaSpoutTest.java | 52 ++++---
.../SpoutWithMockedConsumerSetupHelper.java | 52 ++++---
.../SingleTopicKafkaSpoutConfiguration.java | 2 +-
.../kafka/spout/internal/OffsetManagerTest.java | 15 ++
12 files changed, 266 insertions(+), 207 deletions(-)
----------------------------------------------------------------------
[37/50] [abbrv] storm git commit: STORM-2811: Fix integration test
killing the same topology multiple times
Posted by pt...@apache.org.
STORM-2811: Fix integration test killing the same topology multiple times
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/bbd582b8
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/bbd582b8
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/bbd582b8
Branch: refs/heads/metrics_v2
Commit: bbd582b80716bb8d4782a3f3b9f93123fd9af9ff
Parents: 7e17712
Author: Stig Rohde Døssing <sr...@apache.org>
Authored: Sun Nov 12 10:11:49 2017 +0100
Committer: Stig Rohde Døssing <sr...@apache.org>
Committed: Mon Nov 13 19:48:47 2017 +0100
----------------------------------------------------------------------
integration-test/src/test/java/org/apache/storm/st/DemoTest.java | 1 +
.../java/org/apache/storm/st/tests/window/SlidingWindowTest.java | 1 +
.../java/org/apache/storm/st/tests/window/TumblingWindowTest.java | 3 ++-
.../src/test/java/org/apache/storm/st/wrapper/StormCluster.java | 2 +-
4 files changed, 5 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/bbd582b8/integration-test/src/test/java/org/apache/storm/st/DemoTest.java
----------------------------------------------------------------------
diff --git a/integration-test/src/test/java/org/apache/storm/st/DemoTest.java b/integration-test/src/test/java/org/apache/storm/st/DemoTest.java
index 2c62d3f..133014b 100644
--- a/integration-test/src/test/java/org/apache/storm/st/DemoTest.java
+++ b/integration-test/src/test/java/org/apache/storm/st/DemoTest.java
@@ -80,6 +80,7 @@ public final class DemoTest extends AbstractTest {
public void cleanup() throws Exception {
if (topo != null) {
topo.killOrThrow();
+ topo = null;
}
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/bbd582b8/integration-test/src/test/java/org/apache/storm/st/tests/window/SlidingWindowTest.java
----------------------------------------------------------------------
diff --git a/integration-test/src/test/java/org/apache/storm/st/tests/window/SlidingWindowTest.java b/integration-test/src/test/java/org/apache/storm/st/tests/window/SlidingWindowTest.java
index da65ca5..ebb844f 100644
--- a/integration-test/src/test/java/org/apache/storm/st/tests/window/SlidingWindowTest.java
+++ b/integration-test/src/test/java/org/apache/storm/st/tests/window/SlidingWindowTest.java
@@ -188,6 +188,7 @@ public final class SlidingWindowTest extends AbstractTest {
public void cleanup() throws Exception {
if (topo != null) {
topo.killOrThrow();
+ topo = null;
}
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/bbd582b8/integration-test/src/test/java/org/apache/storm/st/tests/window/TumblingWindowTest.java
----------------------------------------------------------------------
diff --git a/integration-test/src/test/java/org/apache/storm/st/tests/window/TumblingWindowTest.java b/integration-test/src/test/java/org/apache/storm/st/tests/window/TumblingWindowTest.java
index 866e4c8..28b3969 100644
--- a/integration-test/src/test/java/org/apache/storm/st/tests/window/TumblingWindowTest.java
+++ b/integration-test/src/test/java/org/apache/storm/st/tests/window/TumblingWindowTest.java
@@ -30,7 +30,7 @@ import org.testng.annotations.Test;
public final class TumblingWindowTest extends AbstractTest {
private static Logger log = LoggerFactory.getLogger(TumblingWindowTest.class);
- TopoWrap topo;
+ private TopoWrap topo;
@DataProvider
public static Object[][] generateWindows() {
@@ -94,6 +94,7 @@ public final class TumblingWindowTest extends AbstractTest {
public void cleanup() throws Exception {
if (topo != null) {
topo.killOrThrow();
+ topo = null;
}
}
}
http://git-wip-us.apache.org/repos/asf/storm/blob/bbd582b8/integration-test/src/test/java/org/apache/storm/st/wrapper/StormCluster.java
----------------------------------------------------------------------
diff --git a/integration-test/src/test/java/org/apache/storm/st/wrapper/StormCluster.java b/integration-test/src/test/java/org/apache/storm/st/wrapper/StormCluster.java
index 51226e1..8a82589 100644
--- a/integration-test/src/test/java/org/apache/storm/st/wrapper/StormCluster.java
+++ b/integration-test/src/test/java/org/apache/storm/st/wrapper/StormCluster.java
@@ -93,7 +93,7 @@ public class StormCluster {
client.killTopologyWithOpts(topologyName, killOptions);
log.info("Topology killed: " + topologyName);
return;
- } catch (Throwable e) {
+ } catch (TException e) {
log.warn("Couldn't kill topology: " + topologyName + ", going to retry soon. Exception: " + ExceptionUtils.getFullStackTrace(e));
Thread.sleep(TimeUnit.SECONDS.toMillis(2));
}
[23/50] [abbrv] storm git commit: Deprecated FixedTupleSpout
constructor with String. Added constructor with Fields so more than one
output field can be specified.
Posted by pt...@apache.org.
Deprecated FixedTupleSpout constructor with String. Added constructor with Fields so more than one output field can be specified.
Project: http://git-wip-us.apache.org/repos/asf/storm/repo
Commit: http://git-wip-us.apache.org/repos/asf/storm/commit/af1fd216
Tree: http://git-wip-us.apache.org/repos/asf/storm/tree/af1fd216
Diff: http://git-wip-us.apache.org/repos/asf/storm/diff/af1fd216
Branch: refs/heads/metrics_v2
Commit: af1fd216a623e142f18248cd4d40ca1fbc19a9c4
Parents: 3414a8c
Author: Daniel Dara <dd...@g.clemson.edu>
Authored: Fri Oct 27 22:32:29 2017 -0400
Committer: Daniel Dara <dd...@g.clemson.edu>
Committed: Fri Oct 27 22:32:29 2017 -0400
----------------------------------------------------------------------
.../org/apache/storm/testing/FixedTupleSpout.java | 18 +++++++++++++-----
1 file changed, 13 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/storm/blob/af1fd216/storm-core/src/jvm/org/apache/storm/testing/FixedTupleSpout.java
----------------------------------------------------------------------
diff --git a/storm-core/src/jvm/org/apache/storm/testing/FixedTupleSpout.java b/storm-core/src/jvm/org/apache/storm/testing/FixedTupleSpout.java
index 87fc52d..5234d32 100644
--- a/storm-core/src/jvm/org/apache/storm/testing/FixedTupleSpout.java
+++ b/storm-core/src/jvm/org/apache/storm/testing/FixedTupleSpout.java
@@ -59,13 +59,21 @@ public class FixedTupleSpout implements IRichSpout {
private Map<String, FixedTuple> _pending;
private String _id;
- private String _fieldName;
+ private Fields _fields;
public FixedTupleSpout(List tuples) {
- this(tuples, null);
+ this(tuples, (Fields) null);
}
+ /**
+ * @deprecated please use {@link #FixedTupleSpout(List, Fields)}
+ */
+ @Deprecated
public FixedTupleSpout(List tuples, String fieldName) {
+ this(tuples, new Fields(fieldName));
+ }
+
+ public FixedTupleSpout(List tuples, Fields fields) {
_id = UUID.randomUUID().toString();
synchronized(acked) {
acked.put(_id, 0);
@@ -83,7 +91,7 @@ public class FixedTupleSpout implements IRichSpout {
}
_tuples.add(ft);
}
- _fieldName = fieldName;
+ _fields = fields;
}
public List<FixedTuple> getSourceTuples() {
@@ -167,8 +175,8 @@ public class FixedTupleSpout implements IRichSpout {
@Override
public void declareOutputFields(OutputFieldsDeclarer declarer) {
- if (_fieldName != null) {
- declarer.declare(new Fields(_fieldName));
+ if (_fields != null) {
+ declarer.declare(_fields);
}
}