You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@streams.apache.org by sb...@apache.org on 2015/07/07 01:05:27 UTC
[1/2] incubator-streams git commit: improves fix to STREAMS-337
Repository: incubator-streams
Updated Branches:
refs/heads/master 77f38dee5 -> 66836839e
improves fix to STREAMS-337
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/d51bc8ed
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/d51bc8ed
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/d51bc8ed
Branch: refs/heads/master
Commit: d51bc8ed773fc43e0be590b782a3dfe10f8fb38f
Parents: 2cd5df7
Author: Steve Blackmon (@steveblackmon) <sb...@apache.org>
Authored: Sat Jul 4 12:50:52 2015 -0500
Committer: Steve Blackmon (@steveblackmon) <sb...@apache.org>
Committed: Sat Jul 4 12:50:52 2015 -0500
----------------------------------------------------------------------
.../java/org/apache/streams/hdfs/WebHdfsPersistReader.java | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-streams/blob/d51bc8ed/streams-contrib/streams-persist-hdfs/src/main/java/org/apache/streams/hdfs/WebHdfsPersistReader.java
----------------------------------------------------------------------
diff --git a/streams-contrib/streams-persist-hdfs/src/main/java/org/apache/streams/hdfs/WebHdfsPersistReader.java b/streams-contrib/streams-persist-hdfs/src/main/java/org/apache/streams/hdfs/WebHdfsPersistReader.java
index 2cddbc3..f1432a5 100644
--- a/streams-contrib/streams-persist-hdfs/src/main/java/org/apache/streams/hdfs/WebHdfsPersistReader.java
+++ b/streams-contrib/streams-persist-hdfs/src/main/java/org/apache/streams/hdfs/WebHdfsPersistReader.java
@@ -30,6 +30,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hdfs.web.WebHdfsFileSystem;
import org.apache.hadoop.security.UserGroupInformation;
import org.apache.streams.config.ComponentConfigurator;
+import org.apache.streams.config.StreamsConfiguration;
import org.apache.streams.config.StreamsConfigurator;
import org.apache.streams.core.*;
import org.apache.streams.jackson.StreamsJacksonMapper;
@@ -70,6 +71,7 @@ public class WebHdfsPersistReader implements StreamsPersistReader, DatumStatusCo
protected ObjectMapper mapper = StreamsJacksonMapper.getInstance();
protected HdfsReaderConfiguration hdfsConfiguration;
+ protected StreamsConfiguration streamsConfiguration;
private ExecutorService executor;
@@ -118,6 +120,8 @@ public class WebHdfsPersistReader implements StreamsPersistReader, DatumStatusCo
public Void run() throws Exception {
Configuration conf = new Configuration();
conf.set(CommonConfigurationKeysPublic.HADOOP_SECURITY_AUTHENTICATION, "kerberos");
+ conf.set("fs.hdfs.impl", org.apache.hadoop.hdfs.DistributedFileSystem.class.getName());
+ conf.set("fs.file.impl", org.apache.hadoop.fs.LocalFileSystem.class.getName());
LOGGER.info("WebURI : {}", getURI().toString());
client = FileSystem.get(getURI(), conf);
LOGGER.info("Connected to WebHDFS");
@@ -169,7 +173,8 @@ public class WebHdfsPersistReader implements StreamsPersistReader, DatumStatusCo
} catch (IOException e) {
LOGGER.error("IOException", e);
}
- persistQueue = Queues.synchronizedQueue(new LinkedBlockingQueue<StreamsDatum>(10000));
+ streamsConfiguration = StreamsConfigurator.detectConfiguration();
+ persistQueue = Queues.synchronizedQueue(new LinkedBlockingQueue<StreamsDatum>(streamsConfiguration.getBatchSize().intValue()));
//persistQueue = Queues.synchronizedQueue(new ConcurrentLinkedQueue());
executor = Executors.newSingleThreadExecutor();
}
[2/2] incubator-streams git commit: Merge branch 'STREAMS-337.2'
Posted by sb...@apache.org.
Merge branch 'STREAMS-337.2'
* STREAMS-337.2:
improves fix to STREAMS-337
Project: http://git-wip-us.apache.org/repos/asf/incubator-streams/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-streams/commit/66836839
Tree: http://git-wip-us.apache.org/repos/asf/incubator-streams/tree/66836839
Diff: http://git-wip-us.apache.org/repos/asf/incubator-streams/diff/66836839
Branch: refs/heads/master
Commit: 66836839ee57eb2d4fd21ca72b448ddd81a9463b
Parents: 77f38de d51bc8e
Author: Steve Blackmon (@steveblackmon) <sb...@apache.org>
Authored: Mon Jul 6 18:05:29 2015 -0500
Committer: Steve Blackmon (@steveblackmon) <sb...@apache.org>
Committed: Mon Jul 6 18:05:29 2015 -0500
----------------------------------------------------------------------
.../java/org/apache/streams/hdfs/WebHdfsPersistReader.java | 7 ++++++-
1 file changed, 6 insertions(+), 1 deletion(-)
----------------------------------------------------------------------