You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@manifoldcf.apache.org by kw...@apache.org on 2014/01/16 20:25:50 UTC
svn commit: r1558893 - in /manifoldcf/branches/release-1.5-branch: ./
connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/
connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/
connectors/rs...
Author: kwright
Date: Thu Jan 16 19:25:49 2014
New Revision: 1558893
URL: http://svn.apache.org/r1558893
Log:
Pull up part of fix for CONNECTORS-858 from trunk.
Modified:
manifoldcf/branches/release-1.5-branch/ (props changed)
manifoldcf/branches/release-1.5-branch/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputConnector.java
manifoldcf/branches/release-1.5-branch/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSSession.java
manifoldcf/branches/release-1.5-branch/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/HDFSRepositoryConnector.java
manifoldcf/branches/release-1.5-branch/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/HDFSSession.java
manifoldcf/branches/release-1.5-branch/connectors/rss/ (props changed)
manifoldcf/branches/release-1.5-branch/connectors/sharepoint/ (props changed)
manifoldcf/branches/release-1.5-branch/connectors/wiki/ (props changed)
manifoldcf/branches/release-1.5-branch/framework/core/src/test/resources/org/apache/manifoldcf/core/tests/Javascript.py (props changed)
manifoldcf/branches/release-1.5-branch/framework/core/src/test/resources/org/apache/manifoldcf/core/tests/VirtualBrowser.py (props changed)
Propchange: manifoldcf/branches/release-1.5-branch/
------------------------------------------------------------------------------
Merged /manifoldcf/trunk:r1558892
Modified: manifoldcf/branches/release-1.5-branch/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/release-1.5-branch/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputConnector.java?rev=1558893&r1=1558892&r2=1558893&view=diff
==============================================================================
--- manifoldcf/branches/release-1.5-branch/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputConnector.java (original)
+++ manifoldcf/branches/release-1.5-branch/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSOutputConnector.java Thu Jan 16 19:25:49 2014
@@ -33,7 +33,6 @@ import java.util.List;
import java.util.Locale;
import java.util.Map;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -189,26 +188,13 @@ public class HDFSOutputConnector extends
if (user == null)
throw new ManifoldCFException("User must be specified");
- String nameNode = "hdfs://"+nameNodeHost+":"+nameNodePort;
+ String nameNode = "file://"+nameNodeHost+":"+nameNodePort;
//System.out.println("Namenode = '"+nameNode+"'");
/*
- * make Configuration
- */
- Configuration config = null;
- ClassLoader ocl = Thread.currentThread().getContextClassLoader();
- try {
- Thread.currentThread().setContextClassLoader(org.apache.hadoop.conf.Configuration.class.getClassLoader());
- config = new Configuration();
- config.set("fs.default.name", nameNode);
- } finally {
- Thread.currentThread().setContextClassLoader(ocl);
- }
-
- /*
* get connection to HDFS
*/
- GetSessionThread t = new GetSessionThread(nameNode,config,user);
+ GetSessionThread t = new GetSessionThread(nameNode,user);
try {
t.start();
t.finishUp();
@@ -770,15 +756,13 @@ public class HDFSOutputConnector extends
protected static class GetSessionThread extends Thread {
protected final String nameNode;
- protected final Configuration config;
protected final String user;
protected Throwable exception = null;
protected HDFSSession session = null;
- public GetSessionThread(String nameNode, Configuration config, String user) {
+ public GetSessionThread(String nameNode, String user) {
super();
this.nameNode = nameNode;
- this.config = config;
this.user = user;
setDaemon(true);
}
@@ -786,7 +770,7 @@ public class HDFSOutputConnector extends
public void run() {
try {
// Create a session
- session = new HDFSSession(nameNode, config, user);
+ session = new HDFSSession(nameNode, user);
} catch (Throwable e) {
this.exception = e;
}
Modified: manifoldcf/branches/release-1.5-branch/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSSession.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/release-1.5-branch/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSSession.java?rev=1558893&r1=1558892&r2=1558893&view=diff
==============================================================================
--- manifoldcf/branches/release-1.5-branch/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSSession.java (original)
+++ manifoldcf/branches/release-1.5-branch/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/agents/output/hdfs/HDFSSession.java Thu Jan 16 19:25:49 2014
@@ -42,16 +42,24 @@ import java.util.HashMap;
*/
public class HDFSSession {
- private FileSystem fileSystem;
+ private final FileSystem fileSystem;
private final String nameNode;
private final Configuration config;
private final String user;
- public HDFSSession(String nameNode, Configuration config, String user) throws URISyntaxException, IOException, InterruptedException {
+ public HDFSSession(String nameNode, String user) throws URISyntaxException, IOException, InterruptedException {
this.nameNode = nameNode;
- this.config = config;
this.user = user;
- fileSystem = FileSystem.get(new URI(nameNode), config, user);
+ // Switch class loaders so that scheme registration works properly
+ ClassLoader ocl = Thread.currentThread().getContextClassLoader();
+ try {
+ Thread.currentThread().setContextClassLoader(this.getClass().getClassLoader());
+ config = new Configuration();
+ config.set("fs.defaultFS", nameNode);
+ fileSystem = FileSystem.get(new URI(nameNode), config, user);
+ } finally {
+ Thread.currentThread().setContextClassLoader(ocl);
+ }
}
public Map<String, String> getRepositoryInfo() {
@@ -61,10 +69,10 @@ public class HDFSSession {
info.put("Config", config.toString());
info.put("User", user);
info.put("Canonical Service Name", fileSystem.getCanonicalServiceName());
- info.put("Default Block Size", Long.toString(fileSystem.getDefaultBlockSize()));
- info.put("Default Replication", Short.toString(fileSystem.getDefaultReplication()));
- info.put("Home Directory", fileSystem.getHomeDirectory().toUri().toString());
- info.put("Working Directory", fileSystem.getWorkingDirectory().toUri().toString());
+ //info.put("Default Block Size", Long.toString(fileSystem.getDefaultBlockSize()));
+ //info.put("Default Replication", Short.toString(fileSystem.getDefaultReplication()));
+ //info.put("Home Directory", fileSystem.getHomeDirectory().toUri().toString());
+ //info.put("Working Directory", fileSystem.getWorkingDirectory().toUri().toString());
return info;
}
Modified: manifoldcf/branches/release-1.5-branch/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/HDFSRepositoryConnector.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/release-1.5-branch/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/HDFSRepositoryConnector.java?rev=1558893&r1=1558892&r2=1558893&view=diff
==============================================================================
--- manifoldcf/branches/release-1.5-branch/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/HDFSRepositoryConnector.java (original)
+++ manifoldcf/branches/release-1.5-branch/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/HDFSRepositoryConnector.java Thu Jan 16 19:25:49 2014
@@ -19,7 +19,6 @@
package org.apache.manifoldcf.crawler.connectors.hdfs;
import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.manifoldcf.agents.interfaces.RepositoryDocument;
@@ -164,22 +163,9 @@ public class HDFSRepositoryConnector ext
Logging.connectors.debug("HDFS: User = '" + user + "'");
}
- String nameNode = "hdfs://"+nameNodeHost+":"+nameNodePort;
+ String nameNode = "file://"+nameNodeHost+":"+nameNodePort;
- /*
- * make Configuration
- */
- Configuration config = null;
- ClassLoader ocl = Thread.currentThread().getContextClassLoader();
- try {
- Thread.currentThread().setContextClassLoader(org.apache.hadoop.conf.Configuration.class.getClassLoader());
- config = new Configuration();
- config.set("fs.default.name", nameNode);
- } finally {
- Thread.currentThread().setContextClassLoader(ocl);
- }
-
- GetSessionThread t = new GetSessionThread(nameNode,config,user);
+ GetSessionThread t = new GetSessionThread(nameNode,user);
try {
t.start();
t.finishUp();
@@ -1692,15 +1678,13 @@ public class HDFSRepositoryConnector ext
protected static class GetSessionThread extends Thread {
protected final String nameNode;
- protected final Configuration config;
protected final String user;
protected Throwable exception = null;
protected HDFSSession session;
- public GetSessionThread(String nameNode, Configuration config, String user) {
+ public GetSessionThread(String nameNode, String user) {
super();
this.nameNode = nameNode;
- this.config = config;
this.user = user;
setDaemon(true);
}
@@ -1708,7 +1692,7 @@ public class HDFSRepositoryConnector ext
public void run() {
try {
// Create a session
- session = new HDFSSession(nameNode, config, user);
+ session = new HDFSSession(nameNode, user);
} catch (Throwable e) {
this.exception = e;
}
Modified: manifoldcf/branches/release-1.5-branch/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/HDFSSession.java
URL: http://svn.apache.org/viewvc/manifoldcf/branches/release-1.5-branch/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/HDFSSession.java?rev=1558893&r1=1558892&r2=1558893&view=diff
==============================================================================
--- manifoldcf/branches/release-1.5-branch/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/HDFSSession.java (original)
+++ manifoldcf/branches/release-1.5-branch/connectors/hdfs/connector/src/main/java/org/apache/manifoldcf/crawler/connectors/hdfs/HDFSSession.java Thu Jan 16 19:25:49 2014
@@ -43,18 +43,30 @@ import java.util.HashMap;
*/
public class HDFSSession {
- private FileSystem fileSystem;
- private String nameNode;
- private Configuration config;
- private String user;
+ private final FileSystem fileSystem;
+ private final String nameNode;
+ private final Configuration config;
+ private final String user;
- public HDFSSession(String nameNode, Configuration config, String user) throws URISyntaxException, IOException, InterruptedException {
+ public HDFSSession(String nameNode, String user) throws URISyntaxException, IOException, InterruptedException {
this.nameNode = nameNode;
- this.config = config;
this.user = user;
- fileSystem = FileSystem.get(new URI(nameNode), config, user);
+ // Switch class loaders so that scheme registration works properly
+ ClassLoader ocl = Thread.currentThread().getContextClassLoader();
+ try {
+ Thread.currentThread().setContextClassLoader(this.getClass().getClassLoader());
+ config = new Configuration();
+ config.set("fs.defaultFS", nameNode);
+ fileSystem = FileSystem.get(new URI(nameNode), config, user);
+ } finally {
+ Thread.currentThread().setContextClassLoader(ocl);
+ }
}
+ public static void runMe()
+ {
+ }
+
public Map<String, String> getRepositoryInfo() {
Map<String, String> info = new HashMap<String, String>();
Propchange: manifoldcf/branches/release-1.5-branch/connectors/rss/
------------------------------------------------------------------------------
Merged /manifoldcf/trunk/connectors/rss:r1558892
Propchange: manifoldcf/branches/release-1.5-branch/connectors/sharepoint/
------------------------------------------------------------------------------
Merged /manifoldcf/trunk/connectors/sharepoint:r1558892
Propchange: manifoldcf/branches/release-1.5-branch/connectors/wiki/
------------------------------------------------------------------------------
Merged /manifoldcf/trunk/connectors/wiki:r1558892
Propchange: manifoldcf/branches/release-1.5-branch/framework/core/src/test/resources/org/apache/manifoldcf/core/tests/Javascript.py
------------------------------------------------------------------------------
Merged /manifoldcf/trunk/framework/core/src/test/resources/org/apache/manifoldcf/core/tests/Javascript.py:r1558892
Propchange: manifoldcf/branches/release-1.5-branch/framework/core/src/test/resources/org/apache/manifoldcf/core/tests/VirtualBrowser.py
------------------------------------------------------------------------------
Merged /manifoldcf/trunk/framework/core/src/test/resources/org/apache/manifoldcf/core/tests/VirtualBrowser.py:r1558892