You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hdt.apache.org by rs...@apache.org on 2014/06/26 10:36:30 UTC
[07/27] git commit: HDT-50: Adding HDFS site if found - Minor tweaks
to the hadoop perspective - Modified create/open of HDFS project in
WorkspaceJob - Modified RunOnHadoop to add HDFS site. Show error in dialog if
found. - Adding timeoutcheck to MR clus
HDT-50: Adding HDFS site if found - Minor tweaks to the hadoop perspective - Modified create/open of HDFS project in WorkspaceJob - Modified RunOnHadoop to add HDFS site. Show error in dialog if found. - Adding timeoutcheck to MR cluster
Project: http://git-wip-us.apache.org/repos/asf/incubator-hdt/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hdt/commit/00646ae0
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hdt/tree/00646ae0
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hdt/diff/00646ae0
Branch: refs/heads/hadoop-eclipse-merge
Commit: 00646ae0b3e502dda9ec3a4ddb5484412cf709ef
Parents: 0e9e729
Author: Rahul Sharma <rs...@apache.org>
Authored: Tue May 13 17:00:53 2014 +0530
Committer: Rahul Sharma <rs...@apache.org>
Committed: Fri May 23 08:44:02 2014 +0530
----------------------------------------------------------------------
.../hdt/core/internal/hdfs/HDFSManager.java | 61 +++++++++++++++++---
.../hdt/core/launch/AbstractHadoopCluster.java | 4 ++
.../hdt/hadoop/release/HadoopCluster.java | 33 +++++++++++
org.apache.hdt.ui/plugin.xml | 14 ++---
.../hdt/ui/internal/hdfs/NewHDFSWizard.java | 21 ++-----
.../internal/launch/HadoopLocationWizard.java | 43 +++++++++++++-
.../hdt/ui/internal/launch/ServerRegistry.java | 53 ++++++++++++-----
7 files changed, 184 insertions(+), 45 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hdt/blob/00646ae0/org.apache.hdt.core/src/org/apache/hdt/core/internal/hdfs/HDFSManager.java
----------------------------------------------------------------------
diff --git a/org.apache.hdt.core/src/org/apache/hdt/core/internal/hdfs/HDFSManager.java b/org.apache.hdt.core/src/org/apache/hdt/core/internal/hdfs/HDFSManager.java
index 93f0696..5897cea 100644
--- a/org.apache.hdt.core/src/org/apache/hdt/core/internal/hdfs/HDFSManager.java
+++ b/org.apache.hdt.core/src/org/apache/hdt/core/internal/hdfs/HDFSManager.java
@@ -18,6 +18,7 @@
package org.apache.hdt.core.internal.hdfs;
+import java.net.URI;
import java.net.URISyntaxException;
import java.util.HashMap;
import java.util.LinkedHashMap;
@@ -37,13 +38,19 @@ import org.eclipse.core.resources.IResource;
import org.eclipse.core.resources.IWorkspace;
import org.eclipse.core.resources.IWorkspaceRoot;
import org.eclipse.core.resources.ResourcesPlugin;
+import org.eclipse.core.resources.WorkspaceJob;
import org.eclipse.core.runtime.CoreException;
import org.eclipse.core.runtime.IConfigurationElement;
+import org.eclipse.core.runtime.IProgressMonitor;
import org.eclipse.core.runtime.IStatus;
import org.eclipse.core.runtime.NullProgressMonitor;
import org.eclipse.core.runtime.Platform;
import org.eclipse.core.runtime.Status;
+import org.eclipse.core.runtime.SubProgressMonitor;
+import org.eclipse.core.runtime.jobs.Job;
import org.eclipse.emf.common.util.EList;
+import org.eclipse.jface.dialogs.MessageDialog;
+import org.eclipse.swt.widgets.Display;
import org.eclipse.team.core.RepositoryProvider;
/**
@@ -176,14 +183,39 @@ public class HDFSManager {
* @return
* @throws CoreException
*/
- private IProject createIProject(String name, java.net.URI hdfsURI) throws CoreException {
+ private IProject createIProject(String name, final java.net.URI hdfsURI) {
final IWorkspace workspace = ResourcesPlugin.getWorkspace();
- IProject project = workspace.getRoot().getProject(name);
- IProjectDescription pd = workspace.newProjectDescription(name);
- pd.setLocationURI(hdfsURI);
- project.create(pd, new NullProgressMonitor());
- project.open(new NullProgressMonitor());
- RepositoryProvider.map(project, HDFSTeamRepositoryProvider.ID);
+ final IProject project = workspace.getRoot().getProject(name);
+ final IProjectDescription pd = workspace.newProjectDescription(name);
+ WorkspaceJob operation = new WorkspaceJob("Adding HDFS Location") {
+
+ @Override
+ public IStatus runInWorkspace(IProgressMonitor monitor) throws CoreException {
+ monitor.beginTask("Creating Project", 100);
+ try {
+ pd.setLocationURI(hdfsURI);
+ project.create(pd, new SubProgressMonitor(monitor, 70));
+ project.open(IResource.BACKGROUND_REFRESH, new SubProgressMonitor(monitor, 30));
+ RepositoryProvider.map(project, HDFSTeamRepositoryProvider.ID);
+ return Status.OK_STATUS;
+ } catch (final CoreException e) {
+ logger.error("error found in creating HDFS site", e);
+ Display.getDefault().syncExec(new Runnable(){
+ public void run(){
+ MessageDialog.openError(Display.getDefault().getActiveShell(),
+ "HDFS Error", "Unable to create HDFS site :"+e.getMessage());
+ }
+ });
+ return e.getStatus();
+ } finally {
+ monitor.done();
+ }
+ }
+ };
+ operation.setPriority(Job.LONG);
+ operation.setUser(true);
+ operation.setRule(project);
+ operation.schedule();
return project;
}
@@ -204,6 +236,8 @@ public class HDFSManager {
}
return uriToServerCacheMap.get(uri);
}
+
+
public String getProjectName(HDFSServer server) {
return serverToProjectMap.get(server);
@@ -282,4 +316,17 @@ public class HDFSManager {
return hdfsClientsMap.get(serverURI);
}
}
+
+ public static org.eclipse.core.runtime.IStatus addServer(String serverName, String location, String userId, List<String> groupId) {
+ try {
+ HDFSManager.INSTANCE.createServer(serverName, new URI(location), userId, groupId);
+ } catch (CoreException e) {
+ logger.warn(e.getMessage(), e);
+ return e.getStatus();
+ } catch (URISyntaxException e) {
+ logger.warn(e.getMessage(), e);
+ return new Status(Status.ERROR,"unknown",e.getMessage(),e);
+ }
+ return Status.OK_STATUS;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-hdt/blob/00646ae0/org.apache.hdt.core/src/org/apache/hdt/core/launch/AbstractHadoopCluster.java
----------------------------------------------------------------------
diff --git a/org.apache.hdt.core/src/org/apache/hdt/core/launch/AbstractHadoopCluster.java b/org.apache.hdt.core/src/org/apache/hdt/core/launch/AbstractHadoopCluster.java
index 782a89c..cd06f0e 100644
--- a/org.apache.hdt.core/src/org/apache/hdt/core/launch/AbstractHadoopCluster.java
+++ b/org.apache.hdt.core/src/org/apache/hdt/core/launch/AbstractHadoopCluster.java
@@ -64,6 +64,8 @@ public abstract class AbstractHadoopCluster {
abstract public boolean loadFromXML(File file) throws IOException;
+ abstract public boolean isAvailable() throws CoreException;
+
public static AbstractHadoopCluster createCluster(File file) throws CoreException, IOException {
AbstractHadoopCluster hadoopCluster = createCluster();
hadoopCluster.loadFromXML(file);
@@ -81,4 +83,6 @@ public abstract class AbstractHadoopCluster {
return hadoopCluster;
}
+
+
}
http://git-wip-us.apache.org/repos/asf/incubator-hdt/blob/00646ae0/org.apache.hdt.hadoop.release/src/org/apache/hdt/hadoop/release/HadoopCluster.java
----------------------------------------------------------------------
diff --git a/org.apache.hdt.hadoop.release/src/org/apache/hdt/hadoop/release/HadoopCluster.java b/org.apache.hdt.hadoop.release/src/org/apache/hdt/hadoop/release/HadoopCluster.java
index 466739b..67fcb75 100644
--- a/org.apache.hdt.hadoop.release/src/org/apache/hdt/hadoop/release/HadoopCluster.java
+++ b/org.apache.hdt.hadoop.release/src/org/apache/hdt/hadoop/release/HadoopCluster.java
@@ -31,6 +31,13 @@ import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.TreeMap;
+import java.util.concurrent.Callable;
+import java.util.concurrent.ExecutionException;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
+import java.util.concurrent.Future;
+import java.util.concurrent.TimeUnit;
+import java.util.concurrent.TimeoutException;
import java.util.logging.Logger;
import javax.xml.parsers.DocumentBuilder;
@@ -52,6 +59,8 @@ import org.apache.hdt.core.launch.IHadoopJob;
import org.apache.hdt.core.launch.IJarModule;
import org.apache.hdt.core.launch.IJobListener;
import org.eclipse.core.internal.utils.FileUtil;
+import org.eclipse.core.resources.WorkspaceJob;
+import org.eclipse.core.runtime.CoreException;
import org.eclipse.core.runtime.IProgressMonitor;
import org.eclipse.core.runtime.IStatus;
import org.eclipse.core.runtime.Status;
@@ -81,6 +90,7 @@ import org.xml.sax.SAXException;
*/
public class HadoopCluster extends AbstractHadoopCluster {
+ private ExecutorService service= Executors.newSingleThreadExecutor();
/**
* Frequency of location status observations expressed as the delay in ms
@@ -566,4 +576,27 @@ public class HadoopCluster extends AbstractHadoopCluster {
}
}
+
+ /* (non-Javadoc)
+ * @see org.apache.hdt.core.launch.AbstractHadoopCluster#isAvailable()
+ */
+ @Override
+ public boolean isAvailable() throws CoreException {
+ Callable<JobClient> task= new Callable<JobClient>() {
+
+ @Override
+ public JobClient call() throws Exception {
+ return getJobClient();
+ }
+ };
+ Future<JobClient> jobClientFuture = service.submit(task);
+ try{
+ JobClient jobClient = jobClientFuture.get(5, TimeUnit.SECONDS);
+ return jobClient!=null;
+ }catch(Exception e){
+ e.printStackTrace();
+ throw new CoreException(new Status(Status.ERROR,
+ Activator.BUNDLE_ID, "unable to connect to server", e));
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-hdt/blob/00646ae0/org.apache.hdt.ui/plugin.xml
----------------------------------------------------------------------
diff --git a/org.apache.hdt.ui/plugin.xml b/org.apache.hdt.ui/plugin.xml
index 7bc5b47..af68ffa 100644
--- a/org.apache.hdt.ui/plugin.xml
+++ b/org.apache.hdt.ui/plugin.xml
@@ -39,31 +39,31 @@
<perspectiveExtension
targetID="org.apache.hdt.ui.perspective">
<view
- id="org.apache.hdt.ui.view.servers"
+ id="org.eclipse.ui.navigator.ProjectExplorer"
minimized="false"
ratio="0.25"
relationship="left"
relative="org.eclipse.ui.editorss">
</view>
<view
- id="org.eclipse.ui.navigator.ProjectExplorer"
+ id="org.apache.hdt.ui.view.servers"
minimized="false"
ratio="0.5"
relationship="bottom"
- relative="org.apache.hdt.ui.view.servers">
+ relative="org.eclipse.ui.navigator.ProjectExplorer">
</view>
<view
- id="org.eclipse.ui.views.PropertySheet"
+ id="org.apache.hdt.ui.ClusterView"
minimized="false"
ratio="0.66"
relationship="bottom"
relative="org.eclipse.ui.editorss">
</view>
<view
- id=" org.apache.hdt.ui.ClusterView"
+ id="org.eclipse.ui.views.PropertySheet"
minimized="false"
relationship="stack"
- relative="org.apache.hdt.ui.view.servers">
+ relative="org.apache.hdt.ui.ClusterView">
</view>
<newWizardShortcut
id="org.apache.hdt.ui.wizard.newProjectWizard"/>
@@ -441,7 +441,7 @@
class="org.eclipse.ui.navigator.CommonNavigator"
icon="icons/hadoop-logo-16x16.png"
id="org.apache.hdt.ui.view.servers"
- name="Hadoop Servers"
+ name="ZooKeeper Servers"
restorable="true">
</view>
<view
http://git-wip-us.apache.org/repos/asf/incubator-hdt/blob/00646ae0/org.apache.hdt.ui/src/org/apache/hdt/ui/internal/hdfs/NewHDFSWizard.java
----------------------------------------------------------------------
diff --git a/org.apache.hdt.ui/src/org/apache/hdt/ui/internal/hdfs/NewHDFSWizard.java b/org.apache.hdt.ui/src/org/apache/hdt/ui/internal/hdfs/NewHDFSWizard.java
index 545ea3a..4cd40de 100644
--- a/org.apache.hdt.ui/src/org/apache/hdt/ui/internal/hdfs/NewHDFSWizard.java
+++ b/org.apache.hdt.ui/src/org/apache/hdt/ui/internal/hdfs/NewHDFSWizard.java
@@ -17,14 +17,9 @@
*/
package org.apache.hdt.ui.internal.hdfs;
-import java.net.URI;
-import java.net.URISyntaxException;
-
import org.apache.hdt.core.internal.hdfs.HDFSManager;
import org.apache.hdt.ui.Activator;
import org.apache.log4j.Logger;
-import org.eclipse.core.runtime.CoreException;
-import org.eclipse.core.runtime.Status;
import org.eclipse.core.runtime.jobs.Job;
import org.eclipse.jface.preference.IPreferenceStore;
import org.eclipse.jface.viewers.IStructuredSelection;
@@ -73,17 +68,9 @@ public class NewHDFSWizard extends Wizard implements INewWizard {
Job j = new Job("Creating HDFS project [" + serverLocationWizardPage.getHdfsServerName() + "]") {
protected org.eclipse.core.runtime.IStatus run(org.eclipse.core.runtime.IProgressMonitor monitor) {
- try {
- HDFSManager.INSTANCE.createServer(serverLocationWizardPage.getHdfsServerName(), new URI(serverLocationWizardPage
- .getHdfsServerLocation()), serverLocationWizardPage.isOverrideDefaultSecurity() ? serverLocationWizardPage.getUserId()
- : null, serverLocationWizardPage.isOverrideDefaultSecurity() ? serverLocationWizardPage.getGroupIds() : null);
- } catch (CoreException e) {
- logger.warn(e.getMessage(), e);
- return e.getStatus();
- } catch (URISyntaxException e) {
- logger.warn(e.getMessage(), e);
- }
- return Status.OK_STATUS;
+ return HDFSManager.addServer(serverLocationWizardPage.getHdfsServerName(),serverLocationWizardPage.getHdfsServerLocation(),
+ serverLocationWizardPage.isOverrideDefaultSecurity() ? serverLocationWizardPage.getUserId() : null,
+ serverLocationWizardPage.isOverrideDefaultSecurity() ? serverLocationWizardPage.getGroupIds() : null);
};
};
j.schedule();
@@ -93,4 +80,6 @@ public class NewHDFSWizard extends Wizard implements INewWizard {
return false;
}
+
+
}
http://git-wip-us.apache.org/repos/asf/incubator-hdt/blob/00646ae0/org.apache.hdt.ui/src/org/apache/hdt/ui/internal/launch/HadoopLocationWizard.java
----------------------------------------------------------------------
diff --git a/org.apache.hdt.ui/src/org/apache/hdt/ui/internal/launch/HadoopLocationWizard.java b/org.apache.hdt.ui/src/org/apache/hdt/ui/internal/launch/HadoopLocationWizard.java
index 4f11128..3757c05 100644
--- a/org.apache.hdt.ui/src/org/apache/hdt/ui/internal/launch/HadoopLocationWizard.java
+++ b/org.apache.hdt.ui/src/org/apache/hdt/ui/internal/launch/HadoopLocationWizard.java
@@ -28,8 +28,12 @@ import java.util.Set;
import java.util.SortedMap;
import java.util.TreeMap;
-import org.apache.hdt.core.launch.ConfProp;
+import org.apache.hdt.core.internal.hdfs.HDFSManager;
import org.apache.hdt.core.launch.AbstractHadoopCluster;
+import org.apache.hdt.core.launch.ConfProp;
+import org.eclipse.core.resources.IProject;
+import org.eclipse.core.resources.IWorkspaceRoot;
+import org.eclipse.core.resources.ResourcesPlugin;
import org.eclipse.core.runtime.CoreException;
import org.eclipse.jface.dialogs.IMessageProvider;
import org.eclipse.jface.wizard.WizardPage;
@@ -118,17 +122,49 @@ public class HadoopLocationWizard extends WizardPage {
public AbstractHadoopCluster performFinish() {
try {
if (this.original == null) {
+ Display.getDefault().syncExec(new Runnable() {
+ public void run() {
+ HDFSManager.addServer(location.getLocationName(),
+ location.getConfProp(ConfProp.FS_DEFAULT_URI), location
+ .getConfProp(ConfProp.PI_USER_NAME), null);
+ }
+ });
// New location
Display.getDefault().syncExec(new Runnable() {
public void run() {
- ServerRegistry.getInstance().addServer(HadoopLocationWizard.this.location);
+ ServerRegistry.getInstance().addServer(HadoopLocationWizard.this.location);
}
});
return this.location;
} else {
+
// Update location
final String originalName = this.original.getLocationName();
+ final String originalLoc = this.original.getConfProp(ConfProp.FS_DEFAULT_URI);
+ final String newName = this.location.getLocationName();
+ final String newLoc = this.location.getConfProp(ConfProp.FS_DEFAULT_URI);
+
+ if (!originalName.equals(newName) || !originalLoc.equals(newLoc)){
+ IWorkspaceRoot root = ResourcesPlugin.getWorkspace().getRoot();
+ final IProject project = root.getProject(originalName);
+
+ Display.getDefault().syncExec(new Runnable() {
+ public void run() {
+ if(project.exists()){
+ try {
+ project.close(null);
+ project.delete(true, null);
+ } catch (CoreException e) {
+ e.printStackTrace();
+ }
+ }
+ HDFSManager.addServer(location.getLocationName(),
+ location.getConfProp(ConfProp.FS_DEFAULT_URI), location
+ .getConfProp(ConfProp.PI_USER_NAME), null);
+ }
+ });
+ }
this.original.load(this.location);
Display.getDefault().syncExec(new Runnable() {
@@ -139,6 +175,9 @@ public class HadoopLocationWizard extends WizardPage {
return this.original;
}
+
+
+
} catch (Exception e) {
e.printStackTrace();
setMessage("Invalid server location values", IMessageProvider.ERROR);
http://git-wip-us.apache.org/repos/asf/incubator-hdt/blob/00646ae0/org.apache.hdt.ui/src/org/apache/hdt/ui/internal/launch/ServerRegistry.java
----------------------------------------------------------------------
diff --git a/org.apache.hdt.ui/src/org/apache/hdt/ui/internal/launch/ServerRegistry.java b/org.apache.hdt.ui/src/org/apache/hdt/ui/internal/launch/ServerRegistry.java
index 785286c..f03fb50 100644
--- a/org.apache.hdt.ui/src/org/apache/hdt/ui/internal/launch/ServerRegistry.java
+++ b/org.apache.hdt.ui/src/org/apache/hdt/ui/internal/launch/ServerRegistry.java
@@ -33,6 +33,12 @@ import javax.security.auth.login.Configuration;
import org.apache.hdt.core.launch.AbstractHadoopCluster;
import org.apache.hdt.core.launch.IHadoopClusterListener;
import org.apache.hdt.ui.Activator;
+import org.eclipse.core.resources.ResourcesPlugin;
+import org.eclipse.core.resources.WorkspaceJob;
+import org.eclipse.core.runtime.CoreException;
+import org.eclipse.core.runtime.IProgressMonitor;
+import org.eclipse.core.runtime.IStatus;
+import org.eclipse.core.runtime.jobs.Job;
import org.eclipse.jface.dialogs.MessageDialog;
/**
@@ -173,10 +179,22 @@ public class ServerRegistry {
fireListeners(server, SERVER_REMOVED);
}
- public synchronized void addServer(AbstractHadoopCluster server) {
- this.servers.put(server.getLocationName(), server);
- store();
- fireListeners(server, SERVER_ADDED);
+ public synchronized void addServer(final AbstractHadoopCluster server) {
+ WorkspaceJob job= new WorkspaceJob("Adding Hadoop Server") {
+ @Override
+ public IStatus runInWorkspace(IProgressMonitor monitor) throws CoreException {
+ if(server.isAvailable()){
+ servers.put(server.getLocationName(), server);
+ store();
+ fireListeners(server, SERVER_ADDED);
+ }
+ return org.eclipse.core.runtime.Status.OK_STATUS;
+ }};
+
+ job.setPriority(Job.LONG);
+ job.setRule(ResourcesPlugin.getWorkspace().getRoot());
+ job.setUser(true);
+ job.schedule();
}
/**
@@ -187,14 +205,23 @@ public class ServerRegistry {
* @param server
* the location
*/
- public synchronized void updateServer(String originalName, AbstractHadoopCluster server) {
-
- // Update the map if the location name has changed
- if (!server.getLocationName().equals(originalName)) {
- servers.remove(originalName);
- servers.put(server.getLocationName(), server);
- }
- store();
- fireListeners(server, SERVER_STATE_CHANGED);
+ public synchronized void updateServer(final String originalName, final AbstractHadoopCluster server) {
+ WorkspaceJob job= new WorkspaceJob("Updating Hadoop Server") {
+ @Override
+ public IStatus runInWorkspace(IProgressMonitor monitor) throws CoreException {
+ // Update the map if the location name has changed
+ if (!server.getLocationName().equals(originalName) && server.isAvailable()) {
+ servers.remove(originalName);
+ servers.put(server.getLocationName(), server);
+ store();
+ fireListeners(server, SERVER_STATE_CHANGED);
+ }
+
+ return org.eclipse.core.runtime.Status.OK_STATUS;
+ }};
+
+ job.setPriority(Job.LONG);
+ job.setUser(true);
+ job.schedule();
}
}