You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@falcon.apache.org by pa...@apache.org on 2017/01/24 11:05:19 UTC
falcon git commit: FALCON-2261 Job submission fails with
IllegalArgumentException. Creating falconHdfsFilesystem based on the uri
Repository: falcon
Updated Branches:
refs/heads/master 9da6d3804 -> e132065f1
FALCON-2261 Job submission fails with IllegalArgumentException. Creating falconHdfsFilesystem based on the uri
Author: sandeep <sa...@gmail.com>
Reviewers: @pallavi-rao
Closes #346 from sandeepSamudrala/FALCON-2261 and squashes the following commits:
3fefbc7 [sandeep] Fixed checkstyle issues
24b3da9 [sandeep] FALCON-2261 Job submission fails with IllegalArgumentException. Creating falcon hdfs filesystem based on the uri
c5da0a2 [sandeep] Merge branch 'master' of https://github.com/apache/falcon
7e16263 [sandeep] Merge branch 'master' of https://github.com/apache/falcon
a234d94 [sandeep] FALCON-2231 Incoporated review comments and small fixes for duplicate submission and colo addition to schedule command
26e3350 [sandeep] Merge branch 'master' of https://github.com/apache/falcon
73fbf75 [sandeep] Merge branch 'master' of https://github.com/apache/falcon
cc28658 [sandeep] Merge branch 'master' of https://github.com/apache/falcon
089b10d [sandeep] Merge branch 'master' of https://github.com/apache/falcon
456d4ee [sandeep] Merge branch 'master' of https://github.com/apache/falcon
0cf9af6 [sandeep] Merge branch 'master' of https://github.com/apache/falcon
4a2e23e [sandeep] Merge branch 'master' of https://github.com/apache/falcon
b1546ed [sandeep] Merge branch 'master' of https://github.com/apache/falcon
0a433fb [sandeep] Merge branch 'master' of https://github.com/apache/falcon
194f36a [sandeep] Merge branch 'master' of https://github.com/apache/falcon
e0ad358 [sandeep] Merge branch 'master' of https://github.com/apache/falcon
f96a084 [sandeep] Merge branch 'master' of https://github.com/apache/falcon
9cf36e9 [sandeep] Merge branch 'master' of https://github.com/apache/falcon
bbca081 [sandeep] Merge branch 'master' of https://github.com/apache/falcon
48f6afa [sandeep] Merge branch 'master' of https://github.com/apache/falcon
250cc46 [sandeep] Merge branch 'master' of https://github.com/apache/falcon
d0393e9 [sandeep] Merge branch 'master' of https://github.com/apache/falcon
a178805 [sandeep] Merge branch 'master' of https://github.com/apache/falcon
d6dc8bf [sandeep] Merge branch 'master' of https://github.com/apache/falcon
1bb8d3c [sandeep] Merge branch 'master' of https://github.com/apache/falcon
c065566 [sandeep] reverting last line changes made
1a4dcd2 [sandeep] rebased and resolved the conflicts from master
271318b [sandeep] FALCON-2097. Adding UT to the new method for getting next instance time with Delay.
a94d4fe [sandeep] rebasing from master
9e68a57 [sandeep] FALCON-298. Feed update with replication delay creates holes
Project: http://git-wip-us.apache.org/repos/asf/falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/e132065f
Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/e132065f
Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/e132065f
Branch: refs/heads/master
Commit: e132065f1e814f86e0dbc21f473e19c695a01a21
Parents: 9da6d38
Author: sandeep <sa...@gmail.com>
Authored: Tue Jan 24 16:35:09 2017 +0530
Committer: Pallavi Rao <pa...@inmobi.com>
Committed: Tue Jan 24 16:35:09 2017 +0530
----------------------------------------------------------------------
.../java/org/apache/falcon/ExtensionHandler.java | 16 +++++++++-------
.../falcon/extensions/store/ExtensionStore.java | 5 -----
.../org/apache/falcon/unit/FalconUnitClient.java | 3 ++-
3 files changed, 11 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/falcon/blob/e132065f/client/src/main/java/org/apache/falcon/ExtensionHandler.java
----------------------------------------------------------------------
diff --git a/client/src/main/java/org/apache/falcon/ExtensionHandler.java b/client/src/main/java/org/apache/falcon/ExtensionHandler.java
index f6f3346..7d63d11 100644
--- a/client/src/main/java/org/apache/falcon/ExtensionHandler.java
+++ b/client/src/main/java/org/apache/falcon/ExtensionHandler.java
@@ -24,7 +24,7 @@ import org.apache.falcon.client.FalconExtensionConstants;
import org.apache.falcon.entity.v0.Entity;
import org.apache.falcon.entity.v0.EntityType;
import org.apache.falcon.extensions.ExtensionBuilder;
-import org.apache.hadoop.conf.Configuration;
+import org.apache.falcon.hadoop.HadoopClientFactory;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.codehaus.jettison.json.JSONException;
@@ -38,6 +38,8 @@ import java.io.IOException;
import java.io.InputStream;
import java.io.OutputStream;
import java.net.MalformedURLException;
+import java.net.URI;
+import java.net.URISyntaxException;
import java.net.URL;
import java.net.URLEncoder;
import java.util.ArrayList;
@@ -95,11 +97,10 @@ public final class ExtensionHandler {
}
public static List<Entity> loadAndPrepare(String extensionName, String jobName, InputStream configStream,
- String extensionBuildLocation) throws IOException, FalconException {
- Configuration conf = new Configuration();
- FileSystem fs = FileSystem.get(conf);
+ String extensionBuildLocation)
+ throws IOException, FalconException, URISyntaxException {
String stagePath = createStagePath(extensionName, jobName);
- List<URL> urls = ExtensionHandler.copyExtensionPackage(extensionBuildLocation, fs, stagePath);
+ List<URL> urls = ExtensionHandler.copyExtensionPackage(extensionBuildLocation, stagePath);
List<Entity> entities = prepare(extensionName, jobName, configStream, urls);
ExtensionHandler.stageEntities(entities, stagePath);
@@ -152,9 +153,10 @@ public final class ExtensionHandler {
return stagePath;
}
- private static List<URL> copyExtensionPackage(String extensionBuildUrl, FileSystem fs, String stagePath)
- throws IOException {
+ private static List<URL> copyExtensionPackage(String extensionBuildUrl, String stagePath)
+ throws IOException, FalconException, URISyntaxException {
+ FileSystem fs = HadoopClientFactory.get().createProxiedFileSystem(new URI(extensionBuildUrl));
Path libsPath = new Path(extensionBuildUrl, FalconExtensionConstants.LIBS);
Path buildLibsPath = new Path(libsPath, FalconExtensionConstants.BUILD);
Path localStagePath = new Path(stagePath);
http://git-wip-us.apache.org/repos/asf/falcon/blob/e132065f/extensions/src/main/java/org/apache/falcon/extensions/store/ExtensionStore.java
----------------------------------------------------------------------
diff --git a/extensions/src/main/java/org/apache/falcon/extensions/store/ExtensionStore.java b/extensions/src/main/java/org/apache/falcon/extensions/store/ExtensionStore.java
index c50d6de..1d71651 100644
--- a/extensions/src/main/java/org/apache/falcon/extensions/store/ExtensionStore.java
+++ b/extensions/src/main/java/org/apache/falcon/extensions/store/ExtensionStore.java
@@ -29,7 +29,6 @@ import org.apache.falcon.extensions.jdbc.ExtensionMetaStore;
import org.apache.falcon.hadoop.HadoopClientFactory;
import org.apache.falcon.persistence.ExtensionBean;
import org.apache.falcon.util.StartupProperties;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.LocatedFileStatus;
@@ -300,7 +299,6 @@ public final class ExtensionStore {
}
private FileSystem getHdfsFileSystem(String path) throws FalconException {
- Configuration conf = new Configuration();
URI uri;
try {
uri = new URI(path);
@@ -308,19 +306,16 @@ public final class ExtensionStore {
LOG.error("Exception : ", e);
throw new FalconException(e);
}
- conf.set("fs.default.name", uri.getScheme() + "://" + uri.getAuthority());
return HadoopClientFactory.get().createFalconFileSystem(uri);
}
public String registerExtension(final String extensionName, final String path, final String description,
String extensionOwner) throws URISyntaxException, FalconException {
- Configuration conf = new Configuration();
URI uri = new URI(path);
assertURI("Scheme", uri.getScheme());
assertURI("Authority", uri.getAuthority());
assertURI("Path", uri.getPath());
- conf.set("fs.defaultFS", uri.getScheme() + "://" + uri.getAuthority());
FileSystem fileSystem = getHdfsFileSystem(path);
try {
fileSystem.listStatus(new Path(uri.getPath() + "/README"));
http://git-wip-us.apache.org/repos/asf/falcon/blob/e132065f/unit/src/main/java/org/apache/falcon/unit/FalconUnitClient.java
----------------------------------------------------------------------
diff --git a/unit/src/main/java/org/apache/falcon/unit/FalconUnitClient.java b/unit/src/main/java/org/apache/falcon/unit/FalconUnitClient.java
index 9358958..75aeba0 100644
--- a/unit/src/main/java/org/apache/falcon/unit/FalconUnitClient.java
+++ b/unit/src/main/java/org/apache/falcon/unit/FalconUnitClient.java
@@ -56,6 +56,7 @@ import org.slf4j.LoggerFactory;
import java.io.IOException;
import java.io.InputStream;
import java.io.UnsupportedEncodingException;
+import java.net.URISyntaxException;
import java.util.TreeMap;
import java.util.ArrayList;
import java.util.Properties;
@@ -330,7 +331,7 @@ public class FalconUnitClient extends AbstractFalconClient {
try {
entities = ExtensionHandler.loadAndPrepare(extensionName, jobName, configStream,
packagePath);
- } catch (FalconException | IOException e) {
+ } catch (FalconException | IOException | URISyntaxException e) {
throw new FalconCLIException("Failed in generating entities" + jobName);
}
return entities;