You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@falcon.apache.org by pe...@apache.org on 2017/02/16 09:16:02 UTC
falcon git commit: FALCON-2281 HiveDR fails with permissions denied
Repository: falcon
Updated Branches:
refs/heads/master d826852bf -> f9fdf0f7e
FALCON-2281 HiveDR fails with permissions denied
Author: Sowmya Ramesh <sr...@hortonworks.com>
Reviewers: Peeyush <pe...@apache.org>
Closes #366 from sowmyaramesh/FALCON-2281
Project: http://git-wip-us.apache.org/repos/asf/falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/falcon/commit/f9fdf0f7
Tree: http://git-wip-us.apache.org/repos/asf/falcon/tree/f9fdf0f7
Diff: http://git-wip-us.apache.org/repos/asf/falcon/diff/f9fdf0f7
Branch: refs/heads/master
Commit: f9fdf0f7e154958f9ae69cb488c395f4361d7d56
Parents: d826852
Author: Sowmya Ramesh <sr...@hortonworks.com>
Authored: Thu Feb 16 14:44:10 2017 +0530
Committer: peeyush b <pb...@hortonworks.com>
Committed: Thu Feb 16 14:44:10 2017 +0530
----------------------------------------------------------------------
.../service/SharedLibraryHostingService.java | 21 ++++++++++++++------
1 file changed, 15 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/falcon/blob/f9fdf0f7/oozie/src/main/java/org/apache/falcon/service/SharedLibraryHostingService.java
----------------------------------------------------------------------
diff --git a/oozie/src/main/java/org/apache/falcon/service/SharedLibraryHostingService.java b/oozie/src/main/java/org/apache/falcon/service/SharedLibraryHostingService.java
index b862111..9672272 100644
--- a/oozie/src/main/java/org/apache/falcon/service/SharedLibraryHostingService.java
+++ b/oozie/src/main/java/org/apache/falcon/service/SharedLibraryHostingService.java
@@ -54,8 +54,9 @@ public class SharedLibraryHostingService implements ConfigurationChangeListener
private static final String[] LIBS = StartupProperties.get().getProperty("shared.libs").split(",");
- private static class FalconLibPath implements FalconPathFilter {
+ private static class FalconLibPath implements FalconPathFilter {
private String[] shareLibs;
+
FalconLibPath(String[] libList) {
this.shareLibs = Arrays.copyOf(libList, libList.length);
}
@@ -79,7 +80,8 @@ public class SharedLibraryHostingService implements ConfigurationChangeListener
}
throw new IllegalArgumentException(path + " is not accepted!");
}
- };
+ }
+
private void pushExtensionArtifactsToCluster(final Cluster cluster,
final FileSystem clusterFs) throws FalconException {
@@ -95,6 +97,7 @@ public class SharedLibraryHostingService implements ConfigurationChangeListener
return;
}
+ final String filterPath = "/apps/falcon/extensions/mirroring/";
Path extensionStorePath = store.getExtensionStorePath();
LOG.info("extensionStorePath :{}", extensionStorePath);
FileSystem falconFileSystem =
@@ -119,6 +122,12 @@ public class SharedLibraryHostingService implements ConfigurationChangeListener
LocatedFileStatus srcfileStatus = fileStatusListIterator.next();
Path filePath = Path.getPathWithoutSchemeAndAuthority(srcfileStatus.getPath());
+ if (filePath != null && filePath.toString().startsWith(filterPath)) {
+ /* HiveDR uses filter path as store path in DRStatusStore, so skip it. Copy only the extension
+ artifacts */
+ continue;
+ }
+
if (srcfileStatus.isDirectory()) {
if (!clusterFs.exists(filePath)) {
HadoopClientFactory.mkdirs(clusterFs, filePath, srcfileStatus.getPermission());
@@ -160,13 +169,13 @@ public class SharedLibraryHostingService implements ConfigurationChangeListener
nonFalconJarFilter);
pushLibsToHDFS(fs, properties.getProperty("libext.paths"), libext, null);
pushLibsToHDFS(fs, properties.getProperty("libext.feed.paths"),
- new Path(libext, EntityType.FEED.name()) , null);
+ new Path(libext, EntityType.FEED.name()), null);
pushLibsToHDFS(fs, properties.getProperty("libext.feed.replication.paths"),
new Path(libext, EntityType.FEED.name() + "/replication"), null);
pushLibsToHDFS(fs, properties.getProperty("libext.feed.retention.paths"),
new Path(libext, EntityType.FEED.name() + "/retention"), null);
pushLibsToHDFS(fs, properties.getProperty("libext.process.paths"),
- new Path(libext, EntityType.PROCESS.name()) , null);
+ new Path(libext, EntityType.PROCESS.name()), null);
} catch (IOException e) {
throw new FalconException("Failed to copy shared libs to cluster" + cluster.getName(), e);
}
@@ -181,9 +190,9 @@ public class SharedLibraryHostingService implements ConfigurationChangeListener
LOG.debug("Copying libs from {}", src);
createTargetPath(fs, target);
- for(String srcPaths : src.split(",")) {
+ for (String srcPaths : src.split(",")) {
File srcFile = new File(srcPaths);
- File[] srcFiles = new File[] { srcFile };
+ File[] srcFiles = new File[]{srcFile};
if (srcFiles != null) {
if (srcFile.isDirectory()) {
srcFiles = srcFile.listFiles();