You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by ge...@apache.org on 2018/07/30 12:43:07 UTC
oozie git commit: OOZIE-2829 Improve sharelib upload to accept
multiple source folders (kmarton via gezapeti)
Repository: oozie
Updated Branches:
refs/heads/master 2dd10ff0f -> 3c03d03fe
OOZIE-2829 Improve sharelib upload to accept multiple source folders (kmarton via gezapeti)
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/3c03d03f
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/3c03d03f
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/3c03d03f
Branch: refs/heads/master
Commit: 3c03d03fe606659fd80c430b13f413fbdfa21faa
Parents: 2dd10ff
Author: Gezapeti Cseh <ge...@apache.org>
Authored: Mon Jul 30 14:42:51 2018 +0200
Committer: Gezapeti Cseh <ge...@apache.org>
Committed: Mon Jul 30 14:42:51 2018 +0200
----------------------------------------------------------------------
docs/src/site/twiki/AG_Install.twiki | 8 +-
release-log.txt | 1 +
.../apache/oozie/tools/OozieSharelibCLI.java | 113 +++++++--
.../tools/IntegrationTestOozieSharelibCLI.java | 199 ++++++++++++++++
.../tools/OozieSharelibFileOperations.java | 7 +-
.../tools/TestConcurrentCopyFromLocal.java | 4 +-
.../oozie/tools/TestCopyTaskCallable.java | 3 +-
.../oozie/tools/TestOozieSharelibCLI.java | 237 +++++++------------
.../TestOozieSharelibCLIExtraArgsParser.java | 66 ++++++
9 files changed, 463 insertions(+), 175 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/3c03d03f/docs/src/site/twiki/AG_Install.twiki
----------------------------------------------------------------------
diff --git a/docs/src/site/twiki/AG_Install.twiki b/docs/src/site/twiki/AG_Install.twiki
index 46363a3..b8031c8 100644
--- a/docs/src/site/twiki/AG_Install.twiki
+++ b/docs/src/site/twiki/AG_Install.twiki
@@ -47,7 +47,7 @@ The =oozie-setup.sh= script options are:
<verbatim>
Usage : oozie-setup.sh <Command and OPTIONS>
- sharelib create -fs FS_URI [-locallib SHARED_LIBRARY] [-concurrency CONCURRENCY]
+ sharelib create -fs FS_URI [-locallib SHARED_LIBRARY] [-extralib EXTRA_SHARED_LIBRARY] [-concurrency CONCURRENCY]
(create sharelib for oozie,
FS_URI is the fs.default.name
for hdfs uri; SHARED_LIBRARY, path to the
@@ -55,6 +55,12 @@ Usage : oozie-setup.sh <Command and OPTIONS>
or an expanded version of it. If omitted,
the Oozie sharelib tarball from the Oozie
installation directory will be used.
+ EXTRA_SHARED_LIBRARY represents extra sharelib resources.
+ This option requires a pair of sharelibname
+ and comma-separated list of pathnames in the following format:
+ sharelib-name=path-name-1,path-name-2
+ In case of more than one sharelib, this option can be specified
+ multiple times.
CONCURRENCY is a number of threads to be used
for copy operations.
By default 1 thread will be used)
http://git-wip-us.apache.org/repos/asf/oozie/blob/3c03d03f/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index 733ea3d..9096017 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
-- Oozie 5.1.0 release (trunk - unreleased)
+OOZIE-2829 Improve sharelib upload to accept multiple source folders (kmarton via gezapeti)
OOZIE-3309 Runtime error during /v2/sla filtering for bundle (asalamon74 via andras.piros)
OOZIE-3306 Make it possible to override maven dependency plugin version number (asalamon74 via andras.piros)
OOZIE-3208 "It should never happen" error messages should be more specific to root cause (kmarton via andras.piros)
http://git-wip-us.apache.org/repos/asf/oozie/blob/3c03d03f/tools/src/main/java/org/apache/oozie/tools/OozieSharelibCLI.java
----------------------------------------------------------------------
diff --git a/tools/src/main/java/org/apache/oozie/tools/OozieSharelibCLI.java b/tools/src/main/java/org/apache/oozie/tools/OozieSharelibCLI.java
index 75e932c..312770c 100644
--- a/tools/src/main/java/org/apache/oozie/tools/OozieSharelibCLI.java
+++ b/tools/src/main/java/org/apache/oozie/tools/OozieSharelibCLI.java
@@ -26,7 +26,9 @@ import java.util.ArrayList;
import java.util.Collection;
import java.util.Collections;
import java.util.Date;
+import java.util.HashMap;
import java.util.List;
+import java.util.Map;
import java.util.Set;
import java.util.concurrent.Callable;
import java.util.concurrent.CancellationException;
@@ -37,10 +39,12 @@ import java.util.concurrent.Future;
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Preconditions;
+import edu.umd.cs.findbugs.annotations.SuppressFBWarnings;
import org.apache.commons.cli.Option;
import org.apache.commons.cli.Options;
import org.apache.commons.cli.ParseException;
import org.apache.commons.io.FileUtils;
+import org.apache.commons.io.FilenameUtils;
import org.apache.commons.io.filefilter.WildcardFileFilter;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
@@ -64,10 +68,29 @@ public class OozieSharelibCLI {
public static final String CREATE_CMD = "create";
public static final String UPGRADE_CMD = "upgrade";
public static final String LIB_OPT = "locallib";
+ public static final String EXTRALIBS = "extralib";
public static final String FS_OPT = "fs";
public static final String CONCURRENCY_OPT = "concurrency";
public static final String OOZIE_HOME = "oozie.home.dir";
public static final String SHARE_LIB_PREFIX = "lib_";
+ public static final String NEW_LINE = System.lineSeparator();
+ public static final String EXTRALIBS_USAGE = "Extra sharelib resources. " +
+ "This option requires a pair of sharelibname and coma-separated list of pathnames" +
+ " in the following format:" + NEW_LINE +
+ "\"sharelib_name=pathname[,pathname...]\"" + NEW_LINE +
+ "Caveats:" + NEW_LINE +
+ "* Each pathname is either a directory or a regular file (compressed files are not extracted prior to " +
+ "the upload operation)." + NEW_LINE +
+ "* Sharelibname shall be specified only once." + NEW_LINE + NEW_LINE +
+ "* Do not upload multiple conflicting library versions for an extra sharelib directory as it may " +
+ "cause runtime issues." + NEW_LINE +
+ "This option can be present multiple times, in case of more than one sharelib" + NEW_LINE +
+ "Example command:" + NEW_LINE + NEW_LINE +
+ "$ oozie-setup.sh sharelib create -fs hdfs://localhost:9000 -locallib oozie-sharelib.tar.gz " +
+ "-extralib share2=dir2,file2 -extralib share3=file3";
+ public static final String EXTRALIBS_PATH_SEPARATOR = ",";
+ public static final String EXTRALIBS_SHARELIB_KEY_VALUE_SEPARATOR = "=";
+
private boolean used;
public static void main(String[] args) throws Exception{
@@ -86,9 +109,12 @@ public class OozieSharelibCLI {
options.addOption(sharelib);
options.addOption(uri);
options.addOption(concurrency);
+ Option addLibsOption = new Option(EXTRALIBS, true, EXTRALIBS_USAGE);
+ options.addOption(addLibsOption);
return options;
}
+ @SuppressFBWarnings(value = "PATH_TRAVERSAL_IN", justification = "False positive")
public synchronized int run(String[] args) throws Exception{
if (used) {
throw new IllegalStateException("CLI instance already used");
@@ -141,6 +167,12 @@ public class OozieSharelibCLI {
srcFile = files.iterator().next();
}
+ Map<String, String> extraLibs = new HashMap<>();
+ if (command.getCommandLine().hasOption(EXTRALIBS)) {
+ String[] param = command.getCommandLine().getOptionValues(EXTRALIBS);
+ extraLibs = getExtraLibs(param);
+ }
+
File temp = File.createTempFile("oozie", ".dir");
temp.delete();
temp.mkdir();
@@ -183,21 +215,9 @@ public class OozieSharelibCLI {
System.out.println("the destination path for sharelib is: " + dstPath);
- if (!srcFile.exists()){
- throw new IOException(srcPath + " cannot be found");
- }
-
- if (threadPoolSize > 1) {
- long fsLimitsMinBlockSize = fs.getConf()
- .getLong(DFSConfigKeys.DFS_NAMENODE_MIN_BLOCK_SIZE_KEY, DFSConfigKeys.DFS_NAMENODE_MIN_BLOCK_SIZE_DEFAULT);
- long bytesPerChecksum = fs.getConf()
- .getLong(DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY, DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_DEFAULT);
- new ConcurrentCopyFromLocal(threadPoolSize, fsLimitsMinBlockSize, bytesPerChecksum)
- .concurrentCopyFromLocal(fs, srcFile, dstPath);
-
- } else {
- fs.copyFromLocalFile(false, srcPath, dstPath);
- }
+ checkIfSourceFilesExist(srcFile);
+ copyToSharelib(threadPoolSize, srcFile, srcPath, dstPath, fs);
+ copyExtraLibs(threadPoolSize, extraLibs, dstPath, fs);
services.destroy();
FileUtils.deleteDirectory(temp);
@@ -220,6 +240,69 @@ public class OozieSharelibCLI {
}
}
+ @VisibleForTesting
+ static Map<String,String> getExtraLibs(String[] param) {
+ Map<String, String> extraLibs = new HashMap<>();
+
+ for (String lib : param) {
+ String[] addLibParts = lib.split(EXTRALIBS_SHARELIB_KEY_VALUE_SEPARATOR);
+ if (addLibParts.length != 2) {
+ printExtraSharelibUsage();
+ throw new IllegalArgumentException(String
+ .format("Argument of extralibs '%s' is in a wrong format. Exiting.", param));
+ }
+ String sharelibName = addLibParts[0];
+ String sharelibPaths = addLibParts[1];
+ if (extraLibs.containsKey(sharelibName)) {
+ printExtraSharelibUsage();
+ throw new IllegalArgumentException(String
+ .format("Extra sharelib, '%s', has been specified multiple times. " + "Exiting.", param));
+ }
+ extraLibs.put(sharelibName, sharelibPaths);
+ }
+ return extraLibs;
+ }
+
+ private static void printExtraSharelibUsage() {
+ System.err.println(EXTRALIBS_USAGE);
+ }
+
+
+ @VisibleForTesting
+ @SuppressFBWarnings(value = "PATH_TRAVERSAL_IN", justification = "FilenameUtils is used to filter user input. JDK8+ is used.")
+ void copyExtraLibs(int threadPoolSize, Map<String, String> extraLibs, Path dstPath, FileSystem fs) throws IOException {
+ for (Map.Entry<String, String> sharelib : extraLibs.entrySet()) {
+ Path libDestPath = new Path(dstPath.toString() + Path.SEPARATOR + sharelib.getKey());
+ for (String libPath : sharelib.getValue().split(EXTRALIBS_PATH_SEPARATOR)) {
+ File srcFile = new File(FilenameUtils.getFullPath(libPath) + FilenameUtils.getName(libPath));
+ Path srcPath = new Path(FilenameUtils.getFullPath(libPath) + FilenameUtils.getName(libPath));
+ checkIfSourceFilesExist(srcFile);
+ copyToSharelib(threadPoolSize, srcFile, srcPath, libDestPath, fs);
+ }
+ }
+ }
+
+ @VisibleForTesting
+ protected void copyToSharelib(int threadPoolSize, File srcFile, Path srcPath, Path dstPath, FileSystem fs) throws IOException {
+ if (threadPoolSize > 1) {
+ long fsLimitsMinBlockSize = fs.getConf()
+ .getLong(DFSConfigKeys.DFS_NAMENODE_MIN_BLOCK_SIZE_KEY, DFSConfigKeys.DFS_NAMENODE_MIN_BLOCK_SIZE_DEFAULT);
+ long bytesPerChecksum = fs.getConf()
+ .getLong(DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_KEY, DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_DEFAULT);
+ new ConcurrentCopyFromLocal(threadPoolSize, fsLimitsMinBlockSize, bytesPerChecksum)
+ .concurrentCopyFromLocal(fs, srcFile, dstPath);
+
+ } else {
+ fs.copyFromLocalFile(false, srcPath, dstPath);
+ }
+ }
+
+ @VisibleForTesting
+ protected void checkIfSourceFilesExist(File srcFile) throws IOException {
+ if (!srcFile.exists()){
+ throw new IOException(srcFile + " cannot be found");
+ }
+ }
private static void logError(String errorMessage, Throwable ex) {
http://git-wip-us.apache.org/repos/asf/oozie/blob/3c03d03f/tools/src/test/java/org/apache/oozie/tools/IntegrationTestOozieSharelibCLI.java
----------------------------------------------------------------------
diff --git a/tools/src/test/java/org/apache/oozie/tools/IntegrationTestOozieSharelibCLI.java b/tools/src/test/java/org/apache/oozie/tools/IntegrationTestOozieSharelibCLI.java
new file mode 100644
index 0000000..12cb665
--- /dev/null
+++ b/tools/src/test/java/org/apache/oozie/tools/IntegrationTestOozieSharelibCLI.java
@@ -0,0 +1,199 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.oozie.tools;
+
+import org.apache.commons.io.IOUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.oozie.action.hadoop.security.LauncherSecurityManager;
+import org.apache.oozie.service.ConfigurationService;
+import org.apache.oozie.service.HadoopAccessorService;
+import org.apache.oozie.service.ServiceException;
+import org.apache.oozie.service.Services;
+import org.apache.oozie.service.ShareLibService;
+import org.apache.oozie.service.WorkflowAppService;
+import org.apache.oozie.test.XTestCase;
+import org.junit.rules.TemporaryFolder;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.InputStream;
+import java.net.URI;
+import java.util.List;
+
+import static org.apache.oozie.tools.OozieSharelibCLI.EXTRALIBS_SHARELIB_KEY_VALUE_SEPARATOR;
+
+
+public class IntegrationTestOozieSharelibCLI extends XTestCase {
+
+ private final TemporaryFolder tmpFolder = new TemporaryFolder();
+ private File libDirectory;
+ private Services services = null;
+ private Path dstPath = null;
+ private FileSystem fs;
+ private LauncherSecurityManager launcherSecurityManager;
+ public final String outPath = "outFolder";
+
+ @Override
+ protected void setUp() throws Exception {
+ launcherSecurityManager = new LauncherSecurityManager();
+ launcherSecurityManager.enable();
+ tmpFolder.create();
+ libDirectory = tmpFolder.newFolder("lib");
+ super.setUp(false);
+ }
+
+ @Override
+ protected void tearDown() throws Exception {
+ launcherSecurityManager.disable();
+ if (services != null) {
+ services.destroy();
+ }
+ super.tearDown();
+ }
+
+
+ /**
+ * test copy libraries
+ */
+ public void testOozieSharelibCLICreate() throws Exception {
+
+ final int fileNr = 2;
+ List<File> sharelibFiles = OozieSharelibFileOperations.generateAndWriteFiles(libDirectory, fileNr);
+
+ String[] argsCreate = { "create", "-fs", outPath, "-locallib", libDirectory.getParentFile().getAbsolutePath() };
+ assertEquals("Exit code mismatch", 0, execOozieSharelibCLICommands(argsCreate));
+
+ ShareLibService sharelibService = getServices().get(ShareLibService.class);
+ Path latestLibPath = sharelibService.getLatestLibPath(getDistPath(),
+ ShareLibService.SHARE_LIB_PREFIX);
+
+ checkCopiedSharelibFiles(sharelibFiles, latestLibPath);
+ }
+
+ /**
+ * test parallel copy libraries
+ */
+ public void testOozieSharelibCLICreateConcurrent() throws Exception {
+
+ final int testFiles = 7;
+ final int concurrency = 5;
+
+ List<File> sharelibFiles = OozieSharelibFileOperations.generateAndWriteFiles(libDirectory, testFiles);
+ String[] argsCreate = {"create", "-fs", outPath, "-locallib", libDirectory.getParentFile().getAbsolutePath(),
+ "-concurrency", String.valueOf(concurrency)};
+ assertEquals("Exit code mismatch",0, execOozieSharelibCLICommands(argsCreate));
+
+ getTargetFileSysyem();
+ ShareLibService sharelibService = getServices().get(ShareLibService.class);
+ Path latestLibPath = sharelibService.getLatestLibPath(getDistPath(),
+ ShareLibService.SHARE_LIB_PREFIX);
+
+ checkCopiedSharelibFiles(sharelibFiles, latestLibPath);
+ }
+
+ public void testOozieSharelibCreateExtraLibs () throws Exception {
+ File extraLibBirectory1 = tmpFolder.newFolder("extralib1");
+ File extraLibBirectory2 = tmpFolder.newFolder("extralib2");
+ final int sharelibFileNr = 3;
+ final int extraSharelib1FileNr = 4;
+ final int extraSharelib2FileNr = 4;
+ List<File> shareLibFiles = OozieSharelibFileOperations.generateAndWriteFiles(libDirectory, sharelibFileNr);
+ List<File> extraShareLibFiles1 = OozieSharelibFileOperations
+ .generateAndWriteFiles(extraLibBirectory1, extraSharelib1FileNr);
+ List<File> extraShareLibFiles2 = OozieSharelibFileOperations
+ .generateAndWriteFiles(extraLibBirectory2, extraSharelib2FileNr);
+
+ String extraLib1 = extraLibBirectory1.getName() + EXTRALIBS_SHARELIB_KEY_VALUE_SEPARATOR
+ + extraLibBirectory1.getAbsolutePath();
+ String extraLib2 = extraLibBirectory2.getName() + EXTRALIBS_SHARELIB_KEY_VALUE_SEPARATOR
+ + extraLibBirectory2.getAbsolutePath();
+ String[] argsCreate = { "create", "-fs", outPath, "-locallib", libDirectory.getParentFile().getAbsolutePath(),
+ "-" + OozieSharelibCLI.EXTRALIBS, extraLib1, "-" + OozieSharelibCLI.EXTRALIBS, extraLib2};
+
+ assertEquals("Exit code mismatch",0, execOozieSharelibCLICommands(argsCreate));
+
+ ShareLibService sharelibService = getServices().get(ShareLibService.class);
+ Path latestLibPath = sharelibService.getLatestLibPath(getDistPath(), ShareLibService.SHARE_LIB_PREFIX);
+ Path extraSharelibPath1 = new Path(latestLibPath + Path.SEPARATOR + extraLibBirectory1.getName());
+ Path extraSharelibPath2 = new Path(latestLibPath + Path.SEPARATOR + extraLibBirectory2.getName());
+
+ checkCopiedSharelibFiles(shareLibFiles, latestLibPath);
+ checkCopiedSharelibFiles(extraShareLibFiles1, extraSharelibPath1);
+ checkCopiedSharelibFiles(extraShareLibFiles2, extraSharelibPath2);
+ }
+
+ private void checkCopiedSharelibFiles(List<File> fileList, Path libPath) throws Exception {
+ FileSystem fileSystem = getTargetFileSysyem();
+ for (File f: fileList) {
+ try (InputStream originalFileStream = new FileInputStream(f);
+ InputStream copiedFileStream = fileSystem.open(new Path(libPath, f.getName()))) {
+ assertTrue("The content of the files must be equal", IOUtils.contentEquals(originalFileStream, copiedFileStream));
+ }
+ }
+ }
+
+ private FileSystem getTargetFileSysyem() throws Exception {
+ if (fs == null) {
+ HadoopAccessorService has = getServices().get(HadoopAccessorService.class);
+ URI uri = new Path(outPath).toUri();
+ Configuration fsConf = has.createConfiguration(uri.getAuthority());
+ fs = has.createFileSystem(System.getProperty("user.name"), uri, fsConf);
+ }
+ return fs;
+
+ }
+
+ private Services getServices() throws ServiceException {
+ if (services == null) {
+ services = new Services();
+ services.get(ConfigurationService.class).getConf()
+ .set(Services.CONF_SERVICE_CLASSES,"org.apache.oozie.service.LiteWorkflowAppService,"
+ + "org.apache.oozie.service.SchedulerService,"
+ + "org.apache.oozie.service.HadoopAccessorService,"
+ + "org.apache.oozie.service.ShareLibService");
+ services.init();
+ }
+ return services;
+ }
+
+ private Path getDistPath() throws Exception {
+ if (dstPath == null) {
+ WorkflowAppService lwas = getServices().get(WorkflowAppService.class);
+ dstPath = lwas.getSystemLibPath();
+ }
+ return dstPath;
+ }
+
+ private int execOozieSharelibCLICommands(String[] args) throws Exception {
+ try {
+ OozieSharelibCLI.main(args);
+ } catch (SecurityException ex) {
+ if (launcherSecurityManager.getExitInvoked()) {
+ System.out.println("Intercepting System.exit(" + launcherSecurityManager.getExitCode() + ")");
+ System.err.println("Intercepting System.exit(" + launcherSecurityManager.getExitCode() + ")");
+ return launcherSecurityManager.getExitCode();
+ } else {
+ throw ex;
+ }
+ }
+ return 1;
+ }
+}
http://git-wip-us.apache.org/repos/asf/oozie/blob/3c03d03f/tools/src/test/java/org/apache/oozie/tools/OozieSharelibFileOperations.java
----------------------------------------------------------------------
diff --git a/tools/src/test/java/org/apache/oozie/tools/OozieSharelibFileOperations.java b/tools/src/test/java/org/apache/oozie/tools/OozieSharelibFileOperations.java
index d344300..ca31e96 100644
--- a/tools/src/test/java/org/apache/oozie/tools/OozieSharelibFileOperations.java
+++ b/tools/src/test/java/org/apache/oozie/tools/OozieSharelibFileOperations.java
@@ -22,6 +22,7 @@ import java.io.File;
import java.io.FileWriter;
import java.io.IOException;
import java.io.Writer;
+import java.util.ArrayList;
import java.util.List;
public final class OozieSharelibFileOperations {
@@ -36,15 +37,17 @@ public final class OozieSharelibFileOperations {
/**
* generate a number of files equals with fileNr, and save the fileList parameter
* @param fileNr number of files to be generated
- * @param fileList a list of the generated files
+ * @return a list of the generated files
* @throws Exception
*/
- public static void generateAndWriteFiles(File libDirectory, int fileNr, List<File> fileList) throws IOException {
+ public static List<File> generateAndWriteFiles(File libDirectory, int fileNr) throws IOException {
+ List<File> fileList = new ArrayList<>();
for (int i=0; i<fileNr; i++) {
String fileName = generateFileName(i);
String fileContent = generateFileContent(i);
fileList.add(writeFile(libDirectory, fileName, fileContent));
}
+ return fileList;
}
/**
http://git-wip-us.apache.org/repos/asf/oozie/blob/3c03d03f/tools/src/test/java/org/apache/oozie/tools/TestConcurrentCopyFromLocal.java
----------------------------------------------------------------------
diff --git a/tools/src/test/java/org/apache/oozie/tools/TestConcurrentCopyFromLocal.java b/tools/src/test/java/org/apache/oozie/tools/TestConcurrentCopyFromLocal.java
index d77eba6..51cf93d 100644
--- a/tools/src/test/java/org/apache/oozie/tools/TestConcurrentCopyFromLocal.java
+++ b/tools/src/test/java/org/apache/oozie/tools/TestConcurrentCopyFromLocal.java
@@ -100,9 +100,7 @@ public class TestConcurrentCopyFromLocal extends XTestCase {
private void performAndCheckConcurrentCopy(final int testFiles, final int threadPoolSize, final long fsLimitsMinBlockSize,
final long bytesPerChecksum) throws Exception {
- List<File> fileList = new ArrayList<>();
-
- OozieSharelibFileOperations.generateAndWriteFiles(libDirectory, testFiles, fileList);
+ List<File> fileList = OozieSharelibFileOperations.generateAndWriteFiles(libDirectory, testFiles);
File srcFile = new File(libDirectory.getParentFile().getAbsolutePath());
OozieSharelibCLI.ConcurrentCopyFromLocal concurrentCopy = new OozieSharelibCLI
.ConcurrentCopyFromLocal(threadPoolSize, fsLimitsMinBlockSize, bytesPerChecksum);
http://git-wip-us.apache.org/repos/asf/oozie/blob/3c03d03f/tools/src/test/java/org/apache/oozie/tools/TestCopyTaskCallable.java
----------------------------------------------------------------------
diff --git a/tools/src/test/java/org/apache/oozie/tools/TestCopyTaskCallable.java b/tools/src/test/java/org/apache/oozie/tools/TestCopyTaskCallable.java
index bce0433..02f937a 100644
--- a/tools/src/test/java/org/apache/oozie/tools/TestCopyTaskCallable.java
+++ b/tools/src/test/java/org/apache/oozie/tools/TestCopyTaskCallable.java
@@ -110,8 +110,7 @@ public class TestCopyTaskCallable extends XTestCase {
private void performAndCheckCallCopyTask(final long blockSize, final int poolSize, final int testFiles) throws Exception {
Set<OozieSharelibCLI.CopyTaskConfiguration> failedCopyTasks = new ConcurrentHashSet<>();
- List<File> fileList = new ArrayList<>();
- OozieSharelibFileOperations.generateAndWriteFiles(libDirectory, testFiles, fileList);
+ List<File> fileList = OozieSharelibFileOperations.generateAndWriteFiles(libDirectory, testFiles);
File srcFile = new File(libDirectory.getParentFile().getAbsolutePath());
http://git-wip-us.apache.org/repos/asf/oozie/blob/3c03d03f/tools/src/test/java/org/apache/oozie/tools/TestOozieSharelibCLI.java
----------------------------------------------------------------------
diff --git a/tools/src/test/java/org/apache/oozie/tools/TestOozieSharelibCLI.java b/tools/src/test/java/org/apache/oozie/tools/TestOozieSharelibCLI.java
index 5929e5c..18ab8f2 100644
--- a/tools/src/test/java/org/apache/oozie/tools/TestOozieSharelibCLI.java
+++ b/tools/src/test/java/org/apache/oozie/tools/TestOozieSharelibCLI.java
@@ -19,206 +19,139 @@
package org.apache.oozie.tools;
-import org.apache.commons.io.IOUtils;
-import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.oozie.action.hadoop.security.LauncherSecurityManager;
-import org.apache.oozie.service.ConfigurationService;
-import org.apache.oozie.service.HadoopAccessorService;
-import org.apache.oozie.service.ServiceException;
-import org.apache.oozie.service.Services;
-import org.apache.oozie.service.ShareLibService;
-import org.apache.oozie.service.WorkflowAppService;
-import org.apache.oozie.test.XTestCase;
-import org.junit.rules.TemporaryFolder;
+import org.junit.After;
+import org.junit.Assert;
+import org.junit.Before;
+import org.junit.Test;
import java.io.ByteArrayOutputStream;
import java.io.File;
-import java.io.InputStream;
+import java.io.IOException;
import java.io.PrintStream;
-import java.net.URI;
+import java.util.ArrayList;
+import java.util.Arrays;
+import java.util.List;
+import java.util.Map;
+
+import static org.apache.oozie.tools.OozieSharelibCLI.getExtraLibs;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+import static org.mockito.Matchers.any;
+import static org.mockito.Matchers.anyInt;
+import static org.mockito.Matchers.anyMap;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.spy;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
/**
* Test OozieSharelibCLI
*/
-public class TestOozieSharelibCLI extends XTestCase {
- public final String outPath = "outFolder";
- private final TemporaryFolder tmpFolder = new TemporaryFolder();
- private File libDirectory;
- private Services services = null;
- private Path dstPath = null;
- private FileSystem fs;
+public class TestOozieSharelibCLI {
+
+ private final static String TEST_SHAERELIBNAME1 = "sharelibName";
+ private final static String TEST_SHAERELIBNAME2 = "sharelibName2";
+ private final static String TEST_EXTRALIBS_PATHS1 = "/path/to/source/,/path/to/some/file";
+ private final static String TEST_EXTRALIBS_PATHS2 = "hdfs://my/jar.jar#myjar.jar";
+
private LauncherSecurityManager launcherSecurityManager;
- @Override
- protected void setUp() throws Exception {
+
+ @Before
+ public void setUp() {
launcherSecurityManager = new LauncherSecurityManager();
launcherSecurityManager.enable();
- tmpFolder.create();
- libDirectory = tmpFolder.newFolder("lib");
- super.setUp(false);
-
}
- @Override
- protected void tearDown() throws Exception {
+ @After
+ public void tearDown() {
launcherSecurityManager.disable();
- if (services != null) {
- services.destroy();
- }
- super.tearDown();
}
- /**
- * Test help command
- */
- public void testHelp() throws Exception {
+ @Test
+ public void testHelpCommand() throws Exception {
ByteArrayOutputStream data = new ByteArrayOutputStream();
PrintStream oldPrintStream = System.out;
System.setOut(new PrintStream(data));
try {
- String[] argsHelp = { "help" };
- assertEquals(0, execOozieSharelibCLICommands(argsHelp));
+ String[] argsHelp = {"help"};
+ assertEquals("Exit code mismatch",0, execOozieSharelibCLICommands(argsHelp));
String helpMessage = data.toString();
- assertTrue(helpMessage.contains(
+ assertTrue("Missing create <OPTIONS> description from help message", helpMessage.contains(
"oozie-setup.sh create <OPTIONS> : create a new timestamped version of oozie sharelib"));
- assertTrue(helpMessage.contains(
- "oozie-setup.sh upgrade <OPTIONS> : [deprecated][use command \"create\" "
- + "to create new version] upgrade oozie sharelib "));
- assertTrue(helpMessage.contains(" oozie-setup.sh help"));
- assertTrue(helpMessage.contains(
- "-concurrency <arg> Number of threads to be used for copy operations."));
- }
- finally {
+ assertTrue("Missing upgrade <OPTIONS> description from help message",
+ helpMessage.contains("oozie-setup.sh upgrade <OPTIONS> : [deprecated][use command \"create\" "
+ + "to create new version] upgrade oozie sharelib "));
+ assertTrue("Help message mismatch", helpMessage.contains(" oozie-setup.sh help"));
+ assertTrue("Missing -concurrency from help message",
+ helpMessage.contains("-concurrency <arg> Number of threads to be used for copy operations."));
+ } finally {
System.setOut(oldPrintStream);
}
-
- }
-
- /**
- * test copy libraries
- */
- public void testOozieSharelibCLICreate() throws Exception {
-
- OozieSharelibFileOperations.writeFile(libDirectory, "file1", "test File");
- OozieSharelibFileOperations.writeFile(libDirectory, "file2", "test File2");
-
- String[] argsCreate = { "create", "-fs", outPath, "-locallib", libDirectory.getParentFile().getAbsolutePath() };
- assertEquals(0, execOozieSharelibCLICommands(argsCreate));
-
- FileSystem fs = getTargetFileSysyem();
- ShareLibService sharelibService = getServices().get(ShareLibService.class);
-
- // test files in new folder
- assertEquals(9, fs.getFileStatus(new Path(sharelibService.getLatestLibPath(getDistPath(),
- ShareLibService.SHARE_LIB_PREFIX), "file1")).getLen());
- assertEquals(10, fs.getFileStatus(new Path(sharelibService.getLatestLibPath(getDistPath(),
- ShareLibService.SHARE_LIB_PREFIX), "file2")).getLen());
-
- }
-
- /**
- * test parallel copy libraries
- */
- public void testOozieSharelibCLICreateConcurrent() throws Exception {
-
- final int testFiles = 7;
- final int concurrency = 5;
-
- for (int i = 0; i < testFiles; i++) {
- OozieSharelibFileOperations.writeFile(libDirectory, OozieSharelibFileOperations.generateFileName(i),
- OozieSharelibFileOperations.generateFileContent(i));
- }
-
- String[] argsCreate = {"create", "-fs", outPath, "-locallib", libDirectory.getParentFile().getAbsolutePath(),
- "-concurrency", String.valueOf(concurrency)};
- assertEquals(0, execOozieSharelibCLICommands(argsCreate));
-
- getTargetFileSysyem();
- ShareLibService sharelibService = getServices().get(ShareLibService.class);
- Path latestLibPath = sharelibService.getLatestLibPath(getDistPath(),
- ShareLibService.SHARE_LIB_PREFIX);
-
- // test files in new folder
-
- for (int i = 0; i < testFiles; i++) {
- String fileName = OozieSharelibFileOperations.generateFileName(i);
- String expectedFileContent = OozieSharelibFileOperations.generateFileContent(i);
- InputStream in = null;
- try {
- in = getTargetFileSysyem().open(new Path(latestLibPath, fileName));
- String actualFileContent = IOUtils.toString(in);
- assertEquals(fileName, expectedFileContent, actualFileContent);
- } finally {
- IOUtils.closeQuietly(in);
- }
- }
-
}
- /**
- * test fake command
- */
+ @Test
public void testFakeCommand() throws Exception {
ByteArrayOutputStream data = new ByteArrayOutputStream();
PrintStream oldPrintStream = System.err;
System.setErr(new PrintStream(data));
try {
- String[] argsFake = { "fakeCommand" };
- assertEquals(1, execOozieSharelibCLICommands(argsFake));
- assertTrue(data.toString().contains("Invalid sub-command: invalid sub-command [fakeCommand]"));
- assertTrue(data.toString().contains("use 'help [sub-command]' for help details"));
- }
- finally {
+ String[] argsFake = {"fakeCommand"};
+ assertEquals("Exit code mismatch", 1, execOozieSharelibCLICommands(argsFake));
+ assertTrue("Error message missing",
+ data.toString().contains("Invalid sub-command: invalid sub-command [fakeCommand]"));
+ assertTrue("Help message missing", data.toString().contains("use 'help [sub-command]' for help details"));
+ } finally {
System.setErr(oldPrintStream);
}
-
- }
-
- private FileSystem getTargetFileSysyem() throws Exception {
- if (fs == null) {
- HadoopAccessorService has = getServices().get(HadoopAccessorService.class);
- URI uri = new Path(outPath).toUri();
- Configuration fsConf = has.createConfiguration(uri.getAuthority());
- fs = has.createFileSystem(System.getProperty("user.name"), uri, fsConf);
- }
- return fs;
-
- }
-
- private Services getServices() throws ServiceException {
- if (services == null) {
- services = new Services();
- services.get(ConfigurationService.class).getConf()
- .set(Services.CONF_SERVICE_CLASSES,"org.apache.oozie.service.LiteWorkflowAppService,"
- + "org.apache.oozie.service.SchedulerService,"
- + "org.apache.oozie.service.HadoopAccessorService,"
- + "org.apache.oozie.service.ShareLibService");
- services.init();
- }
- return services;
}
- private Path getDistPath() throws Exception {
- if (dstPath == null) {
- WorkflowAppService lwas = getServices().get(WorkflowAppService.class);
- dstPath = lwas.getSystemLibPath();
- }
- return dstPath;
+ @Test
+ public void testCopyingExtraSharelibs() throws IOException {
+ OozieSharelibCLI oozieSharelibCLI = spy(new OozieSharelibCLI() {
+ @Override
+ protected void checkIfSourceFilesExist(File srcFile) { }
+
+ @Override
+ protected void copyToSharelib(int threadPoolSize, File srcFile, Path srcPath, Path dstPath, FileSystem fs) {
+ String dstStr = dstPath.toString();
+ String actualSharelibName = dstStr.substring(dstStr.lastIndexOf(Path.SEPARATOR) + 1);
+ Assert.assertTrue("Expected sharelib name missing",
+ Arrays.asList(TEST_SHAERELIBNAME1, TEST_SHAERELIBNAME2).contains(actualSharelibName));
+
+ List<File> expectedTestFiles = new ArrayList<>();
+ for (String s : Arrays.asList(TEST_EXTRALIBS_PATHS1.split(","))) {
+ expectedTestFiles.add(new File(s));
+ }
+ for (String s : Arrays.asList(TEST_EXTRALIBS_PATHS2.split(","))) {
+ expectedTestFiles.add(new File(s));
+ }
+ Assert.assertTrue("Expected file missing", expectedTestFiles.contains(srcFile));
+ }
+ });
+
+ final FileSystem fs = mock(FileSystem.class);
+ final Path dst = new Path("/share/lib");
+ String extraLibOption1 = TEST_SHAERELIBNAME1 + "=" + TEST_EXTRALIBS_PATHS1;
+ String extraLibOption2 = TEST_SHAERELIBNAME2 + "=" + TEST_EXTRALIBS_PATHS2;
+ Map<String, String> addLibs = getExtraLibs(new String[] {extraLibOption1, extraLibOption2});
+ oozieSharelibCLI.copyExtraLibs(1, addLibs, dst, fs);
+ verify(oozieSharelibCLI, times(1))
+ .copyExtraLibs(anyInt(), anyMap(), any(Path.class), any(FileSystem.class));
}
private int execOozieSharelibCLICommands(String[] args) throws Exception {
try {
OozieSharelibCLI.main(args);
- }
- catch (SecurityException ex) {
+ } catch (SecurityException ex) {
if (launcherSecurityManager.getExitInvoked()) {
System.out.println("Intercepting System.exit(" + launcherSecurityManager.getExitCode() + ")");
System.err.println("Intercepting System.exit(" + launcherSecurityManager.getExitCode() + ")");
return launcherSecurityManager.getExitCode();
- }
- else {
+ } else {
throw ex;
}
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/3c03d03f/tools/src/test/java/org/apache/oozie/tools/TestOozieSharelibCLIExtraArgsParser.java
----------------------------------------------------------------------
diff --git a/tools/src/test/java/org/apache/oozie/tools/TestOozieSharelibCLIExtraArgsParser.java b/tools/src/test/java/org/apache/oozie/tools/TestOozieSharelibCLIExtraArgsParser.java
new file mode 100644
index 0000000..f18db4d
--- /dev/null
+++ b/tools/src/test/java/org/apache/oozie/tools/TestOozieSharelibCLIExtraArgsParser.java
@@ -0,0 +1,66 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.oozie.tools;
+
+import org.junit.Assert;
+import org.junit.Test;
+
+import java.util.Map;
+
+import static org.apache.oozie.tools.OozieSharelibCLI.getExtraLibs;
+
+public class TestOozieSharelibCLIExtraArgsParser {
+
+ private final static String TEST_SHAERELIBNAME1 = "sharelibName";
+ private final static String TEST_SHAERELIBNAME2 = "sharelibName2";
+ private final static String TEST_EXTRALIBS_PATHS1 = "/path/to/source/,/path/to/some/file";
+ private final static String TEST_EXTRALIBS_PATHS2 = "hdfs://my/jar.jar#myjar.jar,path/to/source/";
+
+ @Test
+ public void testParsingExtraSharelibs() {
+ String extraLibOption1 = TEST_SHAERELIBNAME1 + "=" + TEST_EXTRALIBS_PATHS1;
+ String extraLibOption2 = TEST_SHAERELIBNAME2 + "=" + TEST_EXTRALIBS_PATHS2;
+ Map<String, String> addLibs = getExtraLibs(new String[] {extraLibOption1, extraLibOption2});
+
+ Assert.assertEquals("Extra libs count mismatch", 2, addLibs.size());
+ Assert.assertEquals("Missing extra lib", TEST_EXTRALIBS_PATHS1, addLibs.get(TEST_SHAERELIBNAME1));
+ Assert.assertEquals("Missing extra lib", TEST_EXTRALIBS_PATHS2, addLibs.get(TEST_SHAERELIBNAME2));
+ }
+
+ @Test (expected = IllegalArgumentException.class)
+ public void testParsingExtraSharelibsMissingValue() {
+ String extraLibOption1 = TEST_SHAERELIBNAME1 + "=";
+ String extraLibOption2 = TEST_SHAERELIBNAME2 + "=" + TEST_EXTRALIBS_PATHS2;
+ getExtraLibs(new String[] {extraLibOption1, extraLibOption2});
+ }
+
+ @Test (expected = IllegalArgumentException.class)
+ public void testParsingExtraSharelibsMissingKey() {
+ String extraLibOption1 = TEST_EXTRALIBS_PATHS1;
+ String extraLibOption2 = TEST_SHAERELIBNAME2 + "=" + TEST_EXTRALIBS_PATHS2;
+ getExtraLibs(new String[] {extraLibOption1, extraLibOption2});
+ }
+
+ @Test(expected = IllegalArgumentException.class)
+ public void testParsingSharelibNamePresentMultipleTimes() {
+ String extraLibOption1 = TEST_SHAERELIBNAME1 + "=" + TEST_EXTRALIBS_PATHS1;
+ String extraLibOption2 = TEST_SHAERELIBNAME1 + "=" + TEST_EXTRALIBS_PATHS2;
+ getExtraLibs(new String[] {extraLibOption1, extraLibOption2});
+ }
+}
\ No newline at end of file