You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@falcon.apache.org by sa...@apache.org on 2014/08/28 12:49:12 UTC
git commit: FALCON-572 HadoopUtil cleanup in falcon-regression
contributed by Ruslan Ostafiychuk
Repository: incubator-falcon
Updated Branches:
refs/heads/master b1c13df45 -> e21ec9367
FALCON-572 HadoopUtil cleanup in falcon-regression contributed by Ruslan Ostafiychuk
Project: http://git-wip-us.apache.org/repos/asf/incubator-falcon/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-falcon/commit/e21ec936
Tree: http://git-wip-us.apache.org/repos/asf/incubator-falcon/tree/e21ec936
Diff: http://git-wip-us.apache.org/repos/asf/incubator-falcon/diff/e21ec936
Branch: refs/heads/master
Commit: e21ec9367b1a4c8b0a359acee6c35c44569180fb
Parents: b1c13df
Author: Samarth Gupta <sa...@inmobi.com>
Authored: Thu Aug 28 16:18:43 2014 +0530
Committer: Samarth Gupta <sa...@inmobi.com>
Committed: Thu Aug 28 16:18:43 2014 +0530
----------------------------------------------------------------------
falcon-regression/CHANGES.txt | 1 +
.../falcon/regression/core/util/HadoopUtil.java | 66 ++------------------
.../falcon/regression/prism/RetentionTest.java | 9 ++-
3 files changed, 15 insertions(+), 61 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/e21ec936/falcon-regression/CHANGES.txt
----------------------------------------------------------------------
diff --git a/falcon-regression/CHANGES.txt b/falcon-regression/CHANGES.txt
index 52e697e..ad6ef79 100644
--- a/falcon-regression/CHANGES.txt
+++ b/falcon-regression/CHANGES.txt
@@ -10,6 +10,7 @@ Trunk (Unreleased)
IMPROVEMENTS
+ FALCON-572 HadoopUtil cleanup in falcon-regression (Ruslan Ostafiychuk via Samarth Gupta)
FALCON-632 Refactoring, documentation stuff (Paul Isaychuk via Samarth Gupta)
FALCON-609 UpdateAtSpecificTimeTest, InstanceSummaryTest tagged, fixed, refactored
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/e21ec936/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/HadoopUtil.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/HadoopUtil.java b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/HadoopUtil.java
index dc0a4d4..af5c01a 100644
--- a/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/HadoopUtil.java
+++ b/falcon-regression/merlin-core/src/main/java/org/apache/falcon/regression/core/util/HadoopUtil.java
@@ -260,31 +260,6 @@ public final class HadoopUtil {
return locations;
}
- public static void createLateDataFoldersWithRandom(FileSystem fs, String folderPrefix,
- List<String> folderList) throws IOException {
- LOGGER.info("creating late data folders.....");
- folderList.add(SOMETHING_RANDOM);
-
- for (final String folder : folderList) {
- fs.mkdirs(new Path(folderPrefix + folder));
- }
-
- LOGGER.info("created all late data folders.....");
- }
-
- public static void copyDataToFolders(FileSystem fs, List<String> folderList,
- String directory, String folderPrefix) throws IOException {
- LOGGER.info("copying data into folders....");
- List<String> fileLocations = new ArrayList<String>();
- File[] files = new File(directory).listFiles();
- if (files != null) {
- for (final File file : files) {
- fileLocations.add(file.toString());
- }
- }
- copyDataToFolders(fs, folderPrefix, folderList,
- fileLocations.toArray(new String[fileLocations.size()]));
- }
public static void copyDataToFolders(FileSystem fs, final String folderPrefix,
List<String> folderList, String... fileLocations) throws IOException {
@@ -321,12 +296,11 @@ public final class HadoopUtil {
public static void lateDataReplenish(FileSystem fs, int interval,
int minuteSkip, String folderPrefix) throws IOException {
List<String> folderData = TimeUtil.getMinuteDatesOnEitherSide(interval, minuteSkip);
-
- createLateDataFoldersWithRandom(fs, folderPrefix, folderData);
- copyDataToFolders(fs, folderData, OSUtil.NORMAL_INPUT, folderPrefix);
+ folderData.add(SOMETHING_RANDOM);
+ flattenAndPutDataInFolder(fs, OSUtil.NORMAL_INPUT, folderPrefix, folderData);
}
- public static void createLateDataFolders(FileSystem fs, final String folderPrefix,
+ public static void createFolders(FileSystem fs, final String folderPrefix,
List<String> folderList) throws IOException {
for (final String folder : folderList) {
fs.mkdirs(new Path(folderPrefix + folder));
@@ -353,7 +327,7 @@ public final class HadoopUtil {
List<String> folderPaths = TimeUtil.getMinuteDatesOnEitherSide(interval, minuteSkip);
LOGGER.info("folderData: " + folderPaths.toString());
- createLateDataFolders(fs, folderPrefix, folderPaths);
+ createFolders(fs, folderPrefix, folderPaths);
if (fileToBePut.equals("_SUCCESS")) {
copyDataToFolders(fs, folderPrefix, folderPaths, OSUtil.NORMAL_INPUT + "_SUCCESS");
@@ -374,7 +348,7 @@ public final class HadoopUtil {
}
}
- createLateDataFolders(fs, folderPrefix, folderPaths);
+ createFolders(fs, folderPrefix, folderPaths);
copyDataToFolders(fs, folderPrefix, folderPaths,
OSUtil.NORMAL_INPUT + "log_01.txt");
}
@@ -390,36 +364,8 @@ public final class HadoopUtil {
}
}
- createLateDataFolders(fs, folderPrefix, folderPaths);
+ createFolders(fs, folderPrefix, folderPaths);
copyDataToFolders(fs, folderPrefix, folderPaths,
OSUtil.NORMAL_INPUT + "_SUCCESS", OSUtil.NORMAL_INPUT + "log_01.txt");
}
-
- /**
- * Removes general folder on file system. Creates folders according to passed folders list
- * and fills them with data if required.
- *
- * @param fileSystem destination file system
- * @param prefix prefix of path where data should be placed
- * @param folderList list of folders to be created and filled with data if required
- * @param uploadData if folders should be filled with data
- * @throws IOException
- */
- public static void replenishData(FileSystem fileSystem, String prefix, List<String> folderList,
- boolean uploadData) throws IOException {
- //purge data first
- deleteDirIfExists(prefix, fileSystem);
-
- folderList.add(SOMETHING_RANDOM);
-
- for (final String folder : folderList) {
- final String pathString = prefix + folder;
- LOGGER.info(fileSystem.getUri() + pathString);
- fileSystem.mkdirs(new Path(pathString));
- if (uploadData) {
- fileSystem.copyFromLocalFile(new Path(OSUtil.RESOURCES + "log_01.txt"),
- new Path(pathString));
- }
- }
- }
}
http://git-wip-us.apache.org/repos/asf/incubator-falcon/blob/e21ec936/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/RetentionTest.java
----------------------------------------------------------------------
diff --git a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/RetentionTest.java b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/RetentionTest.java
index 1d900d9..991002a 100644
--- a/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/RetentionTest.java
+++ b/falcon-regression/merlin/src/test/java/org/apache/falcon/regression/prism/RetentionTest.java
@@ -30,6 +30,7 @@ import org.apache.falcon.regression.core.supportClasses.JmsMessageConsumer;
import org.apache.falcon.regression.core.util.AssertUtil;
import org.apache.falcon.regression.core.util.BundleUtil;
import org.apache.falcon.regression.core.util.HadoopUtil;
+import org.apache.falcon.regression.core.util.OSUtil;
import org.apache.falcon.regression.core.util.MathUtil;
import org.apache.falcon.regression.core.util.OozieUtil;
import org.apache.falcon.regression.core.util.TimeUtil;
@@ -143,7 +144,13 @@ public class RetentionTest extends BaseTestClass {
feedType.addTime(today, -36), feedType.addTime(today, 36), skip, feedType);
final List<String> dataDates = TimeUtil.convertDatesToString(times, feedType.getFormatter());
logger.info("dataDates = " + dataDates);
- HadoopUtil.replenishData(clusterFS, testHDFSDir, dataDates, withData);
+ dataDates.add(HadoopUtil.SOMETHING_RANDOM);
+ if (withData) {
+ HadoopUtil.flattenAndPutDataInFolder(clusterFS, OSUtil.RESOURCES + "log_01.txt",
+ testHDFSDir, dataDates);
+ } else {
+ HadoopUtil.createFolders(clusterFS, testHDFSDir, dataDates);
+ }
}
/**