You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@oozie.apache.org by ro...@apache.org on 2014/03/25 21:36:07 UTC
git commit: OOZIE-1713 Avoid creating dummy input file for each
launcher job (puru via rohini)
Repository: oozie
Updated Branches:
refs/heads/master 359d08d22 -> 4c28591b7
OOZIE-1713 Avoid creating dummy input file for each launcher job (puru via rohini)
Project: http://git-wip-us.apache.org/repos/asf/oozie/repo
Commit: http://git-wip-us.apache.org/repos/asf/oozie/commit/4c28591b
Tree: http://git-wip-us.apache.org/repos/asf/oozie/tree/4c28591b
Diff: http://git-wip-us.apache.org/repos/asf/oozie/diff/4c28591b
Branch: refs/heads/master
Commit: 4c28591b76e1927af3a2f41b4b295ca5cd3eeb00
Parents: 359d08d
Author: Rohini Palaniswamy <ro...@yahoo-inc.com>
Authored: Tue Mar 25 13:35:58 2014 -0700
Committer: Rohini Palaniswamy <ro...@yahoo-inc.com>
Committed: Tue Mar 25 13:35:58 2014 -0700
----------------------------------------------------------------------
.../oozie/action/hadoop/JavaActionExecutor.java | 1 +
.../action/hadoop/LauncherMapperHelper.java | 9 +-
release-log.txt | 1 +
.../action/hadoop/OozieLauncherInputFormat.java | 118 +++++++++++++++++++
4 files changed, 121 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/oozie/blob/4c28591b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
index 860eb51..52a0953 100644
--- a/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/JavaActionExecutor.java
@@ -134,6 +134,7 @@ public class JavaActionExecutor extends ActionExecutor {
public static List<Class> getCommonLauncherClasses() {
List<Class> classes = new ArrayList<Class>();
classes.add(LauncherMapper.class);
+ classes.add(OozieLauncherInputFormat.class);
classes.addAll(Services.get().get(URIHandlerService.class).getClassesForLauncher());
return classes;
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/4c28591b/core/src/main/java/org/apache/oozie/action/hadoop/LauncherMapperHelper.java
----------------------------------------------------------------------
diff --git a/core/src/main/java/org/apache/oozie/action/hadoop/LauncherMapperHelper.java b/core/src/main/java/org/apache/oozie/action/hadoop/LauncherMapperHelper.java
index eeda715..b893200 100644
--- a/core/src/main/java/org/apache/oozie/action/hadoop/LauncherMapperHelper.java
+++ b/core/src/main/java/org/apache/oozie/action/hadoop/LauncherMapperHelper.java
@@ -155,14 +155,7 @@ public class LauncherMapperHelper {
OutputStream os = fs.create(new Path(actionDir, LauncherMapper.ACTION_CONF_XML));
actionConf.writeXml(os);
os.close();
-
- Path inputDir = new Path(actionDir, "input");
- fs.mkdirs(inputDir);
- Writer writer = new OutputStreamWriter(fs.create(new Path(inputDir, "dummy.txt")));
- writer.write("dummy");
- writer.close();
-
- launcherConf.set("mapred.input.dir", inputDir.toString());
+ launcherConf.setInputFormat(OozieLauncherInputFormat.class);
launcherConf.set("mapred.output.dir", new Path(actionDir, "output").toString());
}
http://git-wip-us.apache.org/repos/asf/oozie/blob/4c28591b/release-log.txt
----------------------------------------------------------------------
diff --git a/release-log.txt b/release-log.txt
index 1358824..0fbdde5 100644
--- a/release-log.txt
+++ b/release-log.txt
@@ -1,5 +1,6 @@
-- Oozie 4.1.0 release (trunk - unreleased)
+OOZIE-1713 Avoid creating dummy input file for each launcher job (puru via rohini)
OOZIE-1701 TestXTestCase#testWaitFor and TestXTestCase#testBaseDir have the errors (omaliuvanchuk via rohini)
OOZIE-1751 Default authentication type using env variable for oozie CLI (puru via rohini)
OOZIE-1753 Update create-release-artifact script for git (rkanter)
http://git-wip-us.apache.org/repos/asf/oozie/blob/4c28591b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/OozieLauncherInputFormat.java
----------------------------------------------------------------------
diff --git a/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/OozieLauncherInputFormat.java b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/OozieLauncherInputFormat.java
new file mode 100644
index 0000000..cf43225
--- /dev/null
+++ b/sharelib/oozie/src/main/java/org/apache/oozie/action/hadoop/OozieLauncherInputFormat.java
@@ -0,0 +1,118 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.oozie.action.hadoop;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+import org.apache.hadoop.io.ObjectWritable;
+import org.apache.hadoop.mapred.InputFormat;
+import org.apache.hadoop.mapred.InputSplit;
+import org.apache.hadoop.mapred.JobConf;
+import org.apache.hadoop.mapred.RecordReader;
+import org.apache.hadoop.mapred.Reporter;
+/**
+ * Dummy input format implementation of Oozie launcher jobs. It returns only one record.
+ */
+public class OozieLauncherInputFormat implements InputFormat<Object, Object> {
+
+ boolean isReadingDone = false;
+
+ public RecordReader<Object, Object> getRecordReader(InputSplit arg0, JobConf arg1, Reporter arg2)
+ throws IOException {
+ return new RecordReader<Object, Object>() {
+
+ @Override
+ public void close() throws IOException {
+ }
+
+ @Override
+ public float getProgress() throws IOException {
+ if (isReadingDone) {
+ return 1.0f;
+ }
+ else
+ return 0.0f;
+ }
+
+ @Override
+ public Object createKey() {
+ return new ObjectWritable();
+ }
+
+ @Override
+ public Object createValue() {
+ return new ObjectWritable();
+ }
+
+ @Override
+ public long getPos() throws IOException {
+ if (isReadingDone) {
+ return 1;
+ }
+ else {
+ return 0;
+ }
+ }
+
+ @Override
+ public boolean next(Object arg0, Object arg1) throws IOException {
+ if (isReadingDone) {
+ return false;
+ }
+ else {
+ isReadingDone = true;
+ return true;
+ }
+ }
+
+ };
+ }
+
+ @Override
+ public InputSplit[] getSplits(JobConf arg0, int arg1) throws IOException {
+ return new InputSplit[] { new EmptySplit() };
+ }
+
+ /**
+ * Empty Split implementation.
+ */
+ public static class EmptySplit implements InputSplit {
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ }
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ }
+
+ @Override
+ public long getLength() {
+ return 0L;
+ }
+
+ @Override
+ public String[] getLocations() {
+ return new String[0];
+ }
+ }
+
+}